fetch.py

# -----------------------------------------------------------------------------
# Predbat Home Battery System
# Copyright Trefor Southwell 2026 - All Rights Reserved
# This application maybe used for personal use only and not for commercial use
# -----------------------------------------------------------------------------
# fmt off
# pylint: disable=consider-using-f-string
# pylint: disable=line-too-long
# pylint: disable=attribute-defined-outside-init
# pyright: reportAttributeAccessIssue=false


"""Data fetching module for energy rates, consumption, and forecasts.

Orchestrates loading of import/export energy rates from multiple sources
(Octopus, Energi Data Service, manual config), historical load and PV data,
battery temperature, and other sensor inputs. Converts all data to per-minute
dictionaries for use by the prediction engine.
"""

from datetime import datetime, timedelta
from utils import minutes_to_time, str2time, dp1, dp2, dp3, dp4, time_string_to_stamp, minute_data, get_now_from_cumulative, MinuteArray
from const import (
    MINUTE_WATT,
    PREDICT_STEP,
    TIME_FORMAT,
    PREDBAT_MODE_OPTIONS,
    PREDBAT_MODE_CONTROL_SOC,
    PREDBAT_MODE_CONTROL_CHARGEDISCHARGE,
    PREDBAT_MODE_CONTROL_CHARGE,
    PREDBAT_MODE_MONITOR,
    LOAD_FORECAST_HISTORY_MAX_DAYS,
    PREDBAT_MAX_CARS,
    CAR_CHARGING_LIMIT_UNCAPPED,
    CLOUD_WINDOW_MINUTES,
    CLOUD_ARRAY_MARGIN,
    PV_ARRAY_KWP_UNKNOWN,
)
from predbat_metrics import metrics
from futurerate import FutureRate
from axle import fetch_axle_sessions, load_axle_slot, fetch_axle_active

import copy


class Fetch:
    """Data fetching mixin for loading energy rates, consumption, and forecasts.

    Orchestrates loading of import/export rates from multiple sources,
    historical load/PV data, battery temperature, car charging energy,
    and other sensor inputs. Converts all data to per-minute dictionaries.
    """

    def get_cloud_factor(self, minutes_now, pv_data, pv_data10):
        """
        Work out approximated cloud factor
        """
        pv_total = 0
        pv_total10 = 0
        for minute in range(self.forecast_minutes):
            pv_total += pv_data.get(minute + minutes_now, 0.0)
            pv_total10 += pv_data10.get(minute + minutes_now, 0.0)

        pv_factor = None
        if pv_total > 0 and (pv_total > pv_total10):
            pv_diff = pv_total - pv_total10
            pv_factor = dp2(pv_diff / pv_total) / 2.0
            pv_factor = min(pv_factor, 1.0)

        if self.metric_cloud_enable:
            pv_factor_round = pv_factor
            if pv_factor_round:
                pv_factor_round = dp1(pv_factor_round)
            self.log("PV Forecast {}kWh and 10% Forecast {}kWh; PV cloud factor {}".format(dp1(pv_total), dp1(pv_total10), pv_factor_round))
            return pv_factor
        else:
            return None

    def get_cloud_duty(self, minutes_now, pv_data, pv_data10, pv_data90):
        """Work out how many steps of each conservation window the envelope model spends raised.

        Peaks reach the p90 ceiling and troughs reach the p10 floor together only when the up:down
        step ratio matches the band's own asymmetry, (p50-p10):(p90-p50). Solcast's downside is
        typically about twice its upside, so a fixed 1:1 alternation - all the pairwise model could
        express - always under-reaches on one side.

        Returns (n_up, n_down), or None when there is no upside band to reach for: a forecast source
        that publishes no p90 falls back to a copy of the p50, and the caller must keep the legacy
        proportional model rather than silently losing the cloud model altogether.
        """
        up_gap = 0.0
        down_gap = 0.0
        for minute in range(self.forecast_minutes):
            nominal = pv_data.get(minute + minutes_now, 0.0)
            up_gap += max(pv_data90.get(minute + minutes_now, 0.0) - nominal, 0.0)
            down_gap += max(nominal - pv_data10.get(minute + minutes_now, 0.0), 0.0)

        if up_gap <= 0:
            return None

        period = CLOUD_WINDOW_MINUTES // PREDICT_STEP
        ratio = down_gap / up_gap
        # Clamped to leave at least one step on each side - a window that is all up or all down has
        # nothing to trade against and would leave the borrow outstanding.
        n_up = int(min(max(round(period * ratio / (1.0 + ratio)), 1), period - 1))
        return (n_up, period - n_up)

    def resolve_pv_array_kwp(self, detected_kwp):
        """Settle the DC array size the p90 cloud ceiling is capped against.

        Forecast Solar and Open-Meteo declare kwp per array and fetch_pv_forecast already totals it,
        so most setups need no configuration. Solcast and the HA integrations publish no array size
        and arrive as the PV_ARRAY_KWP_UNKNOWN sentinel, leaving the cap inert - correct, since an
        invented figure would be worse than none. apps.yaml wins over both: declared figures are
        routinely understated, users entering the inverter rating or one string of two.
        """
        configured = self.get_arg("pv_array_kwp", 0.0)
        if configured and configured > 0:
            self.pv_array_kwp = float(configured)
        elif detected_kwp and 0 < detected_kwp < PV_ARRAY_KWP_UNKNOWN:
            self.pv_array_kwp = float(detected_kwp)
        else:
            self.pv_array_kwp = 0.0
        return self.pv_array_kwp

    def get_pv90_cloud_ceiling(self, minutes_now, pv_data, pv_data90):
        """Build the per-minute ceiling the p90 series modulates toward.

        p50 reaches for p90 and p10 reaches for p50, but p90 has no next percentile above it, so its
        upside is extrapolated by continuing the band one width further. Leaving p90 unmodulated
        instead is not neutral: it makes p90 the one smooth scenario, handing it an efficiency
        bonus the other two do not get and biasing every comparison between them.

        The extrapolation is capped at the DC array size times CLOUD_ARRAY_MARGIN, since a very wide
        band can otherwise reach past anything the array could physically produce. The cap can never
        pull the ceiling below p90 itself - that would invert the upside scenario into a downside
        one, which is the same failure mode load_scaling90 is clamped against.
        """
        array_kwp = getattr(self, "pv_array_kwp", 0) or 0
        cap = (array_kwp * CLOUD_ARRAY_MARGIN / 60.0) if array_kwp > 0 else None

        ceiling = {}
        for minute in range(self.forecast_minutes):
            minute_absolute = minute + minutes_now
            upside = pv_data90.get(minute_absolute, 0.0)
            extended = upside + max(upside - pv_data.get(minute_absolute, 0.0), 0.0)
            if cap is not None:
                extended = max(min(extended, cap), upside)
            ceiling[minute_absolute] = extended
        return ceiling

    def filtered_today(self, time_data, resetmidnight=False, stamp=None):
        """
        Grab figure for today (midnight)
        """
        if stamp is None:
            stamp = self.midnight_utc + timedelta(days=1)

        stamp_minus = stamp - timedelta(minutes=PREDICT_STEP)
        if resetmidnight:
            stamp = stamp_minus
        tomorrow_value = time_data.get(stamp.strftime(TIME_FORMAT), time_data.get(stamp_minus.strftime(TIME_FORMAT), None))
        return tomorrow_value

    def filtered_times(self, time_data):
        """
        Filter out duplicate values in time series data
        """

        prev = None
        new_data = {}
        keys = list(time_data.keys())
        for id in range(len(keys)):
            stamp = keys[id]
            value = time_data[stamp]
            next_value = time_data[keys[id + 1]] if id + 1 < len(keys) else None
            if prev is None or value != prev or next_value != value:
                new_data[stamp] = value
            prev = value
            id += 1
        return new_data

    def inday_yesterday_weight(self, minutes_now):
        """
        Weight the previous day's in-day load adjustment carries at a point in the day: full weight for the
        first three hours (too little of today has happened to measure a divergence), then decaying linearly
        to zero by midnight as today's own measurement takes over.
        """
        if minutes_now < 180:
            return 1.0
        return (24 * 60 - minutes_now) / (24 * 60)

    def inday_adjustment_at(self, minute_absolute, scale_today):
        """
        In-day load adjustment factor to apply at a given minute from midnight today.

        Today keeps the factor in full. Tomorrow it decays on exactly the curve load_today_comparison() will
        use when it seeds tomorrow's factor from today's final value, so the plan agrees with what Predbat
        will actually apply a few hours later - resetting to 1.0 at midnight instead left tonight's overnight
        charge sized against a forecast Predbat was about to correct (batpred#4732). Beyond tomorrow there is
        no measurement left to carry and the factor is neutral.
        """
        day_offset = minute_absolute // (24 * 60)
        if day_offset <= 0:
            return scale_today
        if day_offset > 1:
            return 1.0
        return 1.0 + (scale_today - 1.0) * self.inday_yesterday_weight(minute_absolute % (24 * 60))

    def step_data_history(
        self,
        item,
        minutes_now,
        forward,
        step=PREDICT_STEP,
        scale_today=1.0,
        scale_fixed=1.0,
        type_load=False,
        load_forecast=None,
        cloud_factor=None,
        cloud_ceiling=None,
        cloud_duty=None,
        load_scaling_dynamic=None,
        base_offset=None,
        flip=False,
        load_adjust=None,
        load_baseline=None,
    ):
        """
        Create cached step data for historical array
        """
        if load_baseline is None:
            load_baseline = {}
        if load_adjust is None:
            load_adjust = {}
        if load_forecast is None:
            load_forecast = {}
        values = {}
        cloud_diff = 0

        for minute in range(0, self.forecast_minutes + self.plan_interval_minutes, step):
            value = 0
            minute_absolute = minute + minutes_now

            scaling_dynamic = 1.0
            if load_scaling_dynamic:
                scaling_dynamic = load_scaling_dynamic.get(minute_absolute, scaling_dynamic)

            # Carry the in-day adjustment over midnight on the decay curve load_today_comparison() will
            # apply tomorrow, rather than resetting it to 1.0 (batpred#4732)
            scale_slot = self.inday_adjustment_at(minute_absolute, scale_today)

            if type_load and not forward:
                if self.load_forecast_only:
                    load_yesterday, load_yesterday_raw = (0, 0)
                else:
                    load_yesterday, load_yesterday_raw = self.get_filtered_load_minute(item, minute, historical=True, step=step)
                value += load_yesterday
            else:
                for offset in range(step):
                    if forward:
                        value += item.get(minute + minutes_now + offset, 0.0)
                    else:
                        if base_offset:
                            value += self.get_historical_base(item, minute + offset, base_offset)
                        else:
                            value += self.get_historical(item, minute + offset)

            # Extra load adding in (e.g. heat pump)
            load_extra = 0
            if load_forecast:
                for _offset in range(step):
                    load_extra += self.get_from_incrementing(load_forecast, minute_absolute, backwards=False)
            if load_adjust:
                load_extra += load_adjust.get(minute_absolute, 0) * step / float(self.plan_interval_minutes)  # The kWh figure is for the plan interval period, so divide by plan_interval_minutes and times by step
            load_extra = max(load_extra, -value)  # Don't allow going to negative load values
            values[minute] = dp4((value + load_extra) * scaling_dynamic * scale_slot * scale_fixed)

            # Apply dynamic baseline
            if minute_absolute in load_baseline:
                values[minute] = max(values[minute], load_baseline[minute_absolute])

        # Envelope-driven divergence model. Where the legacy model below swings by a fraction of
        # the local value, this one swings toward a real per-minute ceiling - the next forecast
        # percentile up - so the amplitude follows what the forecaster actually said about
        # uncertainty at that time of day rather than one horizon-wide scalar.
        #
        # Conservation is per CLOUD_WINDOW_MINUTES rather than per adjacent pair. That is the whole
        # point of the window: a pair can only trade 1:1, so a trough can never fall further than
        # the peak beside it rose, and with an asymmetric p10/p50/p90 band the peak is the narrow
        # side. Six steps to a window allow an uneven duty cycle (n_up up, n_down down), which lets
        # peaks reach the ceiling while troughs still fall far enough to change the plan.
        if cloud_ceiling and cloud_duty:
            n_up, n_down = cloud_duty
            period = n_up + n_down
            # flip offsets the pattern by n_up so a flipped series lowers exactly where the plain
            # one raises - the antiphase the legacy model got from a 1:1 alternation, generalised
            # to an uneven duty cycle.
            phase = n_up if flip else 0
            for window_start in range(0, self.forecast_minutes, CLOUD_WINDOW_MINUTES):
                offsets = [minute for minute in range(window_start, min(window_start + CLOUD_WINDOW_MINUTES, self.forecast_minutes), step)]
                ups = [minute for minute in offsets if (((minute + minutes_now) // step) + phase) % period < n_up]
                downs = [minute for minute in offsets if minute not in ups]
                if not ups or not downs:
                    continue
                window_total = sum(values[minute] for minute in offsets)
                headroom = {minute: max(sum(cloud_ceiling.get(minute + minutes_now + offset, 0.0) for offset in range(step)) - values[minute], 0.0) for minute in ups}
                want = sum(headroom.values())
                # Each down step can give back at most what it holds, so the window can never be
                # driven negative and never has to leave a borrow outstanding.
                room = sum(max(values[minute] - (load_baseline.get(minute + minutes_now, 0.0) if load_baseline else 0.0), 0.0) for minute in downs)
                borrow = min(want, room)
                if borrow <= 0:
                    continue
                for minute in ups:
                    values[minute] = dp4(values[minute] + headroom[minute] * borrow / want)
                for minute in downs:
                    floor = load_baseline.get(minute + minutes_now, 0.0) if load_baseline else 0.0
                    values[minute] = dp4(values[minute] - borrow * max(values[minute] - floor, 0.0) / room)
                # dp4 on each step leaves up to half a milliwatt-hour per step unaccounted for.
                # Push that residual onto the roomiest down step, which recovers the window total on
                # any window holding meaningful energy - the scenarios are compared against each
                # other, so a systematic drift in one of them is a bias in the plan, not a rounding
                # detail. It is not guaranteed: the correction is floored at zero, so a window whose
                # roomiest down step holds less than the residual keeps the remainder. In practice
                # that is confined to near-empty dawn and dusk windows and measures around 0.001 kWh
                # over a 48 hour horizon.
                drift = sum(values[minute] for minute in offsets) - window_total
                if drift:
                    roomiest = max(downs, key=lambda minute: values[minute])
                    values[roomiest] = dp4(max(values[roomiest] - drift, 0.0))

        # Simple divergence model that raises and lowers alternate steps so the planner stops
        # assuming PV exactly blankets load. It is approximately - not exactly - total-preserving:
        # a raised step borrows, and the following step repays only what its own value can cover
        # (`min(cloud_diff, values[minute])`), so a borrow larger than the step that follows it is
        # not fully given back. Repaying exactly needs the outstanding debt tracked separately from
        # the next borrow, which is a rewrite of the model rather than a tweak here.
        if cloud_factor and cloud_factor > 0:
            last_minute = self.forecast_minutes - step
            for minute in range(0, self.forecast_minutes, step):
                # The inner parentheses matter: a conditional expression binds looser than "+", so
                # "int(...) + 1 if flip else 0" is "(int(...) + 1) if flip else 0" - which is a
                # constant 0 whenever flip is False, leaving every non-flipped caller unmodulated.
                # flip only means anything as an antiphase to a modulated base, so it is the offset
                # that is conditional, not the whole expression.
                #
                # Keyed off the minutes_now parameter, not self.minutes_now: every other minute in
                # this function comes from the parameter, and output.py already calls it with 0.
                # That also keeps the result a pure function of the inputs, which the replay and
                # test paths rely on for determinism.
                cloud_on = (int((minute + minutes_now) / step) + (1 if flip else 0)) % 2
                if cloud_on > 0:
                    # Don't borrow on the final step - the fill loop above runs a plan interval
                    # past forecast_minutes, so values.get(minute + step) finds a real bucket that
                    # this loop will never reach to repay, leaking that borrow into the total.
                    if minute < last_minute:
                        cloud_diff += min(values[minute] * cloud_factor, values.get(minute + step, 0) * cloud_factor)
                        values[minute] += cloud_diff
                else:
                    # The dynamic load baseline is a floor, applied in the fill loop above. The
                    # subtract has to respect it or the modulation walks straight back through it -
                    # only ever an issue since the load arrays started being modulated at all.
                    floor = load_baseline.get(minute + minutes_now, 0.0) if load_baseline else 0.0
                    subtract = min(cloud_diff, max(values[minute] - floor, 0.0))
                    values[minute] -= subtract
                    cloud_diff = 0
                values[minute] = dp4(values[minute])

        return values

    def get_filtered_load_window(self, data, indices, step):
        """
        Compute the car-charging/iBoost-filtered load for a single window given its list of backwards minute
        indices. Returns (filtered_load, raw_load) for that one window (no base-load applied).
        """
        raw = 0.0
        subtract_energy = 0.0
        for idx in indices:
            raw += self.get_from_incrementing(data, idx)
            if self.car_charging_hold and self.car_charging_energy:
                subtract_energy += self.get_from_incrementing(self.car_charging_energy, idx)
            if self.iboost_energy_subtract and self.iboost_energy_today:
                subtract_energy += self.get_from_incrementing(self.iboost_energy_today, idx)
        load = max(0.0, raw - subtract_energy)
        if self.car_charging_hold and (not self.car_charging_energy) and (load >= (self.car_charging_threshold * step)):
            # Car charging hold - ignore car charging in this window based on the threshold
            load = max(load - (self.car_charging_rate[0] * step / 60.0), 0)
        return load, raw

    def get_filtered_load_minute(self, data, minute_previous, historical, step=1, base_in_raw=True):
        """
        Gets a previous load minute after filtering for car charging.

        For historical (days_previous) data the car-charging hold and car/iBoost subtraction are applied to
        each previous day individually and only then weighted-averaged, so a single day's car charging is not
        diluted below the hold threshold by averaging it with the other (lower-load) days first.

        base_in_raw controls whether the base-load top-up is reflected in the returned raw value. Set it to
        False to keep the raw value at the true measured load so callers can detect genuine zero/gap buckets.
        """
        if historical:
            total = 0.0
            total_raw = 0.0
            total_weight = 0.0
            for point, days in enumerate(self.days_previous):
                use_days = max(min(days, self.load_minutes_age), 1)
                weight = self.days_previous_weight[point]
                full_days = 24 * 60 * (use_days - 1)
                indices = [24 * 60 - (minute_previous + offset) + full_days for offset in range(step)]
                day_load, day_raw = self.get_filtered_load_window(data, indices, step)
                total += day_load * weight
                total_raw += day_raw * weight
                total_weight += weight
            if total_weight > 0:
                load_yesterday = total / total_weight
                load_yesterday_raw = total_raw / total_weight
            else:
                load_yesterday = 0.0
                load_yesterday_raw = 0.0
        else:
            indices = [minute_previous + offset for offset in range(step)]
            load_yesterday, load_yesterday_raw = self.get_filtered_load_window(data, indices, step)

        # Apply base load
        base_load = self.base_load * step / 60.0
        if load_yesterday < base_load:
            add_to_base = base_load - load_yesterday
            load_yesterday += add_to_base
            if base_in_raw:
                load_yesterday_raw += add_to_base

        return load_yesterday, load_yesterday_raw

    def fill_load_from_power(self, load_minutes, load_power_data):
        """
        This function helps to deal with load sensors which don't increment very often leading to poor quality load data
        For example if its in kWh units then it might take many hours to increment.
        Predbat actually wants a much more real time estimate, but integrating power sensors alone leads to drift over time.

        Strategy: Divide data into 30-minute periods. For each period:
        1. Calculate total load consumed (difference between start and end of period)
        2. Integrate power data over that period
        3. Scale power data to match the total load
        4. Generate smooth minute-by-minute load curve

        Since data goes backwards in time (minute 0 is now, higher minutes are further in past),
        the load value DECREASES as we go forward in time (backwards through minutes).
        """

        if not load_power_data:
            self.log("Warn: No power data provided to fill_load_from_power")
            return load_minutes

        # Create a copy to avoid modifying the original
        new_load_minutes = load_minutes.copy()

        # Find all the minutes we have load data for.
        # Deliberately do NOT extend max_minute using load_power_data.keys() even if the power
        # sensor has older history than the load sensor.  Any minutes beyond the load data range
        # have no cumulative energy readings and would all default to 0, which the zero-period
        # detection incorrectly treats as a full-day data gap every run.
        max_minute = max(load_minutes.keys()) if load_minutes else 0

        # Determine gap size threshold for zero-load detection
        # This uses the same threshold as the gap filling logic
        gap_size = max(self.get_arg("load_filter_threshold", self.plan_interval_minutes), 5)

        # Preprocessing: Fill periods of zero load where power data exists
        # Find zero-load periods by looking for consecutive minutes with the same value (or 0)
        zero_periods = []
        period_start = None
        last_value = None

        for minute in range(0, max_minute + 1):
            current_value = new_load_minutes.get(minute, 0)

            # Check if this is a zero or constant period
            if minute == 0:
                last_value = current_value
                if current_value == 0 or current_value == new_load_minutes.get(minute + 1, 0):
                    period_start = minute
            else:
                # If value is same as previous and same as next, it's a flat period
                next_value = new_load_minutes.get(minute + 1, current_value)
                if current_value == last_value and current_value == next_value:
                    if period_start is None:
                        period_start = minute
                else:
                    # End of constant period
                    if period_start is not None and period_start < minute:
                        period_length = minute - period_start
                        # Only consider it zero load if it exceeds the gap_size threshold
                        if period_length >= gap_size:
                            # Check if there's power data in this period
                            has_power = any(load_power_data.get(m, 0) > 0 for m in range(period_start, minute))
                            if has_power and (last_value == 0 or last_value == new_load_minutes.get(minute, 0)):
                                zero_periods.append((period_start, minute - 1, last_value))
                    period_start = None
                last_value = current_value

        # Check final period
        if period_start is not None and period_start < max_minute:
            period_length = max_minute - period_start + 1
            # Only consider it zero load if it exceeds the gap_size threshold
            if period_length >= gap_size:
                has_power = any(load_power_data.get(m, 0) > 0 for m in range(period_start, max_minute + 1))
                if has_power and last_value == 0:
                    zero_periods.append((period_start, max_minute, last_value))

        # Fill zero periods with integrated power data
        if zero_periods:
            self.log("Warn: Found {} periods of zero load with power data, filling using power integration".format(len(zero_periods)))
            # Print the first 5 periods for debugging
            for _i, (period_start, period_end, base_value) in enumerate(zero_periods[:5]):
                start_timestamp = self.now_utc - timedelta(minutes=period_start)
                self.log("Zero load period starting at {} ({} minutes) for {} minutes with base value {}".format(start_timestamp.strftime(TIME_FORMAT), period_start, period_end - period_start + 1, base_value))

            for period_start, period_end, _base_value in zero_periods:
                # Integrate power data over this period
                # First calculate total energy consumed in this period
                total_energy = 0
                for minute in range(period_start, period_end + 1):
                    power = load_power_data.get(minute, 0)
                    total_energy += power / 60.0 / 1000.0

                amount_to_fill = 0
                for minute in range(period_end, period_start, -1):
                    power = max(load_power_data.get(minute, 0), 0)
                    energy = power / 60.0 / 1000.0
                    amount_to_fill += energy
                    new_load_minutes[minute] = new_load_minutes.get(minute, 0) + amount_to_fill
                # Fill to start
                for minute in range(period_start, -1, -1):
                    new_load_minutes[minute] = new_load_minutes.get(minute, 0) + amount_to_fill

        # Process in 30-minute periods
        period_length = 30
        num_periods = (max_minute + period_length) // period_length

        self.log("Processing {} 30-minute periods for power integration".format(num_periods))
        for period_idx in range(num_periods):
            period_start = period_idx * period_length
            period_end = min(period_start + period_length - 1, max_minute)

            # Get load at start and end of period (going backwards in time)
            # period_start is more recent (higher cumulative value)
            # period_end is further in past (lower cumulative value)
            load_at_start = new_load_minutes.get(period_start, 0)
            load_at_end = new_load_minutes.get(period_end + 1, new_load_minutes.get(period_end, 0))

            # Total energy consumed in this period (going backwards means decrement)
            load_total = load_at_start - load_at_end

            # Integrate power data over this period (convert W to kWh)
            integrated_energy = 0.0
            power_count = 0
            for minute in range(period_start, period_end + 1):
                power = max(load_power_data.get(minute, 0), 0)
                if power > 0:
                    power_count += 1
                # Convert watts to kWh per minute: W * (1 hour / 60 minutes) / 1000
                integrated_energy += power / 60.0 / 1000.0

            # If we have both power data and load consumption, scale the power integration
            if integrated_energy > 0 and load_total > 0:
                # Calculate scaling factor to match load total
                scale_factor = load_total / integrated_energy

                # Apply scaled power integration to create smooth load curve
                # Start from the highest value (at period_start = most recent)
                # and decrement going backwards in time
                running_total = load_at_start
                for minute in range(period_start, period_end + 1):
                    new_load_minutes[minute] = dp4(running_total)
                    power = max(load_power_data.get(minute, 0), 0)
                    energy_decrement = (power / 60.0 / 1000.0) * scale_factor
                    running_total -= energy_decrement
            elif load_total > 0:
                # No power data but we have load consumption
                # Distribute the load evenly across the period
                energy_per_minute = load_total / (period_end - period_start + 1)
                running_total = load_at_start
                for minute in range(period_start, period_end + 1):
                    new_load_minutes[minute] = dp4(running_total)
                    running_total -= energy_per_minute

        return new_load_minutes

    def previous_days_modal_filter(self, data):
        """
        Look at the data from previous days and discard the best case one
        """

        total_points = len(self.days_previous)
        sum_days = []
        sum_days_id = {}
        min_sum = 99999999
        min_sum_day = 0
        sum_all_days = {}

        days_list = self.days_previous.copy()
        # Sort days list in numerical order with highest number day first
        days_list.sort(reverse=True)
        max_days = min(max(days_list), self.load_minutes_age) if self.load_minutes_age > 0 else max(days_list)
        for days in range(1, max_days + 1):
            sum_day = 0
            full_days = 24 * 60 * (days - 1)
            for minute in range(0, 24 * 60, PREDICT_STEP):
                minute_previous = 24 * 60 - minute + full_days - 1
                load_yesterday, load_yesterday_raw = self.get_filtered_load_minute(data, minute_previous, historical=False, step=PREDICT_STEP)
                sum_day += load_yesterday
            if days in days_list:
                sum_days.append(dp2(sum_day))
                sum_days_id[days] = sum_day
                if sum_day < min_sum:
                    min_sum_day = days
                    min_sum = dp2(sum_day)
            sum_all_days[days] = dp2(sum_day)

        # Work out the average non-zero day
        average_non_zero_day = 0
        average_non_zero_count = 0
        for day_sum in sum_all_days.values():
            # Sensible threshold to ignore zero days
            if day_sum > 2.5:
                average_non_zero_day += day_sum
                average_non_zero_count += 1
        if average_non_zero_count > 0:
            average_non_zero_day /= average_non_zero_count
        else:
            average_non_zero_day = 24  # Assume a nominal 24kWh day if no data

        # log the unsorted list of days and load kWh, easier to read and ensures correct 'lowest day' is reported
        self.log("Historical load totals for days {} are {}kWh, minimum value {}kWh".format(self.days_previous.copy(), sum_days, min_sum))
        if self.load_filter_modal and total_points >= 3 and (min_sum_day > 0):
            self.log("Modal filter enabled - Discarding day {} as it is the lowest of the {} datapoints".format(min_sum_day, len(days_list)))
            min_sum_day_idx = days_list.index(min_sum_day)
            del days_list[min_sum_day_idx]
            # Remove day 'min_sum_day' from self.days_previous
            min_sum_day_idx = self.days_previous.index(min_sum_day)
            del self.days_previous[min_sum_day_idx]
            del self.days_previous_weight[min_sum_day_idx]

        # Fill all the gaps including days we don't use
        gap_size = max(self.get_arg("load_filter_threshold", self.plan_interval_minutes), 5)
        gap_minutes = 0
        gap_start_minute_previous = None
        gap_list = []
        num_gaps = 0
        max_minute = max(max_days * 24 * 60, max(data.keys()) if data else 0)

        # Find all the gaps
        for minute_previous in range(0, max_minute, PREDICT_STEP):
            if data.get(minute_previous, 0) == data.get(minute_previous + PREDICT_STEP, 0):
                gap_minutes += PREDICT_STEP
                if gap_start_minute_previous is None:
                    gap_start_minute_previous = minute_previous
            else:
                if gap_minutes >= gap_size:
                    num_gaps += gap_minutes
                    gap_list.append((gap_start_minute_previous, gap_minutes))
                gap_minutes = 0
                gap_start_minute_previous = None
        if gap_minutes >= gap_size:
            num_gaps += gap_minutes
            gap_list.append((gap_start_minute_previous, gap_minutes))

        # Work out total number of gap_minutes
        if num_gaps > 0:
            self.log("Warn: Found {} gaps in load_today totalling {} minutes to fill using average data".format(len(gap_list), num_gaps))
            # Print the first 5 gaps for debugging
            for _i, (gap_start, gap_length) in enumerate(gap_list[:5]):
                gap_start_timestamp = self.now_utc - timedelta(minutes=gap_start)
                self.log("Gap starting at {} ({} minutes) for {} minutes".format(gap_start_timestamp.strftime(TIME_FORMAT), gap_start, gap_length))

        # Do the filling
        len_data = len(data) if isinstance(data, MinuteArray) else 99999999
        for gap in gap_list:
            gap_start_minute_previous = gap[0]
            gap_minutes = gap[1]
            gap_end_minute_previous = gap_start_minute_previous + gap_minutes

            total_to_add = 0
            # Fill the gap region by stepping backward through time (decrementing minute_previous)
            # gap_start_minute_previous is the highest index (earliest in gap)
            # We fill from there down to the end of the gap

            minute_previous = min(gap_end_minute_previous, len_data - 1)
            gap_day = None
            while minute_previous > gap_start_minute_previous and minute_previous >= 0:
                # Change of day?
                new_gap_day = max(minute_previous // (24 * 60), 1)
                if gap_day is None or (new_gap_day != gap_day):
                    gap_day = new_gap_day
                    average_day = sum_all_days.get(gap_day, average_non_zero_day)
                    if average_day <= 2.5:
                        average_day = average_non_zero_day
                    per_minute_increment = average_day / (24 * 60)
                    gap_day = new_gap_day

                data[minute_previous] = dp4(data.get(minute_previous, 0) + total_to_add)
                minute_previous -= 1
                total_to_add += per_minute_increment

            # Now ensure the sensor is incrementing all the way to 0
            while minute_previous >= 0:
                data[minute_previous] = dp4(data.get(minute_previous, 0) + total_to_add)
                minute_previous -= 1

        return data

    def get_historical_base(self, data, minute, base_minutes):
        """
        Get historical data from base minute ago
        """
        # No data?
        if not data:
            return 0

        minute_previous = base_minutes - minute
        return self.get_from_incrementing(data, minute_previous)

    def get_historical(self, data, minute):
        """
        Get historical data across N previous days in days_previous array based on current minute
        """
        total = 0
        total_weight = 0
        this_point = 0

        # No data?
        if not data:
            return 0

        for days in self.days_previous:
            use_days = max(min(days, self.load_minutes_age), 1)
            weight = self.days_previous_weight[this_point]
            full_days = 24 * 60 * (use_days - 1)
            minute_previous = 24 * 60 - minute + full_days
            value = self.get_from_incrementing(data, minute_previous)
            total += value * weight
            total_weight += weight
            this_point += 1

        # Zero data?
        if total_weight == 0:
            return 0
        else:
            return total / total_weight

    def get_from_incrementing(self, data, index, backwards=True):
        """
        Get a single value from an incrementing series e.g. kWh today -> kWh this minute
        """
        while index < 0:
            index += 24 * 60
        if backwards:
            return max(data.get(index, 0) - data.get(index + 1, 0), 0)
        else:
            return max(data.get(index + 1, 0) - data.get(index, 0), 0)

    def minute_data_import_export(self, max_days_previous, now_utc, key, scale=1.0, required_unit=None, increment=True, smoothing=True, pad=True, required=True):
        """
        Download one or more entities for import/export data

        :param required: Set False when the data only improves the result rather than being needed for it,
                         e.g. the rate history the ML load model adds as a training feature. A missing
                         history is then reported once as information instead of as a fetch failure, as
                         Home Assistant legitimately has no history for entities its recorder isn't storing.
        """
        if "." not in key:
            entity_ids = self.get_arg(key, indirect=False)
        else:
            entity_ids = key

        if isinstance(entity_ids, str):
            entity_ids = [entity_ids]
        if entity_ids is None:
            self.log("Error: No entity IDs provided for {}".format(key))
            entity_ids = []

        import_today = {}
        for entity_id in entity_ids:
            # Ignore invalid entity IDs, they might be fixed values
            if (not entity_id) or ("." not in entity_id):
                continue

            try:
                history = self.get_history_wrapper(entity_id=entity_id, days=max_days_previous, required=required)
            except (ValueError, TypeError) as exc:
                self.log("Warn: No history data found for {} : {}".format(entity_id, exc))
                history = []

            if history and len(history) > 0 and len(history[0]) > 0:
                item = history[0][0]
                try:
                    last_updated_time = str2time(item["last_updated"])
                except (ValueError, TypeError):
                    last_updated_time = now_utc
                age_days = max_days_previous
                age = now_utc - last_updated_time
                if age_days is None:
                    age_days = age.days
                else:
                    age_days = min(age_days, age.days)

                import_today, _ = minute_data(
                    history[0],
                    max_days_previous if pad else age_days,
                    now_utc,
                    "state",
                    "last_updated",
                    backwards=True,
                    smoothing=smoothing,
                    scale=scale,
                    clean_increment=increment,
                    accumulate=import_today,
                    required_unit=required_unit,
                    can_modify_history=True,  # history is not accessed after this point, so minute_data can freely modify it
                )
            else:
                if not required:
                    # Optional data - Home Assistant simply has no history for this entity, which is
                    # normal when its recorder isn't configured to store it, so don't cry wolf
                    self.log("Info: No history available for {}, continuing without it".format(entity_id))
                elif history is None:
                    # Only record as a failure if it was None (not just empty but failure)
                    self.log("Warn: Failure to fetch history for {}".format(entity_id))
                    self.record_status("Warn: Failure to fetch history from {}".format(entity_id), had_errors=True)
                else:
                    self.log("Warn: Unable to fetch history for {}".format(entity_id))

        if import_today and smoothing:
            size = (max_days_previous * 24 * 60 + 2) if pad else (max(import_today.keys()) + 2)
            return MinuteArray(import_today, size)
        return import_today

    def minute_data_load(self, now_utc, entity_name, max_days_previous, load_scaling=1.0, required_unit=None, interpolate=False, pad=True, clean_increment=True):
        """
        Download one or more entities for load data
        """
        if "." not in entity_name:
            entity_ids = self.get_arg(entity_name, indirect=False)
        else:
            entity_ids = entity_name

        if isinstance(entity_ids, str):
            entity_ids = [entity_ids]
        if entity_ids is None:
            self.log("Error: No entity IDs provided for {}".format(entity_name))
            entity_ids = []

        load_minutes = {}
        age_days = None
        for entity_id in entity_ids:
            if (not entity_id) or ("." not in entity_id):
                # Invalid entity ID, might be a fixed value
                continue

            try:
                history = self.get_history_wrapper(entity_id=entity_id, days=max_days_previous)
            except (ValueError, TypeError):
                history = []

            if isinstance(history, list) and history and history[0]:
                item = history[0][0]
                try:
                    last_updated_time = str2time(item["last_updated"])
                except (ValueError, TypeError):
                    last_updated_time = now_utc
                age = now_utc - last_updated_time
                if age_days is None:
                    age_days = age.days
                else:
                    age_days = min(age_days, age.days)
                load_minutes, _ = minute_data(
                    history[0],
                    max_days_previous if pad else age_days,
                    now_utc,
                    "state",
                    "last_updated",
                    backwards=True,
                    smoothing=True,
                    scale=load_scaling,
                    clean_increment=clean_increment,
                    accumulate=load_minutes,
                    required_unit=required_unit,
                    interpolate=interpolate,
                    can_modify_history=True,  # history is not accessed after this point, so minute_data can freely modify it
                )
            else:
                if history is None:
                    # Only record as a failure if it was None (not just empty but failure)
                    self.log("Warn: Failure to fetch history for {}".format(entity_id))
                    self.record_status("Warn: Failure to fetch history from {}".format(entity_id), had_errors=True)
                else:
                    self.log("Warn: Unable to fetch history for {}".format(entity_id))

        if age_days is None:
            age_days = 0
        if load_minutes:
            size = (max_days_previous * 24 * 60 + 2) if pad else (max(load_minutes.keys()) + 2)
            load_minutes = MinuteArray(load_minutes, size)
        return load_minutes, age_days

    def fetch_pv_forecast_and_dawn(self):
        """
        Fetch the PV forecast, compute the dawn light/dark split from it, and publish the dawn
        binary_sensor - as one call so the ordering that #4699 regressed on (the split must be
        computed from a freshly-fetched forecast, not the empty dict fetch_sensor_data() resets it
        to at the top of each cycle) can't drift apart again. Published unconditionally regardless
        of whether import rates currently exist, so the sensor never goes stale.

        Returns:
            pv_light_dark: dict as returned by calc_pv_light_dark(), also stored on self.pv_light_dark.
        """
        self.pv_forecast_minute, self.pv_forecast_minute10, self.pv_forecast_minute90 = self.fetch_pv_forecast()

        # Stored on self, not just local, so it can be used elsewhere rather than only by the
        # window split below.
        pv_light_dark = self.pv_light_dark = self.calc_pv_light_dark()
        # "on" past dawn, "off" before it or when unclassified (no PV forecast) - based on the PV
        # forecast crossing low_power_pv_threshold_w, not the same as PV actually producing right
        # now, see calc_dawn's docstring. Independent of combine_charge_slots/set_charge_low_power.
        self.dashboard_item(
            "binary_sensor." + self.prefix + "_dawn",
            state="on" if pv_light_dark.get(self.minutes_now) == 1 else "off",
            attributes={"friendly_name": "Predbat is past dawn (light, not dark, by PV forecast)", "icon": "mdi:weather-sunset-up"},
        )
        return pv_light_dark

    def combine_active_keep(self):
        """
        Combine the SOC keep floors (alerts, manual_soc) and ceilings (manual_soc_max) into all_active_keep/all_active_keep_max.
        """
        # Combine keep from alerts and manual SOC into all_active_keep
        self.all_active_keep = self.alert_active_keep.copy()
        if self.manual_soc_keep:
            for minute, soc_value in self.manual_soc_keep.items():
                if minute in self.all_active_keep:
                    self.all_active_keep[minute] = max(self.all_active_keep[minute], soc_value)
                else:
                    self.all_active_keep[minute] = soc_value

        # Manual SOC max is a ceiling rather than a floor - combine separately, taking the
        # tightest (lowest) ceiling if more than one source ever applies to the same minute.
        self.all_active_keep_max = {}
        if self.manual_soc_max_keep:
            for minute, soc_value in self.manual_soc_max_keep.items():
                if minute in self.all_active_keep_max:
                    self.all_active_keep_max[minute] = min(self.all_active_keep_max[minute], soc_value)
                else:
                    self.all_active_keep_max[minute] = soc_value

        # A ceiling below the floor at the same minute is a contradiction (e.g. a leftover manual_soc
        # override never cleared) - the floor wins as the safety-relevant constraint, so drop the
        # conflicting ceiling rather than hand the optimiser two penalties pulling opposite ways.
        for minute in list(self.all_active_keep_max.keys()):
            floor_value = self.all_active_keep.get(minute, 0)
            if floor_value > self.all_active_keep_max[minute]:
                self.log("Warn: manual_soc_max target {}% at minute {} is below the manual_soc/alert floor {}% for the same minute - ignoring the ceiling there".format(self.all_active_keep_max[minute], minute, floor_value))
                del self.all_active_keep_max[minute]

    def fetch_sensor_data(self, save=True):
        """
        Fetch all the data, e.g. energy rates, load, PV predictions, car plan etc.
        """

        prev_octopus_slots = self.octopus_slots.copy()
        prev_octopus_saving_slots = self.octopus_saving_slots.copy()
        prev_octopus_free_slots = self.octopus_free_slots.copy()
        prev_axle_sessions = self.axle_sessions.copy()

        import_rates = {}
        self.rate_import_replicated = {}
        export_rates = {}
        self.rate_export_replicated = {}
        self.rate_slots = []
        self.io_adjusted = {}
        self.low_rates = []
        self.high_export_rates = []
        self.octopus_slots = [[] for _ in range(self.num_cars)]
        self.cost_today_sofar = 0
        self.carbon_today_sofar = 0
        self.import_today = {}
        self.export_today = {}
        self.battery_temperature_history = {}
        self.battery_temperature_prediction = {}
        self.pv_today = {}
        self.load_minutes = {}
        self.load_minutes_age = 0
        self.load_forecast = {}
        self.load_forecast_array = []
        self.pv_forecast_minute = {}
        self.pv_forecast_minute10 = {}
        self.pv_forecast_minute90 = {}
        self.pv_light_dark = {}
        # See Plan.refresh_pv_forecast_minute90(): both series are re-fetched together below, so no
        # earlier pair of signatures may be held against them
        self.pv_forecast_minute90_signatures = None
        self.load_scaling_dynamic = {}
        self.carbon_intensity = {}
        self.carbon_history = {}
        curr = self.currency_symbols[1]

        # Alert feed if enabled
        if self.components:
            alert_feed = self.components.get_component("alert_feed")
            if alert_feed:
                self.alerts, self.alert_active_keep = alert_feed.process_alerts(self.minutes_now, self.midnight_utc)

        # Combine keep floors and ceilings from alerts and manual SOC
        self.combine_active_keep()

        # iBoost load data
        if "iboost_energy_today" in self.args:
            self.iboost_energy_today, iboost_energy_age = self.minute_data_load(self.now_utc, "iboost_energy_today", self.max_days_previous, required_unit="kWh", load_scaling=1.0)
            if iboost_energy_age >= 1:
                self.iboost_today = dp2(abs(self.iboost_energy_today[0] - self.iboost_energy_today[self.minutes_now]))
                self.log("iBoost energy today from sensor reads {} kWh".format(self.iboost_today))

        # Fetch ML forecast if enabled
        load_ml_forecast = {}
        if self.get_arg("load_ml_enable", False) and self.get_arg("load_ml_source", False):
            load_ml_forecast = self.fetch_ml_load_forecast(self.now_utc)
            if load_ml_forecast:
                self.load_forecast_only = True  # Use only ML forecast for load if enabled and we have data

        # Fetch extra load forecast
        self.load_forecast, self.load_forecast_array = self.fetch_extra_load_forecast(self.now_utc, load_ml_forecast)

        # Load previous load data
        if self.get_arg("ge_cloud_data", False):
            self.download_ge_data(self.now_utc)

            if ("load_power" in self.args) and self.get_arg("load_power_fill_enable", True):
                self.log("Using load_power data to fill gaps in load_today data")
                load_power_data, _ = self.minute_data_load(self.now_utc, "load_power", self.max_days_previous, required_unit="W", load_scaling=1.0, interpolate=True, clean_increment=False)
                self.load_minutes = self.fill_load_from_power(self.load_minutes, load_power_data)
        else:
            # Load data
            if "load_today" in self.args:
                self.load_minutes, self.load_minutes_age = self.minute_data_load(self.now_utc, "load_today", self.max_days_previous, required_unit="kWh", load_scaling=1.0, interpolate=True)
                self.log("Found {} load_today datapoints going back {} days".format(len(self.load_minutes), self.load_minutes_age))
                self.load_minutes_now = get_now_from_cumulative(self.load_minutes, self.minutes_now, backwards=True)
                self.load_last_period = (self.load_minutes.get(0, 0) - self.load_minutes.get(PREDICT_STEP, 0)) * 60 / PREDICT_STEP

                if ("load_power" in self.args) and self.get_arg("load_power_fill_enable", True):
                    # Use power data to make load data more accurate.
                    # clean_increment=False: power sensors report instantaneous W, not cumulative kWh.
                    # clean_incrementing_reverse would distort fluctuating power readings into an
                    # ever-growing cumulative series, inflating fill_load_from_power gap-fills.
                    self.log("Using load_power data to fill gaps in load_today data")
                    load_power_data, _ = self.minute_data_load(self.now_utc, "load_power", self.max_days_previous, required_unit="W", load_scaling=1.0, interpolate=True, clean_increment=False)
                    self.load_minutes = self.fill_load_from_power(self.load_minutes, load_power_data)
            else:
                if self.load_forecast:
                    self.log("Using load forecast from load_forecast sensor")
                    self.load_minutes_now = self.load_forecast.get(0, 0)
                    self.load_minutes_age = 0
                    self.load_last_period = 0
                else:
                    self.log("Error: You have not set load_today or load_forecast in apps.yaml, you will have no load data")
                    self.record_status(message="Error: load_today not set correctly", had_errors=True)
                    raise ValueError

            # Load import today data
            if "import_today" in self.args:
                self.import_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "import_today", scale=self.import_export_scaling, required_unit="kWh")
                self.import_today_now = get_now_from_cumulative(self.import_today, self.minutes_now, backwards=True)
            else:
                self.log("Warn: You have not set import_today in apps.yaml, you will have no previous import data")

            # Load export today data
            if "export_today" in self.args:
                self.export_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "export_today", scale=self.import_export_scaling, required_unit="kWh")
                self.export_today_now = get_now_from_cumulative(self.export_today, self.minutes_now, backwards=True)
            else:
                self.log("Warn: You have not set export_today in apps.yaml, you will have no previous export data")

            # PV today data
            if "pv_today" in self.args:
                self.pv_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "pv_today", required_unit="kWh")
                self.pv_today_now = get_now_from_cumulative(self.pv_today, self.minutes_now, backwards=True)
            else:
                self.log("Warn: You have not set pv_today in apps.yaml, you will have no previous PV data")

        # Battery temperature
        if "battery_temperature_history" in self.args:
            self.battery_temperature_history = self.minute_data_import_export(self.max_days_previous, self.now_utc, "battery_temperature_history", scale=1.0, increment=False, smoothing=False)
            data = []
            for minute in range(0, 24 * 60, 5):
                data.append({minute: self.battery_temperature_history.get(minute, 0)})
            self.battery_temperature_prediction = self.predict_battery_temperature(self.battery_temperature_history, step=PREDICT_STEP)
            self.log("Fetched battery temperature history data, current temperature {}°C".format(self.battery_temperature_history.get(0, None)))

        # Car charging hold - when enabled battery is held during car charging in simulation
        self.car_charging_energy = self.load_car_energy(self.now_utc)

        # Log current values
        self.log("Current data so far today: load {}kWh, import {}kWh, export {}kWh, PV {}kWh".format(dp2(self.load_minutes_now), dp2(self.import_today_now), dp2(self.export_today_now), dp2(self.pv_today_now)))
        m = metrics()
        m.load_today_kwh.set(self.load_minutes_now)
        m.import_today_kwh.set(self.import_today_now)
        m.export_today_kwh.set(self.export_today_now)
        m.pv_today_kwh.set(self.pv_today_now)
        m.data_age_days.set(self.load_minutes_age)
        m.data_age_required_days.set(max(self.max_days_previous - 1, 0))

        if "rates_import_octopus_url" in self.args:
            # Fixed URL for rate import
            self.log("Downloading import rates directly from URL {}".format(self.get_arg("rates_import_octopus_url", indirect=False)))
            # Need to take a copy, as saving sessions will repeatedly increment cached import rates
            import_rates = copy.deepcopy(self.download_octopus_rates(self.get_arg("rates_import_octopus_url", indirect=False)))
        elif "metric_octopus_import" in self.args:
            # Octopus import rates
            entity_id = self.get_arg("metric_octopus_import", None, indirect=False)
            import_rates = self.fetch_octopus_rates(entity_id, adjust_key="is_intelligent_adjusted")
            if not import_rates:
                self.log("Error: metric_octopus_import is not set correctly in apps.yaml, or no energy rates can be read")
                self.record_status(message="Error: metric_octopus_import not set correctly in apps.yaml, or no energy rates can be read", had_errors=True)
        elif "metric_energidataservice_import" in self.args:
            # Energi Data Service import rates
            entity_id = self.get_arg("metric_energidataservice_import", None, indirect=False)
            import_rates = self.fetch_energidataservice_rates(entity_id, adjust_key="is_intelligent_adjusted")
            if not import_rates:
                self.log("Error: metric_energidataservice_import is not set correctly in apps.yaml, or no energy rates can be read")
                self.record_status(message="Error: metric_energidataservice_import not set correctly in apps.yaml, or no energy rates can be read", had_errors=True)
        elif "metric_stromligning_import_today" in self.args or "metric_stromligning_import_tomorrow" in self.args:
            # Strømligning import rates
            entity_id_today = self.get_arg("metric_stromligning_import_today", None, indirect=False)
            entity_id_tomorrow = self.get_arg("metric_stromligning_import_tomorrow", None, indirect=False)
            import_rates = self.fetch_stromligning_rates(entity_id_today, entity_id_tomorrow, adjust_key="is_intelligent_adjusted")
            if not import_rates:
                self.log("Error: metric_stromligning_import sensors are not set correctly or no energy rates can be read")
                self.record_status(message="Error: metric_stromligning_import sensors not set correctly or no energy rates can be read", had_errors=True)

        # Fallback if no other rate types are set
        if not import_rates:
            # Basic rates defined by user over time
            rate_import_dict = self.get_arg("rates_import", [], indirect=False)
            if rate_import_dict:
                import_rates = self.basic_rates(rate_import_dict, "rates_import")

        # Gas rates if set
        if "metric_octopus_gas" in self.args:
            entity_id = self.get_arg("metric_octopus_gas", None, indirect=False)
            self.rate_gas = self.fetch_octopus_rates(entity_id)
            if not self.rate_gas:
                self.log("Warn: metric_octopus_gas is not set correctly in apps.yaml, or no energy rates can be read")
                self.record_status(message="Warn: rate_import_gas not set correctly in apps.yaml, or no energy rates can be read", had_errors=True)
            else:
                self.rate_gas, self.rate_gas_replicated = self.rate_replicate(self.rate_gas, is_import=False, is_gas=False)
                self.rate_scan_gas(self.rate_gas, print=True)
        elif "rates_gas" in self.args:
            self.rate_gas = self.basic_rates(self.get_arg("rates_gas", [], indirect=False), "rates_gas")
            self.rate_gas, self.rate_gas_replicated = self.rate_replicate(self.rate_gas, is_import=False, is_gas=False)
            self.rate_scan_gas(self.rate_gas, print=True)

        # Carbon intensity data
        if self.carbon_enable and ("carbon_intensity" in self.args):
            entity_id = self.get_arg("carbon_intensity", None, indirect=False)
            self.carbon_intensity, self.carbon_history = self.fetch_carbon_intensity(entity_id)

        # SoC history
        soc_kwh_data = self.get_history_wrapper(entity_id=self.prefix + ".soc_kw_h0", days=2, required=False)
        if soc_kwh_data:
            self.soc_kwh_history, _ = minute_data(
                soc_kwh_data[0],
                2,
                self.now_utc,
                "state",
                "last_updated",
                backwards=True,
                clean_increment=False,
                smoothing=False,
                divide_by=1.0,
                scale=1.0,
                required_unit="kWh",
                can_modify_history=True,  # soc_kwh_data is not accessed after this point, so minute_data can freely modify it
            )

        # Fetch sensor data for cars, e.g. car plan, car energy, car sessions etc.
        self.dispatch_timeline_pending = []
        self.fetch_sensor_data_cars(save=save)
        # Dynamic load: cancel the slots of a car that is in one but not charging - before the rates are
        # built, so a cancelled Intelligent dispatch never gets its cheap rate
        dynamic_load_car_changed = self.dynamic_load_car_check(save=save)

        if "rates_export_octopus_url" in self.args:
            # Fixed URL for rate export
            self.log("Downloading export rates directly from URL {}".format(self.get_arg("rates_export_octopus_url", indirect=False)))
            # Need to take a copy, as saving sessions will repeatedly increment cached export rates
            export_rates = copy.deepcopy(self.download_octopus_rates(self.get_arg("rates_export_octopus_url", indirect=False)))
        elif "metric_octopus_export" in self.args:
            # Octopus export rates
            entity_id = self.get_arg("metric_octopus_export", None, indirect=False)
            export_rates = self.fetch_octopus_rates(entity_id)
            if not export_rates:
                self.log("Warning: metric_octopus_export is not set correctly in apps.yaml, or no energy rates can be read")
                self.record_status(message="Error: metric_octopus_export not set correctly in apps.yaml, or no energy rates can be read", had_errors=True)
        elif "metric_energidataservice_export" in self.args:
            # Energi Data Service export rates
            entity_id = self.get_arg("metric_energidataservice_export", None, indirect=False)
            export_rates = self.fetch_energidataservice_rates(entity_id)
            if not export_rates:
                self.log("Warning: metric_energidataservice_export is not set correctly in apps.yaml, or no energy rates can be read")
                self.record_status(message="Error: metric_energidataservice_export not set correctly in apps.yaml, or no energy rates can be read", had_errors=True)
        elif "metric_stromligning_export_today" in self.args or "metric_stromligning_export_tomorrow" in self.args:
            # Strømligning export rates
            entity_id_today = self.get_arg("metric_stromligning_export_today", None, indirect=False)
            entity_id_tomorrow = self.get_arg("metric_stromligning_export_tomorrow", None, indirect=False)
            export_rates = self.fetch_stromligning_rates(entity_id_today, entity_id_tomorrow)
            if not export_rates:
                self.log("Warning: metric_stromligning_export sensors are not set correctly or no energy rates can be read")
                self.record_status(message="Error: metric_stromligning_export sensors not set correctly or no energy rates can be read", had_errors=True)

        # Fallback if no other rate types are set
        if not export_rates:
            # Basic rates defined by user over time
            rate_export_dict = self.get_arg("rates_export", [], indirect=False)
            # Allow all zero export rates, as some users have a feed-in tariff that is zero
            export_rates = self.basic_rates(rate_export_dict, "rates_export")

        # Fetch Axle sessions first so Octopus auto-join can skip saving sessions that overlap an Axle VPP session
        self.axle_sessions = fetch_axle_sessions(self)
        # Fetch octopus saving sessions and free sessions
        self.octopus_free_slots, self.octopus_saving_slots = self.fetch_octopus_sessions(self.axle_sessions)

        # Standing charge
        self.metric_standing_charge = self.get_arg("metric_standing_charge", 0.0) * 100.0
        self.log("Standing charge is set to {}{}".format(dp2(self.metric_standing_charge), curr))

        # futurerate data
        futurerate = FutureRate(self)
        self.future_energy_rates_import, self.future_energy_rates_export = futurerate.futurerate_analysis(import_rates, export_rates)

        # Replicate and scan import rates
        if import_rates:
            self.rate_scan(import_rates, print=False)
            self.rate_import_base, self.rate_min_base, self.rate_max_base = self.rate_base_min_max(import_rates)
            import_rates = self.dynamic_load_car_strip_feed_rates(import_rates)
            import_rates, self.rate_import_replicated = self.rate_replicate(import_rates, self.io_adjusted, is_import=True)
            self.rate_import_no_io = import_rates.copy()
            for car_n in range(self.num_cars):
                import_rates = self.rate_add_io_slots(car_n, import_rates, self.octopus_slots[car_n])
            self.load_saving_slot(self.octopus_saving_slots, import_rates, export=False, rate_replicate=self.rate_import_replicated)
            self.load_free_slot(self.octopus_free_slots, import_rates, export=False, rate_replicate=self.rate_import_replicated)
            load_axle_slot(self, self.axle_sessions, import_rates, export=False, rate_replicate=self.rate_import_replicated)
            import_rates = self.basic_rates(self.get_arg("rates_import_override", [], indirect=False), "rates_import_override", import_rates, self.rate_import_replicated)
            import_rates = self.apply_manual_rates(import_rates, self.manual_import_rates, is_import=True, rate_replicate=self.rate_import_replicated)
            self.rate_scan(import_rates, print=True)
        else:
            self.rate_import_no_io = {}
            self.log("Warning: No import rate data provided")
            self.record_status(message="Error: No import rate data provided", had_errors=True)
        # Atomic publish: readers (e.g. async components) never see a half-built or transiently-empty rate_import during rebuild.
        self.rate_import = import_rates

        # Replicate and scan export rates
        if export_rates:
            self.rate_scan_export(export_rates, print=False)
            export_rates, self.rate_export_replicated = self.rate_replicate(export_rates, is_import=False)
            self.rate_export_base = export_rates.copy()
            # Built here, from the base rates, so the saving session and overrides applied below stay
            # out of it - battery_value_rate needs the tariff's own export price, not an event price
            self.rate_export_max_forward = self.rate_export_max_forward_calc(self.rate_export_base)
            # For export tariff only load the saving session if enabled
            if self.rate_export_max > 0:
                self.load_saving_slot(self.octopus_saving_slots, export_rates, export=True, rate_replicate=self.rate_export_replicated)
            load_axle_slot(self, self.axle_sessions, export_rates, export=True, rate_replicate=self.rate_export_replicated)
            export_rates = self.basic_rates(self.get_arg("rates_export_override", [], indirect=False), "rates_export_override", export_rates, self.rate_export_replicated)
            export_rates = self.apply_manual_rates(export_rates, self.manual_export_rates, is_import=False, rate_replicate=self.rate_export_replicated)
            self.rate_scan_export(export_rates, print=True)
        else:
            self.log("Warning: No export rate data provided")
            self.record_status(message="Error: No export rate data provided", had_errors=True)
        self.rate_export = export_rates

        # Set rate thresholds
        if self.rate_import or self.rate_export:
            self.set_rate_thresholds()

        # Needed here, ahead of "Find charging windows" below, because calc_pv_light_dark() reads
        # self.pv_forecast_minute to locate dawn - it was previously fetched further down in this
        # function, after the dawn calculation had already run against the freshly-reset empty dict
        # from the top of fetch_sensor_data(), so the dawn split could never actually trigger (#4699).
        pv_light_dark = self.fetch_pv_forecast_and_dawn()

        # Find discharging windows
        if self.rate_export:
            self.high_export_rates, lowest, highest = self.rate_scan_window(self.rate_export, 5, self.rate_export_cost_threshold, True, alt_rates=self.rate_import)
            # Update threshold automatically
            self.log("High export rate found rates in range {}{} to {}{}".format(lowest, curr, highest, curr))
            if self.rate_high_threshold == 0 and lowest <= self.rate_export_max:
                self.rate_export_cost_threshold = lowest

        # Find charging windows
        if self.rate_import:
            # Find charging window
            self.low_rates, lowest, highest = self.rate_scan_window(self.rate_import, 5, self.rate_import_cost_threshold, False, alt_rates=self.rate_export, pv_light_dark=pv_light_dark)
            self.log("Low Import rate found rates in range {}{} to {}{}".format(lowest, curr, highest, curr))
            # Update threshold automatically
            if self.rate_low_threshold == 0 and highest >= self.rate_min:
                self.rate_import_cost_threshold = highest

        # #4516 Stage 1: render the dispatch timelines captured during the car fetch. This has to
        # come after the low-rate scan above, not merely after set_rate_thresholds(): in automatic
        # mode (rate_low_threshold 0) set_rate_thresholds() only sets a provisional
        # "everything but the most expensive" threshold, which the scan then replaces with the real
        # low-rate band. Rendering on the provisional value painted most of the day as cheap.
        self.log_dispatch_timelines()

        # Work out car plan?
        self.fetch_sensor_data_car_planning()
        # Dynamic load for the cars Predbat plans itself, whose slots only exist now
        dynamic_load_car_changed |= self.dynamic_load_car_check(save=save, late=True)
        # Publish the car plan
        self.publish_car_plan()

        # Work out iBoost plan
        if self.iboost_enable and (((not self.iboost_solar) and (not self.iboost_charging)) or self.iboost_smart):
            self.iboost_plan = self.plan_iboost_smart()
            self.log(
                "iBoost iboost_solar {}, rate threshold import {}{}, rate threshold export {}{}, iboost_gas {}{}, iboost_gas_export {}{}, iboost_smart {}, min_length {}, plan is: {}".format(
                    self.iboost_solar,
                    self.iboost_rate_threshold,
                    curr,
                    self.iboost_rate_threshold_export,
                    curr,
                    self.iboost_gas,
                    curr,
                    self.iboost_gas_export,
                    curr,
                    self.iboost_smart,
                    self.iboost_smart_min_length,
                    self.iboost_plan,
                )
            )

        # Work out cost today
        if self.import_today:
            self.cost_today_sofar, self.carbon_today_sofar = self.today_cost(self.import_today, self.export_today, self.car_charging_energy, self.load_minutes, save=save)

        if self.load_minutes and not self.load_forecast_only and not self.load_forecast_history:
            # Apply modal filter to historical data. Skipped for days_previous_auto: the weighted-bucket
            # forecast deliberately ignores missing-data buckets, so the gap-filling/padding the modal filter
            # applies would mask exactly the gaps it is designed to exclude.
            self.previous_days_modal_filter(self.load_minutes)
            self.log("Historical days now {} weight {}".format(self.days_previous, self.days_previous_weight))

        # Weighted-bucket historical load forecast (days_previous_auto). Built here as it needs load_minutes
        # after the power fill. load_ml takes precedence if it already set the forecast-only load source.
        if self.load_forecast_history and self.load_minutes and self.load_minutes_age >= 1 and not self.load_forecast_only:
            hist_forecast = self.compute_load_forecast_history(self.now_utc)
            if hist_forecast:
                self.load_forecast_only = True
                for minute, value in hist_forecast.items():
                    self.load_forecast[minute] = self.load_forecast.get(minute, 0) + value
                self.load_forecast_array.append(hist_forecast)
                self.log("Using weighted-bucket historical load forecast over {} days".format(min(self.load_minutes_age, self.max_days_previous - 1)))

        # Where Load ML genuinely supplied this cycle's forecast, backfill the elapsed part of today
        # with its own past predictions (the weighted-bucket forecast above is skipped in that case).
        # load_ml_forecast is this cycle's own fetch result from earlier in this function, so there is
        # no cross-cycle state that could leave a stale "ML was active" reading behind (#4762 review).
        self.apply_load_ml_forecast_history(self.now_utc, load_ml_forecast)

        # Load today vs actual
        if self.load_minutes:
            self.load_inday_adjustment = self.load_today_comparison(self.load_minutes, self.load_forecast, self.car_charging_energy, self.import_today, self.minutes_now, save=save)
        else:
            self.load_inday_adjustment = 1.0

        force_replan = dynamic_load_car_changed
        # Compare on the change-detection signature, not the raw slots, so the per-cycle re-clocking
        # of an in-progress dispatch (start advanced to now, energy scaled to remaining time) does not
        # force a replan every cycle while a slot is active - only genuine slot changes do
        if self.octopus_slots_signature(prev_octopus_slots) != self.octopus_slots_signature(self.octopus_slots):
            self.log("Octopus slots changed from {} to {}".format(prev_octopus_slots, self.octopus_slots))
            force_replan = True
        if str(prev_octopus_saving_slots) != str(self.octopus_saving_slots):
            self.log("Octopus saving slots changed from {} to {}".format(prev_octopus_saving_slots, self.octopus_saving_slots))
            force_replan = True
        if str(prev_octopus_free_slots) != str(self.octopus_free_slots):
            self.log("Octopus free slots changed from {} to {}".format(prev_octopus_free_slots, self.octopus_free_slots))
            force_replan = True
        if str(prev_axle_sessions) != str(self.axle_sessions):
            self.log("Axle sessions changed from {} to {}".format(prev_axle_sessions, self.axle_sessions))
            force_replan = True
        return force_replan

    def fetch_sensor_data_car_planning(self):
        # Work out car plan?
        # Determine which cars are configured with Octopus Intelligent to avoid calling plan_car_charging for them
        iog_entity_id_config = self.get_arg("octopus_intelligent_slot", indirect=False)
        if iog_entity_id_config and not isinstance(iog_entity_id_config, list):
            iog_entity_ids = [iog_entity_id_config]
        elif iog_entity_id_config:
            iog_entity_ids = iog_entity_id_config
        else:
            iog_entity_ids = []

        for car_n in range(self.num_cars):
            if self.octopus_intelligent_charging and car_n < len(iog_entity_ids) and iog_entity_ids[car_n]:
                if self.car_charging_planned[car_n]:
                    self.log("Car {} on Octopus Intelligent, active plan for charge".format(car_n))
                else:
                    self.log("Car {} on Octopus Intelligent, no active plan".format(car_n))
            elif self.car_charging_planned[car_n] or self.car_charging_now[car_n]:
                limit_percent = dp1(self.car_charging_limit[car_n] / self.car_charging_battery_size[car_n] * 100) if self.car_charging_battery_size[car_n] else 0
                self.log(
                    "Car {} plan charging from {}kWh to {}% ({}kWh), with slots {}, ready by {}".format(
                        car_n,
                        self.car_charging_soc[car_n],
                        limit_percent,
                        self.car_charging_limit[car_n],
                        self.low_rates,
                        self.car_charging_plan_time[car_n],
                    )
                )
                self.car_charging_slots[car_n] = self.plan_car_charging(car_n, self.low_rates)
            else:
                self.log("Car {} charging is not planned as it is not plugged in".format(car_n))

            # Log the charging plan
            if self.car_charging_slots[car_n]:
                self.log("Car {} charging plan is: {}".format(car_n, self.car_charging_slots[car_n]))

            # Only treat this car as "the" exclusive charging session if it actually produced a
            # real plan - in a shared-charger multi-car setup, car_charging_planned/now can read
            # true for every car profile sharing the same physical sensor (issue #4305), so a car
            # with an empty plan (its SoC already at/above its own limit) shouldn't block a later
            # car - the one actually plugged in and charging - from ever being considered.
            if (self.car_charging_planned[car_n] or self.car_charging_now[car_n]) and self.car_charging_exclusive[car_n] and self.car_charging_slots[car_n]:
                self.log("Car {} charging is exclusive, will not plan other cars".format(car_n))
                break

    def fetch_sensor_data_cars(self, save=True):
        """
        Fetch car specific data such as Octopus intelligent slots and vehicle data if we can get it, and calculate current SoC and limits based on that

        `save` is False when compare.py re-runs the fetch for a candidate tariff rather than the
        live plan; diagnostics that record state across cycles must be skipped in that case.
        """

        # Work out current car SoC and limit
        self.car_charging_loss = 1 - float(self.get_arg("car_charging_loss"))

        # Initialise car_charging_soc BEFORE the IOG processing loop so that load_octopus_slots can access it.
        # Values will be recalculated per-car after car_charging_battery_size is updated from Octopus sensor data.
        self.car_charging_soc = [0.0 for car_n in range(self.num_cars)]
        self.car_charging_soc_next = [None for car_n in range(self.num_cars)]
        for car_n in range(self.num_cars):
            if car_n < len(self.car_charging_manual_soc) and self.car_charging_manual_soc[car_n]:
                car_postfix = "" if car_n == 0 else "_" + str(car_n)
                self.car_charging_soc[car_n] = self.get_arg("car_charging_manual_soc_kwh" + car_postfix, 0.0)
            else:
                self.car_charging_soc[car_n] = (self.get_arg("car_charging_soc", 0.0, index=car_n) * self.car_charging_battery_size[car_n]) / 100.0

        # Get octopus intelligent slot configuration - could be single value or list for multiple cars
        entity_id_config = self.get_arg("octopus_intelligent_slot", indirect=False)

        # Normalize to list for multi-car support
        if entity_id_config and not isinstance(entity_id_config, list):
            entity_id_list = [entity_id_config]
        elif entity_id_config:
            entity_id_list = entity_id_config
        else:
            entity_id_list = []

        # Cars whose charging plan came from Octopus Intelligent dispatch slots this cycle - used
        # below to decide which cars get a model-facing charge limit override (#4967)
        iog_slot_cars = []

        if entity_id_list:
            # Process each car's intelligent slot configuration
            for car_n in range(min(len(entity_id_list), self.num_cars)):
                entity_id = entity_id_list[car_n]
                if not entity_id:
                    continue

                completed = []
                planned = []
                started = []

                if entity_id and "octopus_intelligent_slot_action_config" in self.args:
                    config_entry = self.get_arg("octopus_intelligent_slot_action_config", None, indirect=False)
                    service_name = entity_id.replace(".", "/")
                    result = self.call_service_wrapper(service_name, config_entry=config_entry, return_response=True)
                    if result and ("slots" in result):
                        planned = result["slots"]
                    else:
                        self.log("Warn: Unable to get data from {} for car {} - octopus_intelligent_slot using action config {}, result was {}".format(entity_id, car_n, config_entry, result))
                else:
                    try:
                        completed = self.get_state_wrapper(entity_id=entity_id, attribute="completed_dispatches") or self.get_state_wrapper(entity_id=entity_id, attribute="completedDispatches")
                        planned = self.get_state_wrapper(entity_id=entity_id, attribute="planned_dispatches") or self.get_state_wrapper(entity_id=entity_id, attribute="plannedDispatches")
                        # Not merged into octopus_slots or used for any rate/plan decision yet - read
                        # only for the #4516 Stage 1 diagnostic timeline log below, to observe whether
                        # this is a trustworthy earlier-than-completed confirmation signal before
                        # building any gating logic on it (Stage 2, deferred).
                        started = self.get_state_wrapper(entity_id=entity_id, attribute="started_dispatches") or self.get_state_wrapper(entity_id=entity_id, attribute="startedDispatches")
                    except (ValueError, TypeError):
                        self.log("Warn: Unable to get data from {} for car {} - octopus_intelligent_slot may not be set correctly in apps.yaml".format(entity_id, car_n))
                        self.record_status(message="Error: octopus_intelligent_slot not set correctly in apps.yaml for car {}".format(car_n), had_errors=True)

                # Completed and planned slots - merge from all cars
                if completed:
                    self.octopus_slots[car_n] += completed
                if planned and (not self.octopus_intelligent_ignore_unplugged or self.car_charging_planned[car_n]):
                    # We only count planned slots if the car is plugged in or we are ignoring unplugged cars
                    self.octopus_slots[car_n] += planned

                # Extract vehicle data if we can get it
                size = self.get_state_wrapper(entity_id=entity_id, attribute="vehicle_battery_size_in_kwh")
                rate = self.get_state_wrapper(entity_id=entity_id, attribute="charge_point_power_in_kw")
                try:
                    size = float(size)
                except (ValueError, TypeError):
                    size = None
                try:
                    rate = float(rate)
                except (ValueError, TypeError):
                    rate = None
                if size:
                    self.car_charging_battery_size[car_n] = size
                    # Recalculate car SoC now that battery size has been updated from Octopus data
                    if car_n < len(self.car_charging_manual_soc) and not self.car_charging_manual_soc[car_n]:
                        self.car_charging_soc[car_n] = (self.get_arg("car_charging_soc", 0.0, index=car_n) * self.car_charging_battery_size[car_n]) / 100.0
                if rate:
                    # Take the max as Octopus over reports
                    self.log("Car {} rate from Octopus is {}kW and configured rate {}".format(car_n, rate, self.car_charging_rate[car_n]))
                    self.car_charging_rate[car_n] = max(rate, self.car_charging_rate[car_n])

                # Get car charging limit again from car based on new battery size
                self.car_charging_limit[car_n] = dp3((float(self.get_arg("car_charging_limit", 100.0, index=car_n)) * self.car_charging_battery_size[car_n]) / 100.0)

                # #4516 Stage 1: diagnostic dispatch-timeline log. Purely observational - see
                # build_dispatch_timeline()'s and dispatch_timeline_should_log()'s docstrings.
                # Only the live plan's view is worth recording: compare.py re-runs this whole
                # fetch per candidate tariff (with save=False), which otherwise emits a burst of
                # near-identical timelines for plans that were never active - and, because each
                # one updates the "last logged" state, makes the real cycle's line look changed.
                #
                # Recorded here but rendered later: this runs before the rate fetch, so the cheap
                # ('-') background would read from the previous cycle's rates - and be empty
                # altogether on the first cycle after a restart. The dispatch lists and the
                # plugged-in reading are only correct here though: further down this function the
                # Octopus branch overwrites car_charging_planned with "has dispatch slots", so
                # capture the inputs now and render once the rates are known.
                if save:
                    self.dispatch_timeline_pending.append(
                        {
                            "car_n": car_n,
                            "completed": completed,
                            "started": started,
                            "planned": planned,
                            "plugged": self.car_charging_planned[car_n],
                            "charging_now": self.get_car_charging_now_tristate(car_n),
                        }
                    )

                # Extract vehicle preference if we can get it
                if self.octopus_intelligent_charging:
                    octopus_ready_time = self.get_arg("octopus_ready_time", None, index=car_n)
                    if isinstance(octopus_ready_time, str) and len(octopus_ready_time) == 5:
                        octopus_ready_time += ":00"
                    octopus_limit = self.get_arg("octopus_charge_limit", None, index=car_n)
                    if octopus_limit:
                        try:
                            octopus_limit = float(octopus_limit)
                        except (ValueError, TypeError):
                            self.log("Warn: octopus_charge_limit is set to a bad value {} for car {} in apps.yaml, must be a number".format(octopus_limit, car_n))
                            octopus_limit = None
                    if octopus_limit:
                        octopus_limit = dp3(float(octopus_limit) * self.car_charging_battery_size[car_n] / 100.0)
                        self.car_charging_limit[car_n] = min(self.car_charging_limit[car_n], octopus_limit)
                    if octopus_ready_time:
                        self.car_charging_plan_time[car_n] = octopus_ready_time

            # Get rate for import to compute charging costs
            if self.rate_import:
                self.rate_scan(self.rate_import, print=False)

            # Use octopus slots for charging - process for each car
            if self.octopus_intelligent_charging:
                for car_n in range(min(len(entity_id_list), self.num_cars)):
                    self.octopus_slots[car_n] = self.add_now_to_octopus_slot(car_n, self.octopus_slots[car_n], self.now_utc)
                    if not entity_id_list[car_n]:
                        continue
                    if not self.octopus_intelligent_ignore_unplugged or self.car_charging_planned[car_n] or self.car_charging_now[car_n]:
                        self.car_charging_slots[car_n] = self.load_octopus_slots(car_n, self.octopus_slots[car_n], self.octopus_intelligent_consider_full)
                        if self.car_charging_slots[car_n]:
                            iog_slot_cars.append(car_n)
                            self.log(
                                "Car {} using Octopus Intelligent, charging planned - charging limit {}, ready time {} - battery size {}".format(
                                    car_n, self.car_charging_limit[car_n], self.car_charging_plan_time[car_n], self.car_charging_battery_size[car_n]
                                )
                            )
                            self.car_charging_planned[car_n] = True
                        else:
                            self.log("Car {} using Octopus Intelligent, no charging is planned".format(car_n))
                            self.car_charging_planned[car_n] = False
                    else:
                        self.log("Car {} using Octopus Intelligent is unplugged".format(car_n))
                        self.car_charging_planned[car_n] = False
        else:
            # Disable octopus charging if we don't have the slot sensor
            self.octopus_intelligent_charging = False

        # Model-facing car charge limit (#4967). With octopus_intelligent_consider_full off (the
        # default) the prediction must trust the Octopus dispatch plan rather than modelling the car
        # filling up, so cars carrying IOG slots get an uncapped limit for predict()'s fill clamp
        # (which also releases the battery discharge hold once the modelled car "fills"). The real
        # car_charging_limit is left untouched - execute.py's "car is full" decision, the
        # plan_car_charging path and load_octopus_slots all still need it. None means no override.
        if iog_slot_cars and not self.octopus_intelligent_consider_full:
            self.car_charging_limit_model = self.car_charging_limit[:]
            for car_n in iog_slot_cars:
                self.car_charging_limit_model[car_n] = CAR_CHARGING_LIMIT_UNCAPPED
        else:
            self.car_charging_limit_model = None

        # Log final car SoC (initialised before the IOG loop, updated per-car after Octopus battery_size is read)
        if self.num_cars:
            car_charging_limit_percent = [dp1(limit / size * 100) if size else 0 for limit, size in zip(self.car_charging_limit, self.car_charging_battery_size)]
            self.log("Cars: SoC: {}kWh, Charge limit {}% ({}kWh), plan time {}, battery size {}kWh".format(self.car_charging_soc, car_charging_limit_percent, self.car_charging_limit, self.car_charging_plan_time, self.car_charging_battery_size))

    def fetch_pv_forecast(self):
        """
        Fetch PV forecast data from one or more sensors

        Returns a tuple of (pv_forecast_minute, pv_forecast_minute10, pv_forecast_minute90) - the
        p50, p10 and p90 minute-indexed forecasts respectively. pv_forecast_minute90 is the real
        p90 series when the source published one (currently only Solcast), otherwise it is a plain
        copy of the p50 series - see the design spec for why the p90 upside is never synthesised
        from the p10 spread.
        """
        pv_forecast_minute = {}
        pv_forecast_minute10 = {}
        pv_forecast_minute90 = {}

        # Get data from forecast sensor
        entity_id = "sensor." + self.prefix + "_pv_forecast_raw"
        pv_forecast_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecast")
        pv_forecast10_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecast10")
        pv_forecast90_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecast90")
        relative_time = self.get_state_wrapper(entity_id=entity_id, attribute="relative_time")
        try:
            relative_time = datetime.strptime(relative_time, TIME_FORMAT)
        except (ValueError, TypeError):
            self.log(f"Warn: Unable to parse relative_time from PV forecast sensor: {relative_time}, using midnight UTC as fallback")
            relative_time = self.midnight_utc

        # Convert keys to integers and values to floats
        pv_forecast_packed = {}
        pv_forecast10_packed = {}
        pv_forecast90_packed = {}

        if pv_forecast_packed_ld:
            for key, value in pv_forecast_packed_ld.items():
                try:
                    minute = int(key)
                    pv_forecast_packed[minute] = float(value)
                except (ValueError, TypeError):
                    pass

        if pv_forecast10_packed_ld:
            for key, value in pv_forecast10_packed_ld.items():
                try:
                    minute = int(key)
                    pv_forecast10_packed[minute] = float(value)
                except (ValueError, TypeError):
                    pass

        if pv_forecast90_packed_ld:
            for key, value in pv_forecast90_packed_ld.items():
                try:
                    minute = int(key)
                    pv_forecast90_packed[minute] = float(value)
                except (ValueError, TypeError):
                    pass

        # Unpack the forecast data
        max_minute = max(pv_forecast_packed.keys()) if pv_forecast_packed else 0
        last_value = 0
        last_value10 = 0
        last_value90 = 0
        # The forecast could be for a different time to our relative time, so we need to offset the minutes to align with our midnight_utc.
        # relative_time is the midnight at which the forecast was saved, so stored minute keys are relative to that midnight.
        # We subtract the offset so that stored minute X (= relative_time + X) maps to (relative_time + X - midnight_utc) minutes from today's midnight.
        minute_offset = int((self.midnight_utc - relative_time).total_seconds() / 60)
        for minute in range(0, max_minute + 1):
            target_minute = minute - minute_offset
            last_value = pv_forecast_packed.get(minute, last_value)
            last_value10 = pv_forecast10_packed.get(minute, last_value10)
            last_value90 = pv_forecast90_packed.get(minute, last_value90)
            pv_forecast_minute[target_minute] = last_value
            pv_forecast_minute10[target_minute] = last_value10
            pv_forecast_minute90[target_minute] = last_value90

        # No p90 published (older sensor data, or a source that does not produce one) - fall back to
        # the central forecast. No upside is synthesised; see the design spec for why mirroring the
        # p10 spread was rejected.
        if not pv_forecast90_packed:
            pv_forecast_minute90 = dict(pv_forecast_minute)

        return pv_forecast_minute, pv_forecast_minute10, pv_forecast_minute90

    def predict_battery_temperature(self, battery_temperature_history, step):
        """
        Given historical battery temperature data, predict the future temperature

        For now a fairly simple look back over 24 hours is used, can be improved with outdoor temperature later
        """

        predicted_temp = {}
        current_temp = battery_temperature_history.get(0, 20)
        predict_timestamps = {}

        for minute in range(0, self.forecast_minutes, step):
            timestamp = self.now_utc + timedelta(minutes=minute)
            timestamp_str = timestamp.strftime(TIME_FORMAT)
            predicted_temp[minute] = dp2(current_temp)
            predict_timestamps[timestamp_str] = dp2(current_temp)

            # Look at 30 minute change 24 hours ago to predict the up/down trend
            minute_previous = (24 * 60 - minute) % (24 * 60)
            change = battery_temperature_history.get(minute_previous, 20) - battery_temperature_history.get(minute_previous + step, 20)
            current_temp += change
            current_temp = max(min(current_temp, 30), -20)

        self.dashboard_item(
            self.prefix + ".battery_temperature",
            state=dp2(battery_temperature_history.get(0, 20)),
            attributes={
                "results": self.filtered_times(predict_timestamps),
                "temperature_h1": battery_temperature_history.get(60, 20),
                "temperature_h2": battery_temperature_history.get(60 * 2, 20),
                "temperature_h8": battery_temperature_history.get(60 * 8, 20),
                "friendly_name": "Battery temperature",
                "state_class": "measurement",
                "unit_of_measurement": "°C",
                "device_class": "temperature",
                "icon": "mdi:temperature-celsius",
            },
        )
        return predicted_temp

    def download_ge_data(self, now_utc):
        """
        Gets the GE Cloud data into the required format
        """
        mdata = None
        if self.components:
            ge_cloud_data = self.components.get_component("gecloud_data")
            if ge_cloud_data:
                mdata, oldest_data_time = ge_cloud_data.get_data()

        if not mdata:
            self.log("Warn: No GE Cloud data returned from GECloudData component, check if it is configured correctly")
            return

        age = now_utc - oldest_data_time
        self.load_minutes_age = age.days
        self.load_minutes, _ = minute_data(ge_cloud_data.filter_data(mdata, "consumption"), self.max_days_previous, now_utc, "consumption", "last_updated", backwards=True, smoothing=True, scale=1.0, clean_increment=True, interpolate=True)
        self.import_today, _ = minute_data(ge_cloud_data.filter_data(mdata, "import"), self.max_days_previous, now_utc, "import", "last_updated", backwards=True, smoothing=True, scale=self.import_export_scaling, clean_increment=True)
        self.export_today, _ = minute_data(ge_cloud_data.filter_data(mdata, "export"), self.max_days_previous, now_utc, "export", "last_updated", backwards=True, smoothing=True, scale=self.import_export_scaling, clean_increment=True)
        self.pv_today, _ = minute_data(ge_cloud_data.filter_data(mdata, "pv"), self.max_days_previous, now_utc, "pv", "last_updated", backwards=True, smoothing=True, scale=self.import_export_scaling, clean_increment=True)

        self.load_minutes_now = get_now_from_cumulative(self.load_minutes, self.minutes_now, backwards=True)
        self.load_last_period = (self.load_minutes.get(0, 0) - self.load_minutes.get(PREDICT_STEP, 0)) * 60 / PREDICT_STEP
        self.import_today_now = get_now_from_cumulative(self.import_today, self.minutes_now, backwards=True)
        self.export_today_now = get_now_from_cumulative(self.export_today, self.minutes_now, backwards=True)
        self.pv_today_now = get_now_from_cumulative(self.pv_today, self.minutes_now, backwards=True)

        # More up to date sensors for current values if set
        if "load_today" in self.args:
            load_minutes, load_minutes_age = self.minute_data_load(self.now_utc, "load_today", self.max_days_previous, required_unit="kWh", load_scaling=1.0, interpolate=True)
            self.load_minutes_now = get_now_from_cumulative(load_minutes, self.minutes_now, backwards=True)
            self.load_last_period = (load_minutes.get(0, 0) - load_minutes.get(PREDICT_STEP, 0)) * 60 / PREDICT_STEP
            self.log("GECloudData load_last_period from immediate sensor is {} kW".format(dp2(self.load_last_period)))

        if "import_today" in self.args:
            import_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "import_today", scale=self.import_export_scaling, required_unit="kWh")
            self.import_today_now = get_now_from_cumulative(import_today, self.minutes_now, backwards=True)

        # Load export today data
        if "export_today" in self.args:
            export_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "export_today", scale=self.import_export_scaling, required_unit="kWh")
            self.export_today_now = get_now_from_cumulative(export_today, self.minutes_now, backwards=True)

        # PV today data
        if "pv_today" in self.args:
            pv_today = self.minute_data_import_export(self.max_days_previous, self.now_utc, "pv_today", required_unit="kWh")
            self.pv_today_now = get_now_from_cumulative(pv_today, self.minutes_now, backwards=True)

        self.log("Downloaded {} datapoints from GECloudData going back {} days".format(len(self.load_minutes), self.load_minutes_age))
        return True

    def rate_replicate(self, rates, rate_io=None, is_import=True, is_gas=False):
        """
        We don't get enough hours of data for Octopus, so lets assume it repeats until told others
        """
        if rate_io is None:
            rate_io = {}
        minute = -24 * 60
        rate_last = 0
        rate_last_valid = False  # Track if we've seen any real rates yet
        adjusted_rates = {}
        replicated_rates = {}

        # Add 48 extra hours to make sure the whole cycle repeats another day
        while minute < (self.forecast_minutes + 48 * 60):
            if minute not in rates:
                adjust_type = "copy"
                # Take 24-hours previous if missing rate
                if (minute - 24 * 60) in rates:
                    minute_mod = minute - 24 * 60
                elif (minute >= 24 * 60) and ((minute - 24 * 60) in rates):
                    minute_mod = minute - 24 * 60
                else:
                    minute_mod = minute % (24 * 60)

                if rate_io and (minute_mod in rate_io) and rate_io[minute_mod]:
                    # Dont replicate Intelligent rates into the next day as it will be different
                    rate_offset = self.rate_max
                    using_last = False
                elif minute_mod in rates:
                    rate_offset = rates[minute_mod]
                    using_last = False
                else:
                    # Missing rate within 24 hours - fill with dummy last rate
                    rate_offset = rate_last
                    using_last = True

                # Only offset once not every day
                futurerate_adjust_import = self.get_arg("futurerate_adjust_import", False)
                futurerate_adjust_export = self.get_arg("futurerate_adjust_export", False)
                if minute_mod not in adjusted_rates:
                    if is_import and futurerate_adjust_import and (minute in self.future_energy_rates_import) and (minute_mod in self.future_energy_rates_import):
                        rate_offset = self.future_energy_rates_import[minute]
                        adjust_type = "future"
                        using_last = False
                    elif (not is_import) and (not is_gas) and futurerate_adjust_export and (minute in self.future_energy_rates_export) and (minute_mod in self.future_energy_rates_export):
                        rate_offset = max(self.future_energy_rates_export[minute], 0)
                        adjust_type = "future"
                        using_last = False
                    elif is_import:
                        rate_offset = rate_offset + self.metric_future_rate_offset_import
                        if self.metric_future_rate_offset_import:
                            adjust_type = "offset"
                    elif (not is_import) and (not is_gas):
                        rate_offset = max(rate_offset + self.metric_future_rate_offset_export, 0)
                        if self.metric_future_rate_offset_export:
                            adjust_type = "offset"

                    adjusted_rates[minute] = True

                # Don't add rates if we haven't seen any real rates yet
                # This prevents filling negative minutes with invalid fallback values
                if rate_last_valid or not using_last:
                    rates[minute] = rate_offset
                    replicated_rates[minute] = adjust_type
            else:
                rate_last = rates[minute]
                rate_last_valid = True
            minute += 1

        return rates, replicated_rates

    def carbon_replicate(self, carbon_data):
        """
        Extend the carbon intensity forecast forward to cover the whole plan horizon.

        The Carbon Intensity API only publishes about 48 hours ahead, and publishes a good deal
        less than that whenever the upstream forecast is late, so the tail of the plan can easily
        have no data at all. A missing minute is scored as zero gCO2/kWh by prediction.py, which
        makes the uncovered part of the plan look carbon free and biases the optimiser towards it,
        so fill the gaps the same way rate_replicate does for missing rates - repeat the same time
        of day 24 hours earlier, falling back to the last known value where there is no such point
        yet. Carbon intensity has a strong daily cycle (the solar dip and the evening peak), so the
        previous day is a far better estimate than either zero or a flat average.

        Nothing is invented before the first real value, so an empty forecast stays empty rather
        than becoming a plan full of fabricated zeroes.

        :param carbon_data: carbon intensity in gCO2/kWh by minute relative to now
        :return: the extended data and a dict of which minutes were replicated
        """
        replicated_carbon = {}
        carbon_last = None
        minute = 0

        while minute < self.forecast_minutes:
            if minute in carbon_data:
                carbon_last = carbon_data[minute]
            elif carbon_last is not None:
                # Take the same time of day yesterday, which may itself have been replicated
                # so that the daily cycle keeps repeating across the whole horizon
                previous_day = minute - 24 * 60
                carbon_data[minute] = carbon_data[previous_day] if previous_day in carbon_data else carbon_last
                replicated_carbon[minute] = True
            minute += 1

        return carbon_data, replicated_carbon

    def calc_pv_light_dark(self):
        """
        Compute the dawn light/dark split via calc_dawn().

        Always computed, even when combine_charge_slots is off: with it off, find_charge_window
        already forces a window break every charge_slot_split minutes (which equals
        plan_interval_minutes, the same granularity calc_dawn buckets at), so the dawn boundary can
        never actually be reached there and calc_dawn's result is a no-op for window splitting in
        that case - but binary_sensor.predbat_dawn (published from this same result, see
        fetch_pv_forecast_and_dawn) is a useful standalone signal regardless of combine_charge_slots,
        and used to read permanently "off" for anyone with it disabled (the default) even in broad
        daylight, which is what it is not meant to mean.
        """
        return self.calc_dawn()

    def calc_dawn(self):
        """
        Find dawn in self.pv_forecast_minute and return a pv_light_dark dict classifying each minute
        as light (1, at/after dawn) or dark (0, before it). Used by find_charge_window (via
        calc_pv_light_dark) to split a charge window at the light/dark boundary - originally so it
        wouldn't abandon low power charging for a whole window just because its tail overlaps the sun
        (#4557), and now also so the plan optimizer can choose the dark portion of a combined window
        independently of the light portion.

        Classified per plan_interval_minutes bucket (averaged), not per raw minute - a threshold
        compared minute to minute would let ordinary forecast noise near the cutoff (e.g. a patchy dawn
        hovering around the threshold) retrigger the split repeatedly and chop the window into several
        small pieces. Bucketing makes the classification a step function that can change at most once
        per bucket boundary, at the same granularity the plan already displays.

        Within each calendar day, light is a one-way latch: the first bucket that crosses the threshold
        confirms dawn, and every later bucket that day stays light too, even if PV genuinely dips back
        under the threshold later (a cloud passing). This is a dawn detector, not a tracker of every
        rise and fall - it only cares about finding the first dawn each day, so an intermittently
        cloudy morning can't chop the window into several flip-flopping pieces. The latch resets at
        each day boundary so the next day's dawn is found independently rather than one early crossing
        holding "light" for the rest of the multi-day forecast horizon - this also makes a polar-night
        day correctly stay all-dark (dawn never crosses) and a polar-day day correctly stay all-light
        (crossed from the very first bucket).

        The threshold is the user-configurable low_power_pv_threshold_w, an absolute Watts figure,
        rather than a fraction of this forecast's own peak. A fraction-of-peak threshold sounds like it
        "scales with the site", but it actually scales with today's weather: a heavily overcast day has
        a low peak of its own, so a fixed fraction of it can call a trickle of PV "light" at a much
        lower absolute wattage than the same fraction would on a clear day - and there is nothing in
        Predbat today recording the site's real panel capacity to normalise against instead (only
        inverter_limit, which caps the inverter's own output, not the array's rating). An absolute
        figure the user sets once for their own system does not have that problem (#4699 follow-up).
        The same value and the same comparison is reused by find_charge_rate's low-power abandon check
        (utils.py) - one threshold answering "is this bright enough to matter" everywhere it is asked,
        rather than two independently-tuned ones that could disagree on genuine twilight PV.

        Built from self.pv_forecast_minute as populated earlier this same fetch_sensor_data() call by
        fetch_pv_forecast() - see #4699, where calc_pv_light_dark() ran before that populating call and
        so always saw the freshly-reset empty dict, silently disabling the dawn split for everyone.

        Logs the calculated dawn time (today's, or the earliest day the forecast reaches if today's PV
        data isn't there) each time it runs, so it's visible whether the detected dawn looks sane.
        """
        pv_light_dark = {}
        if not self.pv_forecast_minute:
            return pv_light_dark

        light_threshold = self.low_power_pv_threshold_w / MINUTE_WATT
        interval = self.plan_interval_minutes
        bucket_sums = {}
        bucket_counts = {}
        for pv_minute, pv in self.pv_forecast_minute.items():
            bucket = pv_minute // interval
            bucket_sums[bucket] = bucket_sums.get(bucket, 0.0) + pv
            bucket_counts[bucket] = bucket_counts.get(bucket, 0) + 1
        bucket_crossed = {bucket: (1 if (bucket_sums[bucket] / bucket_counts[bucket]) >= light_threshold else 0) for bucket in bucket_sums}

        bucket_light = {}
        dawn_minute_by_day = {}
        after_dawn = False
        current_day = None
        for bucket in sorted(bucket_crossed):
            bucket_day = (bucket * interval) // (24 * 60)
            if bucket_day != current_day:
                after_dawn = False
                current_day = bucket_day
            if bucket_crossed[bucket]:
                if not after_dawn:
                    dawn_minute_by_day[bucket_day] = bucket * interval
                after_dawn = True
            bucket_light[bucket] = 1 if after_dawn else 0

        # Day 0 is today (self.minutes_now is itself minutes since midnight_utc), so report today's
        # dawn when the forecast reaches it; otherwise fall back to the earliest day that does (e.g. a
        # forecast that only starts covering PV from tomorrow) so the log still says something useful.
        report_day = 0 if 0 in dawn_minute_by_day else (min(dawn_minute_by_day) if dawn_minute_by_day else None)
        if report_day is not None:
            dawn_timestamp = self.midnight_utc + timedelta(minutes=dawn_minute_by_day[report_day])
            self.log("Calculated dawn (start of daylight, used to split charge windows at) at {}".format(dawn_timestamp.strftime(TIME_FORMAT)))
        else:
            self.log("Calculated dawn (start of daylight, used to split charge windows at) - no dawn found in the PV forecast")

        pv_light_dark = {pv_minute: bucket_light[pv_minute // interval] for pv_minute in self.pv_forecast_minute}
        return pv_light_dark

    def find_charge_window(self, rates, minute, threshold_rate, find_high, alt_rates=None, pv_light_dark=None):
        """
        Find the charging windows based on the low rate threshold (percent below average)

        pv_light_dark, when scanning for charge (not find_high) windows, is a minute-indexed dict of 0/1
        marking whether PV forecast is at/after dawn ("light") or not ("dark") - see calc_dawn and
        calc_pv_light_dark. A transition between the two forces a window split, so a charge window
        that would otherwise span sunrise (e.g. a single long cheap-rate period) is instead built as
        separate dark and light windows. Originally added (#4557) so low power charging wasn't
        defeated for the whole window, including the still-dark hours, just because the window's tail
        overlapped PV later on - it also lets the plan optimizer pick the dark portion of a combined
        window without the light portion, regardless of low power charging.
        """
        alt_rates = alt_rates or {}
        pv_light_dark = pv_light_dark or {}
        rate_low_start = -1
        rate_low_end = -1
        rate_low_average = 0
        rate_low_rate = 0
        rate_low_count = 0
        alternate_rate_boundary = False
        alt_rate_last = None
        pv_boundary = False
        pv_light_dark_last = None

        # Work out alternate rate threshold
        alt_rate_max = max(alt_rates.values()) if alt_rates else 0
        alt_rate_min = min(alt_rates.values()) if alt_rates else 0
        alt_rate_threshold = abs(alt_rate_max - alt_rate_min) * 0.1

        stop_at = self.forecast_minutes + self.minutes_now + 12 * 60
        # Scan for lower rate start and end
        while minute < stop_at and not (minute >= (self.forecast_minutes + self.minutes_now) and (rate_low_start < 0)):
            alt_rate = alt_rates.get(minute, None)
            if (alt_rate is not None) and (alt_rate_last is not None) and (abs(alt_rate - alt_rate_last) >= alt_rate_threshold):
                # Create alternate rate boundary if the rate is different
                alternate_rate_boundary = True
            pv_state = pv_light_dark.get(minute, None)
            if (rate_low_start >= 0) and (pv_state is not None) and (pv_light_dark_last is not None) and (pv_state != pv_light_dark_last):
                # Create a PV boundary when the light/dark classification changes since the window
                # started. Gated on rate_low_start so a transition crossed only during the pre-window
                # search phase (e.g. scanning past last night's sunset before this window is even
                # found) doesn't leave a stale boundary that breaks the window the moment it starts.
                pv_boundary = True
            if minute in rates:
                rate = rates[minute]
                if ((not find_high) and (rate <= threshold_rate)) or (find_high and (rate >= threshold_rate) and (rate > 0)) or (minute in self.manual_all_times) or (rate_low_start in self.manual_all_times):
                    rate_diff = abs(rate - rate_low_rate)
                    if (rate_low_start >= 0) and rate_diff > self.combine_rate_threshold:
                        # Refuse mixed rates that are different by more than threshold
                        rate_low_end = minute
                        break
                    if find_high and (not self.combine_export_slots) and (rate_low_start >= 0) and ((minute - rate_low_start) >= self.export_slot_split):
                        # If combine is disabled, for export slots make them all N minutes so we can select some not all
                        rate_low_end = minute
                        break
                    if (not find_high) and (not self.combine_charge_slots) and (rate_low_start >= 0) and ((minute - rate_low_start) >= self.charge_slot_split):
                        # If combine is disabled, for import slots make them all N minutes so we can select some not all
                        rate_low_end = minute
                        break
                    if (rate_low_start in self.manual_all_times or minute in self.manual_all_times) and (rate_low_start >= 0) and ((minute - rate_low_start) >= self.plan_interval_minutes):
                        # Manual slot
                        rate_low_end = minute
                        break
                    if find_high and (rate_low_start >= 0) and (((minute - rate_low_start) >= 60 * 24) or (((minute - rate_low_start) >= self.plan_interval_minutes) and alternate_rate_boundary)):
                        # Export slot can never be bigger than 4 hours
                        rate_low_end = minute
                        break
                    if (not find_high) and (rate_low_start >= 0) and ((minute - rate_low_start) >= self.plan_interval_minutes) and pv_boundary:
                        # Split a charge window where PV forecast crosses the light/dark threshold
                        rate_low_end = minute
                        break
                    if rate_low_start < 0:
                        rate_low_start = minute
                        rate_low_end = stop_at
                        rate_low_count = 1
                        rate_low_average = rate
                        rate_low_rate = rate
                    elif rate_low_end > minute:
                        rate_low_average += rate
                        rate_low_count += 1
                else:
                    if rate_low_start >= 0:
                        rate_low_end = minute
                        break
            else:
                if rate_low_start >= 0 and rate_low_end >= minute:
                    rate_low_end = minute
                    break
            minute += 5
            alt_rate_last = alt_rate
            pv_light_dark_last = pv_state

        if rate_low_count:
            rate_low_average = dp2(rate_low_average / rate_low_count)
        return rate_low_start, rate_low_end, rate_low_average

    def apply_manual_rates(self, rates, manual_items, is_import=True, rate_replicate=None):
        """
        Apply manual rates to the rates dictionary
        """
        if rate_replicate is None:
            rate_replicate = {}
        if not manual_items:
            return rates

        for minute in manual_items:
            rate = manual_items[minute]
            try:
                rate = float(rate)
            except (ValueError, TypeError):
                self.log("Warn: Bad rate {} provided in manual rates".format(rate))
                self.record_status("Warn: Bad rate {} provided in manual rates".format(rate), had_errors=True)
                continue
            rates[minute] = rate
            rate_replicate[minute] = "manual"

        return rates

    def basic_rates(self, info, rtype, prev=None, rate_replicate=None, include_manual_api=True):
        """
        Work out the energy rates based on user supplied time periods. Weekly rules on the base
        tariff are stamped across the whole horizon, each day against its own day_of_week; any
        minute no rule claims is left for rate_replicate() to fill later

        include_manual_api should be False for callers (e.g. tariff comparison, annual replay)
        that simulate a tariff other than the live one - the live system's manual API overrides
        apply to the actual running plan and must not leak into those simulations.
        """
        rates = {}
        if rate_replicate is None:
            rate_replicate = {}
        curr = self.currency_symbols[1]

        if prev:
            rates = prev.copy()
            max_minute = max(rates) + 1
        else:
            # Set to zero
            for minute in range(24 * 60):
                rates[minute] = 0
            max_minute = 48 * 60

        # get_manual_api() returns each override wrapped as {"index": ..., "value": {...}}, not the
        # flat {"start": ..., "end": ..., "rate": ...} shape this loop expects below - unwrap it here.
        # Before this fix, an unwrapped entry's this_rate.get("start")/get("rate") always missed,
        # falling through to the "00:00:00" start/end default (which, since end<=start, wraps to a
        # full 24-hour range) and the rate=0/rate_increment=True default (a genuine no-op on the
        # rate value) - silently marking every minute of the day as overridden in rate_replicate
        # (and hence in the plan display) even for the narrowest override window (batpred#2578).
        # For most callers (anything sourced via get_arg(..., default=[]), which already merges
        # manual API overrides into its own return value) this would duplicate an override already
        # present in `info`. Avoid double-applying, as incremental overrides would otherwise stack.
        manual_items = []
        if include_manual_api:
            for item in self.get_manual_api(rtype):
                value = item.get("value")
                if not isinstance(value, dict):
                    self.log("Warn: Manual API override for {} must use the '?start=...&end=...&rate=...' form, got {}".format(rtype, value))
                    self.record_status("Warn: Manual API override for {} must use the '?start=...&end=...&rate=...' form".format(rtype), had_errors=True)
                    continue
                if value not in info:
                    manual_items.append(value)
        if manual_items:
            self.log("Basic rate API override items for {} are {}".format(rtype, manual_items))

        midnight = time_string_to_stamp("00:00:00")
        for this_rate in info + manual_items:
            if this_rate and isinstance(this_rate, dict):
                start_str = this_rate.get("start") or "00:00:00"
                start_str = self.resolve_arg("start", start_str, "00:00:00") or "00:00:00"
                end_str = this_rate.get("end") or "00:00:00"
                end_str = self.resolve_arg("end", end_str, "00:00:00") or "00:00:00"
                load_scaling = this_rate.get("load_scaling", None)

                if start_str.count(":") < 2:
                    start_str += ":00"
                if end_str.count(":") < 2:
                    end_str += ":00"

                start = time_string_to_stamp(start_str)
                if start is None:
                    self.log("Warn: Bad start time {} provided in energy rates".format(start_str))
                    self.record_status("Warn: Bad start time {} provided in energy rates".format(start_str), had_errors=True)
                    continue

                end = time_string_to_stamp(end_str)
                if end is None:
                    self.log("Warn: Bad end time {} provided in energy rates".format(end_str))
                    self.record_status("Warn: Bad end time {} provided in energy rates".format(end_str), had_errors=True)
                    continue

                date = None
                if "date" in this_rate:
                    date_str = self.resolve_arg("date", this_rate["date"])
                    try:
                        date = datetime.strptime(date_str, "%Y-%m-%d")
                    except (ValueError, TypeError):
                        self.log("Warn: Bad date {} provided in energy rates".format(this_rate["date"]))
                        self.record_status("Warn: Bad date {} provided in energy rates".format(this_rate["date"]), had_errors=True)
                        continue
                day_of_week = []
                if "day_of_week" in this_rate:
                    day = str(this_rate["day_of_week"])
                    days = day.split(",")
                    for day in days:
                        try:
                            day = int(day)
                        except (ValueError, TypeError):
                            self.log("Warn: Bad day_of_week {} provided in energy rates, should be 1-7".format(day_of_week))
                            self.record_status("Warn: Bad day_of_week {} provided in energy rates, should be 1-7".format(day_of_week), had_errors=True)
                            continue
                        if day < 1 or day > 7:
                            self.log("Warn: Bad day_of_week {} provided in energy rates, should be 1-7".format(day))
                            self.record_status("Warn: Bad day_of_week {} provided in energy rates, should be 1-7".format(day), had_errors=True)
                            continue
                        # Store as Python day of week
                        day_of_week.append(day - 1)

                # Support increment to existing rates (for override)
                if "rate" in this_rate:
                    rate = this_rate.get("rate", 0.0)
                    rate_increment = False
                elif "rate_increment" in this_rate:
                    rate = this_rate.get("rate_increment", 0.0)
                    rate_increment = True
                else:
                    rate = 0
                    rate_increment = True

                # Resolve any sensor links
                if isinstance(rate, str) and rate and rate[0].isalpha():
                    rate = self.resolve_arg("rate", rate, 0.0)
                if isinstance(load_scaling, str) and load_scaling and load_scaling[0].isalpha():
                    load_scaling = self.resolve_arg("load_scaling", load_scaling, 1.0)

                # Ensure the end result is a float
                try:
                    rate = float(rate)
                except (ValueError, TypeError):
                    self.log("Warn: Bad rate {} provided in energy rates".format(rate))
                    self.record_status("Warn:Bad rate {} provided in energy rates".format(rate), had_errors=True)
                    continue

                if load_scaling is not None:
                    try:
                        load_scaling = float(load_scaling)
                    except (ValueError, TypeError):
                        self.log("Warn: Bad load_scaling {} provided in energy rates".format(load_scaling))
                        self.record_status("Warn: Bad load_scaling {} provided in energy rates".format(load_scaling), had_errors=True)
                        continue

                # Time in minutes
                start_minutes = max(minutes_to_time(start, midnight), 0)
                end_minutes = min(minutes_to_time(end, midnight), 24 * 60 - 1)

                # Make end > start
                if end_minutes <= start_minutes:
                    end_minutes += 24 * 60

                # A window flagged utc is given in UTC rather than local wall-clock time, so shift it
                # by the local offset. That keeps a UTC-fixed schedule - an Economy 7 smart meter, for
                # instance - aligned with the meter through British Summer Time instead of running an
                # hour early. The offset is taken at local midnight, which is the reference the
                # returned minute keys are relative to.
                if this_rate.get("utc", False):
                    utc_offset = self.midnight_utc.utcoffset()
                    offset_minutes = int(utc_offset.total_seconds() // 60) if utc_offset else 0
                    start_minutes += offset_minutes
                    end_minutes += offset_minutes

                # Adjust for date if specified
                if date:
                    # Carry midnight_utc's offset so this stays a plain wall-clock difference in
                    # whole days, whichever side of a DST change the date falls on.
                    delta_minutes = minutes_to_time(date.replace(tzinfo=self.midnight_utc.tzinfo), self.midnight_utc)
                    start_minutes += delta_minutes
                    end_minutes += delta_minutes

                self.log(
                    "Adding rate {}: {}{} => {} to {} @ {}{}, date {}, day_of_week {}, increment {}{}".format(rtype, this_rate, curr, self.time_abs_str(start_minutes), self.time_abs_str(end_minutes), rate, curr, date, day_of_week, rate_increment, curr)
                )

                day_of_week_midnight = self.midnight_utc.weekday()

                # A weekly rule on the base tariff is stamped over the whole horizon rate_replicate()
                # goes on to cover, not just the days modelled above, so every day is filtered against
                # its own weekday. rate_replicate() fills a missing minute from the same time of day 24
                # hours earlier and knows nothing about day_of_week, so a day left to it takes the
                # previous day's pattern - a weekend rule flattening the weekday peaks (batpred#5168).
                # Overrides (prev) already span the replicated horizon via max_minute.
                if not date and not prev:
                    stamp_end = max(max_minute + 24 * 60, self.forecast_minutes + 48 * 60)
                else:
                    stamp_end = max_minute

                # Store rates against range
                if end_minutes >= (-48 * 60) and start_minutes < max_minute:
                    for minute in range(start_minutes, end_minutes):
                        minute_mod = minute % max_minute
                        if (not date) or (minute >= (-24 * 60) and minute < max_minute):
                            minute_index = minute_mod - 24 * 60
                            # For incremental adjustments we have to loop over 24-hour periods
                            while minute_index < stamp_end:
                                if not date or (minute_index >= start_minutes and minute_index < end_minutes):
                                    # Floor division, so yesterday's minutes are checked against yesterday's weekday
                                    current_day_of_week = (day_of_week_midnight + minute_index // (24 * 60)) % 7
                                    if not day_of_week or (current_day_of_week in day_of_week):
                                        if rate_increment:
                                            rates[minute_index] = rates.get(minute_index, 0.0) + rate
                                            rate_replicate[minute_index] = "increment"
                                        else:
                                            rates[minute_index] = rate
                                            rate_replicate[minute_index] = "user"
                                        if load_scaling is not None:
                                            self.load_scaling_dynamic[minute_index] = load_scaling
                                        if date:
                                            break
                                minute_index += 24 * 60
            else:
                self.log("Warn: Bad rate data provided in energy rates type {} {}".format(rtype, this_rate))

        return rates

    def rate_scan_export(self, rates, print=True):
        """
        Scan the rates and work out min/max
        """
        curr = self.currency_symbols[1]

        self.rate_export_min, self.rate_export_max, self.rate_export_average, self.rate_export_min_minute, self.rate_export_max_minute = self.rate_minmax(rates)
        if print:
            self.log("Export rates: min {}{}, max {}{}, average {}{}".format(self.rate_export_min, curr, self.rate_export_max, curr, self.rate_export_average, curr))

    def rate_minmax(self, rates):
        """
        Work out min and max rates
        """
        rate_min = 99999
        rate_min_minute = 0
        rate_max_minute = 0
        rate_max = 0
        rate_average = 0
        rate_n = 0
        rate = 0

        # Scan rates and find min/max/average
        if rates:
            rate = rates.get(self.minutes_now, 0)
            for minute in range(self.minutes_now, self.forecast_minutes + self.minutes_now):
                if minute in rates:
                    rate = rates[minute]
                    if rate > rate_max:
                        rate_max = rate
                        rate_max_minute = minute
                    if rate < rate_min:
                        rate_min = rate
                        rate_min_minute = minute
                    rate_average += rate
                    rate_n += 1
                minute += 1
        if rate_n:
            rate_average /= rate_n

        return dp2(rate_min), dp2(rate_max), dp2(rate_average), rate_min_minute, rate_max_minute

    def rate_base_min_max(self, rates):
        """
        Gap-fill `rates` into a "base" import curve (replicated, but without IO-slot/saving-session/
        override distortion) and work out its true min/max.

        Must scan the gap-filled curve, not the raw input: some tariffs (e.g. a fixed day/night
        product fetched with a short forward window - #4544) only have the currently-active segment
        in the raw data at fetch time, so a scan taken before replication can miss a cheaper/dearer
        segment that hasn't started yet and lock in the wrong "true" min/max for the rest of the day.
        rate_add_io_slots() then stamps IOG/SmartFlex dispatch slots with that stale value instead of
        the tariff's real off-peak price. Mirrors how rate_export_base is already built downstream of
        rate_replicate() on the export side.
        """
        rate_base, _ = self.rate_replicate(rates.copy(), {}, is_import=True)
        rate_min_base, rate_max_base, _, _, _ = self.rate_minmax(rate_base)
        return rate_base, rate_min_base, rate_max_base

    def rate_min_forward_calc(self, rates):
        """
        Work out lowest rate from time forwards
        """
        rate_array = []
        rate_min_forward = {}
        rate = self.rate_min

        for minute in range(self.forecast_minutes + self.minutes_now + 48 * 60):
            if minute in rates:
                rate = rates[minute]
            rate_array.append(rate)

        # Work out the min rate going forward — O(n) right-to-left scan avoids O(n²) slice allocations
        running_min = rate_array[-1] if rate_array else self.rate_min
        for minute in range(len(rate_array) - 1, self.minutes_now - 1, -1):
            running_min = min(running_min, rate_array[minute])
            if minute < self.forecast_minutes + 24 * 60 + self.minutes_now:
                rate_min_forward[minute] = running_min

        self.log("Rate min forward looking: now {}{} at end of forecast {}{}".format(dp2(rate_min_forward[self.minutes_now]), self.currency_symbols[1], dp2(rate_min_forward[self.forecast_minutes]), self.currency_symbols[1]))

        return rate_min_forward

    def rate_scan_window(self, rates, rate_low_min_window, threshold_rate, find_high, return_raw=False, alt_rates=None, pv_light_dark=None):
        """
        Scan for the next high/low rate window
        """
        alt_rates = alt_rates or {}
        pv_light_dark = pv_light_dark or {}
        minute = 0
        found_rates = []
        lowest = 99
        highest = -99

        while True:
            rate_low_start, rate_low_end, rate_low_average = self.find_charge_window(rates, minute, threshold_rate, find_high, alt_rates=alt_rates, pv_light_dark=pv_light_dark)
            window = {}
            window["start"] = rate_low_start
            window["end"] = rate_low_end
            window["average"] = dp2(rate_low_average)

            if rate_low_start >= 0:
                if (return_raw or (rate_low_end > self.minutes_now)) and (rate_low_end - rate_low_start) >= rate_low_min_window:
                    if rate_low_average < lowest:
                        lowest = rate_low_average
                    if rate_low_average > highest:
                        highest = rate_low_average

                    found_rates.append(window)
                minute = rate_low_end
            else:
                break

        return found_rates, lowest, highest

    def set_rate_thresholds(self):
        """
        Set the high and low rate thresholds
        """
        curr = self.currency_symbols[1]

        have_alerts = len(self.alert_active_keep) > 0
        have_manual_soc = len(self.manual_soc_keep) > 0

        car_planning_on_rates = self.num_cars > 0 and not self.octopus_intelligent_charging
        car_charging_max_price = max(self.car_charging_plan_max_price[: self.num_cars]) if car_planning_on_rates else 0.0

        if self.rate_low_threshold > 0:
            self.rate_import_cost_threshold = dp2(self.rate_average * self.rate_low_threshold)
        else:
            # In automatic mode select the only rate or everything but the most expensive
            if (self.rate_max == self.rate_min) or (self.rate_export_max > self.rate_max) or have_alerts or have_manual_soc:
                self.rate_import_cost_threshold = self.rate_max + 0.1
            else:
                self.rate_import_cost_threshold = self.rate_max - 0.5

        # When we plan car on the rates we need to include all the rates up to the max car price
        if car_planning_on_rates:
            self.rate_import_cost_threshold = max(self.rate_import_cost_threshold, car_charging_max_price + 0.1)

        # Compute the export rate threshold
        if self.rate_high_threshold > 0:
            self.rate_export_cost_threshold = dp2(self.rate_export_average * self.rate_high_threshold)
        else:
            # In automatic mode select the only rate or everything but the most cheapest
            if (self.rate_export_max == self.rate_export_min) or (self.rate_export_min > self.rate_min) or have_alerts:
                self.rate_export_cost_threshold = self.rate_export_min - 0.1
            else:
                self.rate_export_cost_threshold = self.rate_export_min + 0.5

        self.log(
            "Rate thresholds (for charge/export) are import {}{} ({}{}), export {}{} ({}{})".format(
                dp2(self.rate_import_cost_threshold), curr, dp2(self.rate_low_threshold), curr, dp2(self.rate_export_cost_threshold), curr, dp2(self.rate_high_threshold), curr
            )
        )

    def rate_scan(self, rates, print=True):
        """
        Scan the rates and work out min/max
        """
        self.rate_min, self.rate_max, self.rate_average, self.rate_min_minute, self.rate_max_minute = self.rate_minmax(rates)
        curr = self.currency_symbols[1]

        if print:
            # Calculate minimum forward rates only once rate replicate has run (when print is True).
            # Use the `rates` argument, not self.rate_import: during fetch's atomic rebuild self.rate_import
            # still holds the PREVIOUS cycle's data (publish is deferred to the end of the block), so
            # reading it here would make rate_min_forward - which drives plan.py charge/discharge economics
            # - a cycle stale. `rates` is the freshly-built current-cycle dict being scanned.
            self.rate_min_forward = self.rate_min_forward_calc(rates)
            self.log("Import rates: min {}{}, max {}{}, average {}{}".format(self.rate_min, curr, self.rate_max, curr, self.rate_average, curr))

    def rate_export_max_forward_calc(self, rates):
        """
        Work out the highest export rate from each minute forwards

        The export mirror of rate_min_forward_calc. Fed from the base export tariff (see
        rate_export_base) so a saving session cannot stand in for the tariff's general ability to sell
        surplus - that question is what the export haircut in battery_value_rate asks, and answering it
        with a one-off event price makes stored energy look fully recoverable on a system that in fact
        cannot sell a single kWh.
        """
        rate_array = []
        rate_export_max_forward = {}
        rate = 0.0

        for minute in range(self.forecast_minutes + self.minutes_now + 48 * 60):
            if minute in rates:
                rate = rates[minute]
            rate_array.append(rate)

        # Work out the max rate going forward — O(n) right-to-left scan avoids O(n²) slice allocations
        running_max = rate_array[-1] if rate_array else 0.0
        for minute in range(len(rate_array) - 1, self.minutes_now - 1, -1):
            running_max = max(running_max, rate_array[minute])
            if minute < self.forecast_minutes + 24 * 60 + self.minutes_now:
                rate_export_max_forward[minute] = running_max

        return rate_export_max_forward

    def rate_scan_gas(self, rates, print=True):
        """
        Scan the gas rates and work out min/max
        """
        self.rate_gas_min, self.rate_gas_max, self.rate_gas_average, self.rate_gas_min_minute, self.rate_gas_max_minute = self.rate_minmax(rates)
        curr = self.currency_symbols[1]

        if print:
            self.log("Gas rates: min {}{}, max {}{}, average {}{}".format(self.rate_gas_min, curr, self.rate_gas_max, curr, self.rate_gas_average, curr))

    def get_car_charging_now_tristate(self, car_n):
        """
        Read car_charging_now as a tri-state (True/False/None) for the #4516 Stage 1 dispatch
        timeline's GH#5080 reconciliation note.

        self.car_charging_now (set by get_car_charging_planned()) is a plain boolean used
        throughout the planner, where "no signal" and "confirmed not charging" both have to mean
        False - there is no room there for a genuine unknown. This diagnostic needs that
        distinction: flagging a slot as billing-risk when Predbat actually has no idea whether the
        car is charging would be a false positive on every dispatch, for every IOG user who hasn't
        configured the (optional) car_charging_now sensor, or during the "unknown"/"unavailable"
        state HA reports for a real sensor briefly after a restart.

        Returns None (unknown) when the sensor isn't configured, is out of range for this car, or
        HA is currently reporting "unknown"/"unavailable"; otherwise the normalised boolean.
        """
        if "car_charging_now" not in self.args:
            return None
        raw = self.get_arg("car_charging_now", "no", index=car_n)
        if raw is None:
            return None
        if isinstance(raw, str) and raw.lower() in ("unknown", "unavailable"):
            return None
        return self.car_charging_now[car_n]

    def get_car_charging_planned(self):
        """
        Get the car attributes
        """
        self.car_charging_planned = [False for c in range(self.num_cars)]
        self.car_charging_now = [False for c in range(self.num_cars)]
        self.car_charging_plan_smart = [False for c in range(self.num_cars)]
        self.car_charging_plan_max_price = [0 for c in range(self.num_cars)]
        self.car_charging_plan_time = ["07:00:00" for c in range(self.num_cars)]
        self.car_charging_battery_size = [100.0 for c in range(self.num_cars)]
        self.car_charging_limit = [100.0 for c in range(self.num_cars)]
        self.car_charging_rate = [7.4 for c in range(max(self.num_cars, 1))]
        self.car_charging_slots = [[] for c in range(self.num_cars)]
        self.car_charging_exclusive = [False for c in range(self.num_cars)]

        self.car_charging_planned_response = [str(response).lower() for response in self.get_arg("car_charging_planned_response", ["yes", "on", "enable", "true"])]
        self.car_charging_now_response = [str(response).lower() for response in self.get_arg("car_charging_now_response", ["yes", "on", "enable", "true"])]
        self.car_charging_from_battery = self.get_arg("car_charging_from_battery")

        # Car charging planned sensor
        for car_n in range(self.num_cars):
            # Get car N planned status
            planned = self.get_arg("car_charging_planned", "no", index=car_n)
            if isinstance(planned, str):
                if planned.lower() in self.car_charging_planned_response:
                    planned = True
                else:
                    planned = False
            elif not isinstance(planned, bool):
                planned = False
            self.car_charging_planned[car_n] = planned

            # Car is charging now sensor
            charging_now = self.get_arg("car_charging_now", "no", index=car_n)
            if isinstance(charging_now, str):
                if charging_now.lower() in self.car_charging_now_response:
                    charging_now = True
                else:
                    charging_now = False
            elif not isinstance(charging_now, bool):
                charging_now = False
            self.car_charging_now[car_n] = charging_now

            # Other car related configuration
            self.car_charging_plan_smart[car_n] = self.get_arg("car_charging_plan_smart", False)
            self.car_charging_plan_max_price[car_n] = self.get_arg("car_charging_plan_max_price", 0.0)
            self.car_charging_plan_time[car_n] = self.get_arg("car_charging_plan_time", "07:00:00")
            self.car_charging_battery_size[car_n] = dp2(float(self.get_arg("car_charging_battery_size", 100.0, index=car_n)))
            car_postfix = "" if car_n == 0 else "_" + str(car_n)
            self.car_charging_rate[car_n] = float(self.get_arg("car_charging_rate" + car_postfix))
            self.car_charging_limit[car_n] = dp3((float(self.get_arg("car_charging_limit", 100.0, index=car_n)) * self.car_charging_battery_size[car_n]) / 100.0)
            self.car_charging_exclusive[car_n] = self.get_arg("car_charging_exclusive", False, index=car_n)

        if self.num_cars > 0:
            car_charging_limit_percent = [dp1(limit / size * 100) if size else 0 for limit, size in zip(self.car_charging_limit, self.car_charging_battery_size)]
            self.log(
                "Cars {} charging from battery {} planned {}, charging_now {} smart {}, max_price {}{}, plan_time {}, battery size {}kWh, limit {}% ({}kWh), rate {}kW, exclusive {} (Predbat-led car settings, not Octopus Intelligent state)".format(
                    self.num_cars,
                    self.car_charging_from_battery,
                    self.car_charging_planned,
                    self.car_charging_now,
                    self.car_charging_plan_smart,
                    self.car_charging_plan_max_price,
                    self.currency_symbols[1],
                    self.car_charging_plan_time,
                    self.car_charging_battery_size,
                    car_charging_limit_percent,
                    self.car_charging_limit,
                    self.car_charging_rate,
                    self.car_charging_exclusive,
                )
            )

    def fetch_ml_load_forecast(self, now_utc):
        """
        Fetches ML load forecast from sensor
        and returns it as a minute_data dictionary
        """
        # Use ML Model for load prediction
        load_ml_forecast = self.get_state_wrapper("sensor." + self.prefix + "_load_ml_forecast", attribute="results")
        if load_ml_forecast:
            self.log("Loading ML load forecast from sensor.{}_load_ml_forecast".format(self.prefix))
            # Convert format from dict to array
            if isinstance(load_ml_forecast, dict):
                data_array = []
                for key, value in load_ml_forecast.items():
                    data_array.append({"energy": value, "last_updated": key})

                # Load data
                load_forecast, _ = minute_data(
                    data_array,
                    self.forecast_days + 1,
                    self.midnight_utc,
                    "energy",
                    "last_updated",
                    backwards=False,
                    clean_increment=False,
                    smoothing=True,
                    divide_by=1.0,
                    scale=1.0,
                )

                if load_forecast:
                    self.log("Loaded the ML load forecast; load so far today {}kWh, load today at midnight {}kWh".format(dp1(load_forecast.get(self.minutes_now, 0)), dp1(load_forecast.get(24 * 60 - PREDICT_STEP, 0))))
                    return load_forecast
        return {}

    def fetch_ml_load_forecast_history(self, now_utc):
        """
        Reconstruct a genuine past Load ML forecast for the elapsed part of today.

        sensor.<prefix>_load_ml_stats publishes a load_today_h1 attribute every cycle: the model's
        cumulative-load-since-midnight prediction for 60 minutes after that cycle ran. Shifting each
        recorded reading's timestamp forward by that same 60-minute lead turns the entity's history
        into a genuine record of what the model predicted for each past target minute - unlike a
        same-time-of-day historical average, this is the model's own past output (batpred#4750).

        Covers 2 days of history so a reading from shortly before local midnight (whose target minute
        falls just after midnight) is included, rather than leaving the first hour of the day as a gap.
        """
        entity_id = "sensor." + self.prefix + "_load_ml_stats"
        history = self.get_history_wrapper(entity_id, days=2, required=False)
        if not history:
            return {}

        data_array = []
        for record in history[0]:
            attributes = record.get("attributes") or {}
            value = attributes.get("load_today_h1")
            last_updated = record.get("last_updated")
            if value is None or not last_updated:
                continue
            try:
                shifted_time = str2time(last_updated) + timedelta(minutes=60)
            except (ValueError, TypeError):
                continue
            data_array.append({"energy": value, "last_updated": shifted_time.isoformat()})

        if not data_array:
            return {}

        load_forecast, _ = minute_data(
            data_array,
            self.forecast_days + 1,
            self.midnight_utc,
            "energy",
            "last_updated",
            backwards=False,
            clean_increment=False,
            smoothing=True,
            divide_by=1.0,
            scale=1.0,
        )
        return load_forecast or {}

    def get_holiday_minutes(self, now_utc, num_days):
        """
        Build a per-minute history of the holiday_days_left value (indexed by minutes-ago) from the recorded
        entity history using minute_data, which holds each state forward until the next change.

        Covers num_days + 1 days rather than num_days: a sample taken "num_days whole days ago at time of
        day tod" reaches minutes_now past the num_days boundary, so a num_days map leaves the oldest day
        in the window unreadable for every slot with tod <= minutes_now (batpred#4732). This mirrors the
        load history window, which is deliberately max_days_previous = window_days + 1 for the same reason.
        Both the fetch and the minute_data span have to be widened - minute_data bounds its map at
        days * 24 * 60, so raising the index alone would just return the not-on-holiday default instead.

        Returns the minute_data dict, or None when no usable history is available.
        """
        item = self.config_index.get("holiday_days_left", None) if getattr(self, "config_index", None) else None
        entity_id = item.get("entity") if item else ("input_number." + self.prefix + "_holiday_days_left")
        if not entity_id:
            return None

        history_days = num_days + 1
        try:
            history = self.get_history_wrapper(entity_id=entity_id, days=history_days, required=False)
        except (ValueError, TypeError):
            history = None

        if not history or not isinstance(history, list) or not history[0]:
            return None

        holiday_minutes, _ = minute_data(history[0], history_days, now_utc, "state", "last_updated", backwards=True)
        return holiday_minutes or None

    def compute_load_forecast_history(self, now_utc):
        """
        Build a forward load estimate from all available history (up to LOAD_FORECAST_HISTORY_MAX_DAYS days)
        using per-5-minute weighted-bucket averaging.

        For each forward 5-minute slot the historical sample at the same time-of-day is gathered from each
        available past day and combined as a weighted average, ignoring zero (missing-data) buckets entirely.

        Only days whose holiday state matches that of the day the slot falls on are averaged - the holiday
        state is matched per sample (not per whole day) so a mid-day change of holiday mode is handled
        correctly, and per forward day (not just today's) so the day you come home is planned against
        non-holiday history. Per-sample weight = weekday_factor * age_factor:
          - weekday_factor: 1.0 if the historical day is the same weekday as today; else 0.7 if both are
            weekend or both are weekday; else 0.5 (one weekday, one weekend). Held at 1.0 between two
            holiday days: holiday load has no weekday structure, and discarding weight from a pool of two
            or three matching days only amplifies noise.
          - age_factor: 0.9 for yesterday, reducing by 0.03 per day down to a floor of 0.1.

        A slot with no matching history at all - the first 24 hours of a holiday, or a return from one
        longer than the search window - falls back to the plain weighted average of every day scaled by
        holiday_load_scaling (or divided by it when coming home), so holiday mode acts from the moment it
        is switched on rather than waiting for a day of holiday history to accumulate (batpred#4732).

        Returns a cumulative-from-midnight kWh dict (same format as the ML load forecast), or {} if no data.
        """
        if not self.load_minutes or self.load_minutes_age < 1:
            return {}
        # Search window comes from max_days_previous (derived from days_previous), bounded by available history
        num_days = min(self.load_minutes_age, self.max_days_previous - 1)
        if num_days < 1:
            return {}

        today_dow = now_utc.weekday()
        holiday_minutes = self.get_holiday_minutes(now_utc, num_days)
        # get_holiday_minutes covers num_days + 1 days so the oldest day in the window stays readable for
        # every slot; anything older than that is genuinely unknown (a purged or short recorder history)
        max_holiday_index = (num_days + 1) * 24 * 60 - 1
        # Assumed holiday load as a fraction of normal, used only for slots with no matching history
        holiday_load_scaling = min(max(self.holiday_load_scaling, 0.1), 1.0)

        # Precompute the per-day weights; the holiday state is matched per 5-minute bucket below. The age-only
        # weight is what a holiday-to-holiday match uses, the weekday * age weight everything else.
        day_static_weight = {}
        day_age_weight = {}
        for d in range(1, num_days + 1):
            hist_dow = (now_utc - timedelta(days=d)).weekday()
            if hist_dow == today_dow:
                weekday_factor = 1.0
            elif (hist_dow >= 5) == (today_dow >= 5):
                # Both weekend or both weekday
                weekday_factor = 0.7
            else:
                weekday_factor = 0.5
            # Age: 0.9 for yesterday (d=1), reducing by 0.03 per day down to a floor of 0.1
            age_factor = max(0.1, 0.9 - (d - 1) * 0.03)
            day_age_weight[d] = age_factor
            day_static_weight[d] = weekday_factor * age_factor

        # Build the per-step (5-minute) weighted-average estimate, keyed by minute-from-midnight, across the
        # whole horizon from midnight today through the end of the plan so the resulting array is genuinely
        # cumulative-from-midnight (consumers such as load_today_comparison read minutes before minutes_now too).
        horizon_end = self.minutes_now + self.forecast_minutes + self.plan_interval_minutes
        per_step = {}
        for minute_absolute in range(0, horizon_end, PREDICT_STEP):
            tod = minute_absolute % (24 * 60)  # time of day of this slot
            # Holiday state of the day this slot falls on, not of today. holiday_days_left counts whole days
            # remaining including today and is decremented at midnight, so the day you travel home (and every
            # day after it) is planned against non-holiday history while you are still away.
            slot_holiday = self.holiday_days_left > (minute_absolute // (24 * 60))
            match_total = 0.0
            match_weight = 0.0
            all_total = 0.0
            all_weight = 0.0
            for d in range(1, num_days + 1):
                # Sample d whole days ago at this slot's time of day. Counting in whole days from today keeps
                # each day distinct and handles midnight crossings (the slot may be tomorrow or later).
                minute_previous = (self.minutes_now - tod) + d * 24 * 60
                # Skip zero (missing-data) buckets entirely so gaps in the history do not drag the estimate down.
                # base_in_raw=False keeps raw at the true measured load so genuine gaps are detected even when
                # a base load is configured (the filtered sample still has the base load applied).
                sample, raw = self.get_filtered_load_minute(self.load_minutes, minute_previous, historical=False, step=PREDICT_STEP, base_in_raw=False)
                if raw <= 0:
                    continue
                # Holiday state at the moment this individual sample was recorded (per bucket). A sample older
                # than the holiday history we hold is treated as not on holiday, the same default minute_data
                # gives for a gap inside the window - treating it as matching instead would hand full weight to
                # exactly the pre-holiday days holiday mode exists to discount.
                if holiday_minutes is None or minute_previous > max_holiday_index:
                    holiday_active = False
                else:
                    holiday_active = holiday_minutes.get(minute_previous, 0) > 0
                all_total += sample * day_static_weight[d]
                all_weight += day_static_weight[d]
                if holiday_active == slot_holiday:
                    weight = day_age_weight[d] if slot_holiday else day_static_weight[d]
                    match_total += sample * weight
                    match_weight += weight
            if match_weight > 0:
                per_step[minute_absolute] = match_total / match_weight
            elif all_weight > 0:
                # Nothing in the window shares this slot's holiday state. Rather than leaving holiday mode inert
                # (a normalised mean cannot express "all of my data is wrong"), fall back to every day scaled by
                # the assumed holiday ratio - or divided by it when we are home and the whole window was holiday.
                per_step[minute_absolute] = (all_total / all_weight) * (holiday_load_scaling if slot_holiday else 1.0 / holiday_load_scaling)
            else:
                per_step[minute_absolute] = 0.0

        # Convert per-step kWh buckets into a cumulative-from-midnight dict, filling every minute by linear
        # interpolation across each 5-minute span so get_from_incrementing(..., backwards=False) reproduces
        # the per-step energy exactly (mirroring the dense output of minute_data with smoothing).
        load_forecast = {}
        cumulative = 0.0
        for minute_absolute in range(0, horizon_end, PREDICT_STEP):
            step_energy = per_step[minute_absolute]
            for offset in range(PREDICT_STEP):
                load_forecast[minute_absolute + offset] = dp4(cumulative + step_energy * offset / PREDICT_STEP)
            cumulative += step_energy
        # Final boundary so the last span's increment is well defined
        load_forecast[horizon_end] = dp4(cumulative)
        return load_forecast

    def apply_load_ml_forecast_history(self, now_utc, ml_forecast=None):
        """
        Backfill the elapsed part of today with genuine past Load ML predictions.

        Only takes effect when Load ML is genuinely the active forecast source this cycle, which is
        exactly "fetch_sensor_data() fetched a non-empty ml_forecast this cycle" - a Load ML component
        running in the background without being selected as the forecast source
        (load_ml_enable/load_ml_source) must not influence the chart, so it has no effect and the
        weighted-bucket historical forecast above (skipped while Load ML owns load_forecast_only) is
        what would apply instead. This is passed in rather than held on self so there is no cycle-scoped
        flag whose correctness depends on fetch_config_options() always running first (#4762 review).
        Load ML's own forecast entity only ever publishes predictions from "now" onward
        (load_ml_component.py _publish_entity), so without this the already-elapsed part of today has no
        forecast data at all and load_today_comparison's Predicted chart series sits at zero from
        midnight until "now" (batpred#4750).
        """
        if not ml_forecast:
            return
        h1_forecast = self.fetch_ml_load_forecast_history(now_utc)
        for minute, value in h1_forecast.items():
            if minute < self.minutes_now:
                self.load_forecast[minute] = value

    def fetch_extra_load_forecast(self, now_utc, ml_forecast=None):
        """
        Fetch extra load forecast, this is future load data
        """
        load_forecast_final = {}
        load_forecast_array = []

        # Add ML forecast if available
        if ml_forecast:
            load_forecast_array.append(ml_forecast)

        if "load_forecast" in self.args:
            entity_ids = self.get_arg("load_forecast", indirect=False)
            if isinstance(entity_ids, str):
                entity_ids = [entity_ids]

            for entity_id in entity_ids:
                if not entity_id:
                    self.log("Warn: Unable to fetch load forecast data, check your load_forecast setting in apps.yaml")
                    continue

                attribute = None
                if "$" in entity_id:
                    entity_id, attribute = entity_id.split("$")
                try:
                    self.log("Loading extra load forecast from {}, attribute {}".format(entity_id, attribute))
                    data = self.get_state_wrapper(entity_id=entity_id, attribute=attribute)
                except (ValueError, TypeError) as e:
                    self.log("Error: Unable to fetch load forecast data from sensor {}, exception {}".format(entity_id, e))
                    data = None

                # Convert format from dict to array
                if isinstance(data, dict):
                    data_array = []
                    for key, value in data.items():
                        data_array.append({"energy": value, "last_updated": key})
                    data = data_array

                # Load data
                load_forecast, _ = minute_data(
                    data,
                    self.forecast_days + 1,
                    self.midnight_utc,
                    "energy",
                    "last_updated",
                    backwards=False,
                    clean_increment=False,
                    smoothing=True,
                    divide_by=1.0,
                    scale=1.0,
                    required_unit="kWh",
                )

                if load_forecast:
                    self.log("Loaded the load forecast from {} load sensor; from midnight {}kWh to now {}kWh to midnight {}kwh".format(entity_id, load_forecast.get(0, 0), load_forecast.get(self.minutes_now, 0), load_forecast.get(24 * 60, 0)))
                    load_forecast_array.append(load_forecast)
                else:
                    self.log("Warn: Unable to load the load forecast from {}. Skipping forecast source.".format(entity_id))

        # Add all the load forecasts together
        for load in load_forecast_array:
            for minute, value in load.items():
                if minute in load_forecast_final:
                    load_forecast_final[minute] += value
                else:
                    load_forecast_final[minute] = value
        return load_forecast_final, load_forecast_array

    def fetch_carbon_intensity(self, entity_id):
        """
        Fetch the carbon intensity from the sensor
        Returns a dictionary with the carbon intensity data for the next 24 hours
        And a secondary dictionary with the carbon intensity data for the last 24 hours

        :param entity_id: The entity_id of the sensor
        """
        data_all = []
        carbon_data = {}
        carbon_history = {}

        if entity_id:
            self.log("Fetching carbon intensity data from {}".format(entity_id))
            data_all = self.get_state_wrapper(entity_id=entity_id, attribute="forecast")
            if data_all:
                carbon_data, _ = minute_data(data_all, self.forecast_days, self.now_utc, "intensity", "from", backwards=False, to_key="to")
                carbon_data, carbon_replicated = self.carbon_replicate(carbon_data)
                if carbon_replicated:
                    self.log("Warn: Carbon intensity forecast only covers {} hours of the {} hour plan, replicating it forward to cover the rest".format(dp1(min(carbon_replicated) / 60), dp1(self.forecast_minutes / 60)))

        entity_id = self.prefix + ".carbon_now"
        state = self.get_state_wrapper(entity_id=entity_id)
        if state is not None:
            try:
                carbon_history = self.minute_data_import_export(self.max_days_previous, self.now_utc, entity_id, required_unit="g/kWh", increment=False, smoothing=False)
            except (ValueError, TypeError):
                self.log("Warn: No carbon intensity history in sensor {}".format(entity_id))
        else:
            self.log("Warn: No carbon intensity history in sensor {}".format(entity_id))

        return carbon_data, carbon_history

    def validate_curve(self, curve, curve_name):
        """
        Validate a power or temperature curve dictionary
        Ensures all keys are integers and all values are floats

        Args:
            curve: Dictionary to validate
            curve_name: Name of the curve for logging

        Returns:
            Validated dictionary with integer keys and float values
        """
        if not isinstance(curve, dict):
            self.log("Warn: {} is incorrectly configured - ignoring".format(curve_name))
            self.record_status("Warn: {} is incorrectly configured - ignoring".format(curve_name), had_errors=True)
            return {}

        validated_curve = {}
        for key in curve:
            try:
                int_key = int(key)
                value = float(curve[key])
                validated_curve[int_key] = value
            except (ValueError, TypeError):
                self.log("Warn: {} has bad key/value for key {} - ignoring".format(curve_name, key))
                self.record_status("Warn: {} has bad key/value for key {} - ignoring".format(curve_name, key), had_errors=True)

        return validated_curve

    def fetch_config_options(self):
        """
        Fetch all the configuration options
        """

        self.debug_enable = self.get_arg("debug_enable")
        self.plan_debug = self.get_arg("plan_debug")
        self.previous_status = self.get_state_wrapper(self.prefix + ".status")
        forecast_hours = max(self.get_arg("forecast_hours", 48), 24)

        self.num_cars = self.get_arg("num_cars", 1)
        if self.num_cars > PREDBAT_MAX_CARS:
            self.log("Warn: num_cars {} exceeds the {} cars Predbat supports - clamping to {}".format(self.num_cars, PREDBAT_MAX_CARS, PREDBAT_MAX_CARS))
            self.record_status("Warn: num_cars {} clamped to the maximum of {} supported cars".format(self.num_cars, PREDBAT_MAX_CARS), had_errors=True)
            self.num_cars = PREDBAT_MAX_CARS
        self.calculate_plan_every = max(self.get_arg("calculate_plan_every"), 5)
        self.calculate_savings_max_charge_slots = self.get_arg("calculate_savings_max_charge_slots", 1)

        self.log("Configuration: forecast_hours {}, num_cars {}, debug enable is {}, calculate_plan_every {} minutes".format(forecast_hours, self.num_cars, self.debug_enable, self.calculate_plan_every))

        # Days previous
        self.holiday_days_left = self.get_arg("holiday_days_left")
        self.holiday_load_scaling = self.get_arg("holiday_load_scaling", 0.7)
        self.load_forecast_only = self.get_arg("load_forecast_only", False)

        self.days_previous = self.get_arg("days_previous", [7])
        self.days_previous_weight = self.get_arg("days_previous_weight", [1 for i in range(len(self.days_previous))])
        if len(self.days_previous) > len(self.days_previous_weight):
            # Extend weights with 1 if required
            self.days_previous_weight += [1 for i in range(len(self.days_previous) - len(self.days_previous_weight))]

        # days_previous_auto enables the weighted-bucket historical load forecast. The number of days of history
        # it searches comes from max(days_previous) (or 7 when days_previous is not set), capped to the history
        # Predbat can hold (LOAD_FORECAST_HISTORY_MAX_DAYS). Enabled by default.
        self.load_forecast_history = self.get_arg("days_previous_auto", True)
        if self.load_forecast_history:
            window_days = min(max(self.days_previous) if self.days_previous else 7, LOAD_FORECAST_HISTORY_MAX_DAYS)
            # Config-time log only - describes what's enabled, not what happened this cycle. This
            # runs unconditionally every cycle regardless of whether the weighted-bucket forecast
            # actually gets used: Load ML (or any other source that sets load_forecast_only) takes
            # precedence and skips it entirely (fetch_sensor_data(), guarded by
            # "not self.load_forecast_only") - apply_load_ml_forecast_history() backfills the
            # elapsed part of today from Load ML's own history instead in that case (batpred#4750).
            # The "using weighted-bucket..." wording previously here read as if it was happening
            # every cycle regardless, which is what actually gets logged only when the forecast is
            # genuinely used (fetch_sensor_data()'s own "Using weighted-bucket historical load
            # forecast over N days" line) - confusing on a Load ML setup where this fallback is
            # rarely/never actually invoked (#4496 follow-up).
            self.log("days_previous_auto enabled - will fall back to a weighted-bucket historical load forecast over up to {} days if no other load forecast source takes precedence".format(window_days))
            self.max_days_previous = window_days + 1
        elif self.holiday_days_left > 0:
            self.days_previous = [1]
            self.log("Holiday mode is active, {} days remaining, setting days previous to 1".format(self.holiday_days_left))
            self.max_days_previous = max(self.days_previous) + 1
        else:
            self.max_days_previous = max(self.days_previous) + 1

        self.forecast_days = int((forecast_hours + 23) / 24)
        self.forecast_minutes = forecast_hours * 60
        self.forecast_plan_hours = max(min(self.get_arg("forecast_plan_hours"), forecast_hours), 8)
        self.inverter_clock_skew_start = self.get_arg("inverter_clock_skew_start", 0)
        self.inverter_clock_skew_end = self.get_arg("inverter_clock_skew_end", 0)
        self.inverter_clock_skew_discharge_start = self.get_arg("inverter_clock_skew_discharge_start", 0)
        self.inverter_clock_skew_discharge_end = self.get_arg("inverter_clock_skew_discharge_end", 0)
        self.inverter_can_charge_during_export = self.get_arg("inverter_can_charge_during_export", True)

        # Log clock skew
        if self.inverter_clock_skew_start != 0 or self.inverter_clock_skew_end != 0:
            self.log("Inverter clock skew start {} end {} applied".format(self.inverter_clock_skew_start, self.inverter_clock_skew_end))
        if self.inverter_clock_skew_discharge_start != 0 or self.inverter_clock_skew_discharge_end != 0:
            self.log("Inverter clock skew discharge start {} end {} applied".format(self.inverter_clock_skew_discharge_start, self.inverter_clock_skew_discharge_end))

        # Metric config
        self.metric_min_improvement = self.get_arg("metric_min_improvement")
        self.metric_min_improvement_export = self.get_arg("metric_min_improvement_export")
        self.metric_min_improvement_swap = self.get_arg("metric_min_improvement_swap")
        self.metric_min_improvement_plan = self.get_arg("metric_min_improvement_plan")
        self.metric_min_improvement_export_freeze = self.get_arg("metric_min_improvement_export_freeze")
        self.metric_battery_cycle = self.get_arg("metric_battery_cycle")
        self.metric_self_sufficiency = self.get_arg("metric_self_sufficiency")
        self.metric_future_rate_offset_import = self.get_arg("metric_future_rate_offset_import")
        self.metric_future_rate_offset_export = self.get_arg("metric_future_rate_offset_export")
        self.metric_inday_adjust_damping = self.get_arg("metric_inday_adjust_damping")
        self.metric_pv_calibration_enable = self.get_arg("metric_pv_calibration_enable")
        self.metric_dynamic_load_adjust = self.get_arg("metric_dynamic_load_adjust")
        self.rate_low_threshold = self.get_arg("rate_low_threshold")
        self.rate_high_threshold = self.get_arg("rate_high_threshold")
        self.inverter_soc_reset = self.get_arg("inverter_soc_reset")

        self.metric_battery_value_scaling = self.get_arg("metric_battery_value_scaling")
        self.metric_battery_value_export_scaling = self.get_arg("metric_battery_value_export_scaling")
        self.notify_devices = self.get_arg("notify_devices", ["notify"])
        self.pv_scaling = self.get_arg("pv_scaling")
        self.pv_metric10_weight = self.get_arg("pv_metric10_weight")
        self.calculate_pv90_plan = self.get_arg("calculate_pv90_plan")
        self.pv_metric90_weight = self.get_arg("pv_metric90_weight")
        self.load_scaling = self.get_arg("load_scaling")
        self.load_scaling10 = self.get_arg("load_scaling10")
        self.load_scaling90 = self.get_arg("load_scaling90")
        if not self.calculate_pv90_plan:
            # calculate_pv90_plan (CHANGE 2) gates the whole feature: pv_metric90_weight's own
            # CONFIG_ITEMS default is 0.15, not 0.0, so the user sees a non-zero weight in Home
            # Assistant even while the switch is off. Force the runtime weight back to 0.0 here so
            # every one of the pre-existing `pv_metric90_weight > 0` gates elsewhere (the pv90 launch
            # paths in plan.py, and the pv90 term collapsing out of compute_metric) stays inert until
            # the user explicitly turns the switch on - no separate gating on calculate_pv90_plan is
            # added anywhere else, this is the single choke point.
            self.pv_metric90_weight = 0.0

        self.charge_scaling10 = self.get_arg("charge_scaling10")
        self.load_scaling_saving = self.get_arg("load_scaling_saving")
        self.load_scaling_free = self.get_arg("load_scaling_free")
        self.battery_rate_max_scaling = self.get_arg("battery_rate_max_scaling")
        self.battery_rate_max_scaling_discharge = self.get_arg("battery_rate_max_scaling_discharge")

        self.best_soc_step = 0.25
        self.metric_cloud_enable = self.get_arg("metric_cloud_enable")
        self.metric_load_divergence_enable = self.get_arg("metric_load_divergence_enable")

        # Battery charging options
        self.battery_capacity_nominal = self.get_arg("battery_capacity_nominal")
        self.battery_scaling_auto = self.get_arg("battery_scaling_auto", default=False)
        self.battery_loss = 1.0 - self.get_arg("battery_loss")
        self.battery_loss_discharge = 1.0 - self.get_arg("battery_loss_discharge")
        self.inverter_loss = 1.0 - self.get_arg("inverter_loss")
        self.inverter_hybrid = self.get_arg("inverter_hybrid")
        self.pv_ac_limit = self.get_arg("pv_ac_limit", 0.0) / MINUTE_WATT
        self.inverter_freeze_export_discharge_rate = max(self.get_arg("inverter_freeze_export_discharge_rate", 0.0), 0.0) / MINUTE_WATT
        self.log("Freeze Export discharge rate configured: {:.0f} W".format(self.inverter_freeze_export_discharge_rate * MINUTE_WATT))
        self.base_load = self.get_arg("base_load", 100) / 1000.0

        # Charge curve
        if self.args.get("battery_charge_power_curve", "") == "auto":
            self.battery_charge_power_curve_auto = True
        else:
            self.battery_charge_power_curve_auto = False
            self.battery_charge_power_curve = self.validate_curve(self.args.get("battery_charge_power_curve", {}), "battery_charge_power_curve")
        self.battery_charge_power_curve_default = self.validate_curve(self.args.get("battery_charge_power_curve_default", {}), "battery_charge_power_curve_default")

        # Discharge curve
        if self.args.get("battery_discharge_power_curve", "") == "auto":
            self.battery_discharge_power_curve_auto = True
        else:
            self.battery_discharge_power_curve_auto = False
            self.battery_discharge_power_curve = self.validate_curve(self.args.get("battery_discharge_power_curve", {}), "battery_discharge_power_curve")
        self.battery_discharge_power_curve_default = self.validate_curve(self.args.get("battery_discharge_power_curve_default", {}), "battery_discharge_power_curve_default")

        # Temperature curve charge/discharge
        self.battery_temperature_charge_curve = self.validate_curve(self.args.get("battery_temperature_charge_curve", {}), "battery_temperature_charge_curve")
        self.battery_temperature_discharge_curve = self.validate_curve(self.args.get("battery_temperature_discharge_curve", {}), "battery_temperature_discharge_curve")

        self.import_export_scaling = self.get_arg("import_export_scaling", 1.0)
        self.best_soc_min = self.get_arg("best_soc_min")
        self.best_soc_max = self.get_arg("best_soc_max")
        self.best_soc_keep = self.get_arg("best_soc_keep")
        self.best_soc_keep_weight = self.get_arg("best_soc_keep_weight")
        self.set_soc_minutes = self.plan_interval_minutes
        self.set_window_minutes = self.plan_interval_minutes
        self.inverter_set_charge_before = self.get_arg("inverter_set_charge_before")
        if not self.inverter_set_charge_before:
            self.set_soc_minutes = 0
            self.set_window_minutes = 0
        self.octopus_intelligent_charging = self.get_arg("octopus_intelligent_charging")
        self.octopus_intelligent_ignore_unplugged = self.get_arg("octopus_intelligent_ignore_unplugged")
        self.octopus_intelligent_consider_full = self.get_arg("octopus_intelligent_consider_full")
        self.octopus_intelligent_trust_slots = self.get_arg("octopus_intelligent_trust_slots")
        self.car_energy_reported_load = self.get_arg("car_energy_reported_load")
        self.get_car_charging_planned()
        self.dynamic_load_car_check_config()
        self.load_inday_adjustment = 1.0

        self.combine_rate_threshold = self.get_arg("combine_rate_threshold")
        self.combine_export_slots = self.get_arg("combine_export_slots")
        self.combine_charge_slots = self.get_arg("combine_charge_slots")
        self.charge_slot_split = self.plan_interval_minutes
        self.export_slot_split = self.plan_interval_minutes
        self.calculate_best = True
        self.set_read_only = self.get_arg("set_read_only")
        self.set_read_only_axle = False

        # Check if Axle control is enabled and an event is active
        if self.get_arg("axle_control", False) and not self.set_read_only:
            if fetch_axle_active(self):
                self.log("Axle VPP event is active - enabling read-only mode")
                self.set_read_only = True
                self.set_read_only_axle = True

        # hard wired options, can be configured per inverter later on
        self.set_soc_enable = True
        self.set_reserve_enable = self.get_arg("set_reserve_enable")
        self.set_reserve_hold = True
        self.set_export_freeze = self.get_arg("set_export_freeze")
        self.set_charge_freeze = self.get_arg("set_charge_freeze")
        self.set_charge_freeze_only = self.get_arg("set_charge_freeze_only")
        self.set_charge_low_power = self.get_arg("set_charge_low_power")
        self.set_export_low_power = self.get_arg("set_export_low_power")
        self.charge_low_power_margin = self.get_arg("charge_low_power_margin")
        self.low_power_pv_threshold_w = self.get_arg("low_power_pv_threshold_w")
        self.set_charge_low_power_solar_full_rate = self.get_arg("set_charge_low_power_solar_full_rate")
        self.calculate_export_first = True

        self.set_status_notify = self.get_arg("set_status_notify")
        self.set_inverter_notify = self.get_arg("set_inverter_notify")
        self.set_export_freeze_only = self.get_arg("set_export_freeze_only")
        self.set_discharge_during_charge = self.get_arg("set_discharge_during_charge")
        self.set_freeze_export_during_demand = self.get_arg("set_freeze_export_during_demand")
        self.export_more_solar = self.get_arg("export_more_solar")
        self.export_more_solar_threshold = self.get_arg("export_more_solar_threshold")
        # Mode
        self.predbat_mode = self.get_arg("mode")
        if self.predbat_mode == PREDBAT_MODE_OPTIONS[PREDBAT_MODE_CONTROL_SOC]:
            self.calculate_best_charge = True
            self.calculate_best_export = False
            self.set_charge_window = False
            self.set_export_window = False
            self.set_soc_enable = True
        elif self.predbat_mode == PREDBAT_MODE_OPTIONS[PREDBAT_MODE_CONTROL_CHARGE]:
            self.calculate_best_charge = True
            self.calculate_best_export = False
            self.set_charge_window = True
            self.set_export_window = False
            self.set_soc_enable = True
        elif self.predbat_mode == PREDBAT_MODE_OPTIONS[PREDBAT_MODE_CONTROL_CHARGEDISCHARGE]:
            self.calculate_best_charge = True
            self.calculate_best_export = True
            self.set_charge_window = True
            self.set_export_window = True
            self.set_soc_enable = True
        else:  # PREDBAT_MODE_OPTIONS[PREDBAT_MODE_MONITOR]
            self.calculate_best_charge = False
            self.calculate_best_export = False
            self.set_charge_window = False
            self.set_export_window = False
            self.predbat_mode = PREDBAT_MODE_OPTIONS[PREDBAT_MODE_MONITOR]
            self.set_soc_enable = False
            self.expose_config("mode", self.predbat_mode)

        self.log("Predbat mode is set to {}".format(self.predbat_mode))

        self.calculate_export_oncharge = self.get_arg("calculate_export_oncharge")
        self.calculate_export_on_pv = self.get_arg("calculate_export_on_pv")
        self.calculate_second_pass = self.get_arg("calculate_second_pass")
        self.prediction_kernel_enable = self.get_arg("prediction_kernel_enable", True)
        self.calculate_inday_adjustment = self.get_arg("calculate_inday_adjustment")
        self.calculate_regions = True
        self.calculate_import_low_export = self.get_arg("calculate_import_low_export")
        self.calculate_export_high_import = self.get_arg("calculate_export_high_import")

        self.balance_inverters_enable = self.get_arg("balance_inverters_enable")
        self.balance_inverters_charge = self.get_arg("balance_inverters_charge")
        self.balance_inverters_discharge = self.get_arg("balance_inverters_discharge")
        self.balance_inverters_crosscharge = self.get_arg("balance_inverters_crosscharge")
        self.balance_inverters_threshold_charge = max(self.get_arg("balance_inverters_threshold_charge"), 1.0)
        self.balance_inverters_threshold_discharge = max(self.get_arg("balance_inverters_threshold_discharge"), 1.0)

        if self.set_read_only:
            self.log("Info: Read-only mode is enabled, the inverter controls will not be used!!")

        # Enable load filtering
        self.load_filter_modal = self.get_arg("load_filter_modal")

        # Calculate savings
        self.calculate_savings = True

        # Carbon
        self.carbon_enable = self.get_arg("carbon_enable")
        self.carbon_metric = self.get_arg("carbon_metric")

        # iBoost solar diverter model
        self.iboost_enable = self.get_arg("iboost_enable")
        self.iboost_gas = self.get_arg("iboost_gas")
        self.iboost_gas_export = self.get_arg("iboost_gas_export")
        self.iboost_smart = self.get_arg("iboost_smart")
        self.iboost_smart_min_length = self.get_arg("iboost_smart_min_length")
        self.iboost_on_export = self.get_arg("iboost_on_export")
        self.iboost_prevent_discharge = self.get_arg("iboost_prevent_discharge")
        self.iboost_solar = self.get_arg("iboost_solar")
        self.iboost_solar_excess = self.get_arg("iboost_solar_excess")
        self.iboost_rate_threshold = self.get_arg("iboost_rate_threshold")
        self.iboost_rate_threshold_export = self.get_arg("iboost_rate_threshold_export")
        self.iboost_charging = self.get_arg("iboost_charging")
        self.iboost_gas_scale = self.get_arg("iboost_gas_scale")
        self.iboost_max_energy = self.get_arg("iboost_max_energy")
        self.iboost_max_power = self.get_arg("iboost_max_power") / MINUTE_WATT
        self.iboost_min_power = self.get_arg("iboost_min_power") / MINUTE_WATT
        self.iboost_min_soc = self.get_arg("iboost_min_soc")
        self.iboost_today = self.get_arg("iboost_today")
        self.iboost_value_scaling = self.get_arg("iboost_value_scaling")
        self.iboost_energy_subtract = self.get_arg("iboost_energy_subtract")
        self.iboost_next = self.iboost_today
        self.iboost_running = False
        self.iboost_running_solar = False
        self.iboost_running_full = False
        self.iboost_energy_today = {}
        self.iboost_plan = []

        # Car options
        self.car_charging_hold = self.get_arg("car_charging_hold")
        if not self.car_energy_reported_load:
            # If car energy is not reported as load then we should not attempt to remove car energy from the load data.
            self.car_charging_hold = False
        self.car_charging_manual_soc = [False for c in range(max(self.num_cars, 1))]
        for car_n in range(self.num_cars):
            car_postfix = "" if car_n == 0 else "_" + str(car_n)
            self.car_charging_manual_soc[car_n] = self.get_arg("car_charging_manual_soc" + car_postfix, False)
        self.car_charging_threshold = float(self.get_arg("car_charging_threshold")) / 60.0
        self.car_charging_energy_scale = self.get_arg("car_charging_energy_scale")

        # Update list of slot times
        self.manual_charge_times = self.manual_times("manual_charge")
        self.manual_export_times = self.manual_times("manual_export")
        self.manual_freeze_charge_times = self.manual_times("manual_freeze_charge")
        self.manual_freeze_export_times = self.manual_times("manual_freeze_export")
        self.manual_demand_times = self.manual_times("manual_demand")
        self.manual_all_times = self.manual_charge_times + self.manual_export_times + self.manual_demand_times + self.manual_freeze_charge_times + self.manual_freeze_export_times
        self.manual_api = self.api_select_update("manual_api")
        self.manual_import_rates = self.manual_rates("manual_import_rates", default_rate=self.get_arg("manual_import_value"))
        self.manual_export_rates = self.manual_rates("manual_export_rates", default_rate=self.get_arg("manual_export_value"))
        self.manual_load_adjust = self.manual_rates("manual_load_adjust", default_rate=self.get_arg("manual_load_value"))
        self.manual_soc_keep = self.manual_rates("manual_soc", default_rate=self.get_arg("manual_soc_value"))
        self.manual_soc_max_keep = self.manual_rates("manual_soc_max", default_rate=self.get_arg("manual_soc_max_value"))

        # Update list of config options to save/restore to
        self.update_save_restore_list()

    def load_car_energy(self, now_utc):
        """
        Load previous car charging energy data
        """
        self.car_charging_energy = {}
        if "car_charging_energy" in self.args:
            self.car_charging_energy = self.minute_data_import_export(self.max_days_previous, now_utc, "car_charging_energy", scale=self.car_charging_energy_scale, required_unit="kWh")
            if self.car_charging_hold and not self.car_charging_energy:
                # car_charging_energy is configured (an entity is set) but resolved to nothing at
                # all - most likely the entity no longer exists (e.g. removed upstream, see #4458),
                # though it can also happen for a recorder-excluded entity or a fresh install with
                # no history yet, which aren't necessarily broken and can persist indefinitely.
                # minute_data_import_export() already logs a benign "Unable to fetch history"
                # warning per entity with no had_errors flag, so this degrades to the
                # car_charging_threshold heuristic below completely silently otherwise. Log the
                # loud warning only once per incident (not every cycle - see PR review on #4469)
                # so a persistent-but-intentional setup doesn't get spammed every 5 minutes; the
                # status sensor still reflects the degraded state every cycle via record_status.
                if not self.car_charging_energy_warned:
                    self.log(
                        "Warn: car_charging_hold is enabled and car_charging_energy is configured ({}) but no data could be loaded for it - falling back to the car_charging_threshold heuristic instead of precise car energy subtraction. Check the entity still exists.".format(
                            self.get_arg("car_charging_energy", indirect=False)
                        )
                    )
                    self.car_charging_energy_warned = True
                self.record_status("Warn: car_charging_energy entity has no data, falling back to car_charging_threshold heuristic", had_errors=True)
            else:
                self.car_charging_energy_warned = False
        else:
            self.log("Car charging hold {}, threshold {}kWh".format(self.car_charging_hold, self.car_charging_threshold * 60.0))
        return self.car_charging_energy