fix(db): treat an unreadable image as proof of corruption

.cspell/custom-dictionary-workspace.txt

@@ -94,6 +94,7 @@ datalog
 datap
 datapoints
 datas
+datasheet
 dataviz
 dateutil
 dayname
@@ -150,6 +151,7 @@ enphaseenergy
 Enpower
 entrez
 epod
+errorcode
 etok
 euids
 EVDC
@@ -180,6 +182,7 @@ freephase
 fronius
 funcs
 futurerate
+gasbill
 gecloud
 gekey
 generationmix
@@ -336,6 +339,7 @@ nordpool
 noreferrer
 nosc
 nosuchrun
+NOTADB
 nrgheat
 numpy
 OCPP
@@ -533,6 +537,7 @@ unnormalised
 unparseable
 unsmoothed
 unstaged
+unsubtracted
 urandom
 urlsafe
 useid

apps/predbat/annual.py

@@ -1057,9 +1057,7 @@ def _billed_result(predbat, end_record, pv_step):
     was_debug_enable = pred.debug_enable
     pred.debug_enable = True
     try:
-        cost, _, _, _, _, final_soc, _, battery_cycle, _, final_iboost, _ = predbat.run_prediction(
-            predbat.charge_limit_best, predbat.charge_window_best, predbat.export_window_best, predbat.export_limits_best, False, end_record=end_record
-        )
+        cost, _, _, _, _, final_soc, _, battery_cycle, _, final_iboost, _ = predbat.run_prediction(predbat.charge_limit_best, predbat.charge_window_best, predbat.export_window_best, predbat.export_limits_best, False, end_record=end_record)
         # Deliberately unguarded: if upstream ever renames these, a hard AttributeError is
         # the outcome to want. Falling back to the returned tuple would silently restore
         # the whole-plan double-count this exists to avoid.
@@ -1662,7 +1660,9 @@ async def run(self, progress=None):
                 self.gas_load_source = self.load_source
             self.heat_load_source = HeatPumpLoadProfile(self.gas_load_source, self.heat_model)
             self.caveats.append(
-                "The heat pump comparison runs every scenario twice - once on gas, once on the heat pump - so the heat pump's extra electricity is what remains after PV, the battery and Predbat, not its consumption at an average rate. Gas is priced at a flat {} p/kWh with no seasonal variation.".format(self.heat_model.settings["gas_p_per_kwh"])
+                "The heat pump comparison runs every scenario twice - once on gas, once on the heat pump - so the heat pump's extra electricity is what remains after PV, the battery and Predbat, not its consumption at an average rate. Gas is priced at a flat {} p/kWh with no seasonal variation.".format(
+                    self.heat_model.settings["gas_p_per_kwh"]
+                )
             )
             self.caveats.append(
                 "Heat demand is spread across the year by heating degree days from {}'s real hourly temperatures, and COP varies with outdoor temperature around your {} SCOP (realised {}). Hot water runs at a lower COP than space heating because the cylinder needs a higher flow temperature.".format(
@@ -1683,7 +1683,11 @@ async def run(self, progress=None):
                     )
                 )
             else:
-                self.caveats.append("The hot water cylinder is on a fixed timer, reheating to be ready for {} draw-off(s) a day. Ticking the smart cylinder option lets it charge on the cheapest rates instead, which is usually worth considerably more than anything else in this comparison.".format(heat_settings["water_charges_per_day"]))
+                self.caveats.append(
+                    "The hot water cylinder is on a fixed timer, reheating to be ready for {} draw-off(s) a day. Ticking the smart cylinder option lets it charge on the cheapest rates instead, which is usually worth considerably more than anything else in this comparison.".format(
+                        heat_settings["water_charges_per_day"]
+                    )
+                )
 
             # A cylinder that cannot physically hold a day's hot water in the configured
             # number of charges is a config error the user can fix (more charges, or a
@@ -1957,10 +1961,7 @@ def _build_heat_results(self, included):
 
         scenarios = {}
         for key in SCENARIO_KEYS:
-            scenarios[key] = {
-                field: round(sum(entry["heat"]["scenarios"][key][field] for entry in heat_months), 3)
-                for field in ["electricity_gas_p", "electricity_heatpump_p", "extra_electricity_p", "annual_saving_p"]
-            }
+            scenarios[key] = {field: round(sum(entry["heat"]["scenarios"][key][field] for entry in heat_months), 3) for field in ["electricity_gas_p", "electricity_heatpump_p", "extra_electricity_p", "annual_saving_p"]}
 
         # The electricity the heat pump itself drew, before PV or the battery served any of
         # it. Reported alongside the extra COST because the two are very different numbers

apps/predbat/annual_heat.py

@@ -37,7 +37,6 @@
 
 import calendar
 import math
-from datetime import date, timedelta
 
 # Ofgem's medium Typical Domestic Consumption Value for gas, kWh a year. Only ever a
 # default for a user who has not looked at a bill yet.
@@ -231,9 +230,7 @@ def resolve_heat(raw):
         # round. space_heat_kwh/water_heat_kwh are filled in by HeatModel once the COP curve
         # has been calibrated - see HeatModel._derive_existing_heat().
         if settings["water_heat_pump_kwh"] > settings["heat_pump_kwh"]:
-            raise AnnualHeatError(
-                "annual.heat.water_heat_pump_kwh ({}) cannot exceed heat_pump_kwh ({}): hot water is part of what the heat pump uses, not on top of it".format(settings["water_heat_pump_kwh"], settings["heat_pump_kwh"])
-            )
+            raise AnnualHeatError("annual.heat.water_heat_pump_kwh ({}) cannot exceed heat_pump_kwh ({}): hot water is part of what the heat pump uses, not on top of it".format(settings["water_heat_pump_kwh"], settings["heat_pump_kwh"]))
         settings["space_heat_kwh"] = None
         settings["water_heat_kwh"] = None
         return settings

apps/predbat/db_engine.py

@@ -31,6 +31,26 @@
 # whole file, evicting a database-sized amount of page cache, so it is not worth doing
 # for a small saving on a memory-constrained machine.
 VACUUM_MIN_FREE_BYTES = 256 * 1024 * 1024
+# SQLite result codes that positively identify a damaged image, as opposed to a file that is
+# merely busy, unreachable, or held open by someone else
+SQLITE_CORRUPT = 11
+SQLITE_NOTADB = 26
+
+
+def _is_corruption_error(error):
+    """
+    Is this SQLite error a positive report that the database image is damaged?
+
+    Matched on the result code rather than the exception class, because sqlite3.DatabaseError
+    covers far more than corruption - ProgrammingError ("Cannot operate on a closed database")
+    is one of its subclasses, and a bug of our own must never be read as a reason to throw the
+    user's history away.
+    """
+    code = getattr(error, "sqlite_errorcode", None)
+    if code is None:
+        return False
+    # Extended result codes carry the primary code in the low byte (SQLITE_CORRUPT_INDEX = 779)
+    return (code & 0xFF) in (SQLITE_CORRUPT, SQLITE_NOTADB)
 
 
 class DatabaseEngine:
@@ -110,13 +130,22 @@ def _cleanup_db_retry(self):
             self.log("Warn: db_engine: Giving up after {} attempts ({}) - not corruption, leaving the database alone".format(DB_OPEN_RETRIES, last_error))
             return
 
-        # Quarantine only on a *positive* report of damage. quick_check failing to run at all
-        # says nothing about the file, so anything other than a clear non-ok verdict has to
-        # keep the database - throwing away history needs proof, not the absence of proof.
+        # Quarantine only on a *positive* report of damage. A check that simply cannot reach a
+        # verdict has to keep the database - throwing away history needs proof, not the absence
+        # of proof.
         try:
             result = self.db_cursor.execute("PRAGMA quick_check(10)").fetchone()
         except sqlite3.DatabaseError as e:
-            self.log("Warn: db_engine: quick_check could not run ({}) - keeping the database".format(e))
+            # An unreadable image is reported this way rather than as a verdict: SQLite that
+            # cannot recognise the header never gets far enough to grade the pages. That is a
+            # positive finding, and a file SQLite will not read is the permanently corrupt file
+            # quarantine exists for. Keeping it would leave Predbat running with db_primary set
+            # and every history write failing for the life of the process, unnoticed.
+            if _is_corruption_error(e):
+                self.log("Error: db_engine: quick_check could not run, the image is unreadable: {}".format(e))
+                self._quarantine_db()
+            else:
+                self.log("Warn: db_engine: quick_check could not run ({}) - keeping the database".format(e))
             return
 
         if not result: