@@ -185,6 +185,11 @@ class Redactor:
185185 key, which is rewritten by exact match only - see _substitute_key), and a value inside
186186 hardware_ids is only shape-flagged when it is nothing BUT digits, since a letter-prefixed
187187 vendor serial with a long digit tail is that container's entire declared purpose.
188+
189+ A hardware serial is not secret, so a serial a record declares - hardware_ids.serial or an
190+ entry in the structural serials list - is clear wherever it appears as a whole token, however
191+ many digits it has (Solis, Deye and Sunsynk serials are nothing but digits): in hardware_ids,
192+ in a device_id built from it, in a duplicate_serial observation. See _collect_serials.
188193 """
189194
190195 # Minimum length of an original before it is substituted inside other strings; below this a
@@ -206,6 +211,8 @@ def __init__(self, salt, log=None):
206211 # unrelated text it happens to share a substring with.
207212 self .substring_ok = set ()
208213 self ._substring_order = []
214+ # Whole-token patterns for every serial the document's records declare - see _collect_serials
215+ self ._serial_patterns = []
209216
210217 def token (self , value ):
211218 """The stable pseudonym for one value under this installation's salt."""
@@ -290,6 +297,88 @@ def _numeric_variants(self, text):
290297 variants .add (sign + body [:- 2 ])
291298 return variants
292299
300+ @staticmethod
301+ def _declared_serials (record ):
302+ """The serials one record declares for itself: hardware_ids.serial and every entry in serials."""
303+ declared = set ()
304+ hardware_ids = record .get ("hardware_ids" )
305+ if isinstance (hardware_ids , dict ) and hardware_ids .get ("serial" ) not in None , "" ):
306+ declared .add (str (hardware_ids ["serial" ]))
307+ serials = record .get ("serials" )
308+ if isinstance (serials , list ):
309+ declared |= {str (serial ) for serial in serials if serial not in None , "" )}
310+ return declared
311+
312+ def _collect_serials (self , node ):
313+ """Every serial any record in the document declares, anywhere in it.
314+
315+ Collected before the walk so the shape guard can leave a serial readable wherever it
316+ turns up - a device_id built from it, the duplicate_serial observation that names it -
317+ not only inside the record that declared it.
318+ """
319+ found = set ()
320+ if isinstance (node , dict ):
321+ found |= self ._declared_serials (node )
322+ for value in node .values ():
323+ found |= self ._collect_serials (value )
324+ elif isinstance (node , list ):
325+ for entry in node :
326+ found |= self ._collect_serials (entry )
327+ return found
328+
329+ def _strip_serials (self , text ):
330+ """`text` with every declared serial removed where it stands as a whole token.
331+
332+ Whole-token only - bounded by a non-alphanumeric character or the string's ends, matched
333+ case-insensitively as entity ids fold case - so "solis:1031260253072197" and
334+ "sensor.predbat_solis_1031260253072197_soc" lose the serial, but a short serial that
335+ happens to sit inside a longer number does not split that number's digit run and let a
336+ misfiled MPAN through.
337+ """
338+ for pattern in self ._serial_patterns :
339+ text = pattern .sub ("" , text )
340+ return text
341+
342+ def _pseudonym_values (self , node ):
343+ """Every value held in a pseudonym container (account_ids) anywhere in this record."""
344+ values = set ()
345+ if isinstance (node , dict ):
346+ for key , value in node .items ():
347+ if key in PSEUDONYM_CONTAINERS and isinstance (value , dict ):
348+ values |= {str (entry ) for entry in value .values ()}
349+ else :
350+ values |= self ._pseudonym_values (value )
351+ elif isinstance (node , list ):
352+ for entry in node :
353+ values |= self ._pseudonym_values (entry )
354+ return values
355+
356+ @staticmethod
357+ def _whole_token (text ):
358+ """A case-insensitive pattern matching `text` only where it stands as a whole token.
359+
360+ Bounded by a non-alphanumeric character or the string's ends, so "2306178123" is found in
361+ "deye:2306178123" and "sensor.predbat_solis_2306178123_soc" but not inside "inv98765432".
362+ """
363+ return re .compile (r"(?<![0-9A-Za-z]){}(?![0-9A-Za-z])" .format (re .escape (text )), re .IGNORECASE )
364+
365+ def _serial_derived (self , record ):
366+ """Whether a record's device_id is built from its own serial rather than from an account identifier.
367+
368+ True only when a serial the record declares stands in the device_id as a whole token and
369+ none of the record's own pseudonym values does - whole tokens both ways, as the shape
370+ guard matches serials (see _strip_serials). A serial merely sitting inside a longer token
371+ is coincidence, not construction; and an account id counts however short it is, since one
372+ under MIN_SUBSTITUTE is not otherwise caught by the substring pass. Deye's "deye:{serial}"
373+ beside a station id in account_ids is serial-derived; Octopus's "octopus:{mpan}" declares
374+ no serial and a device_id that embeds an account id does, so both stay identity-derived
375+ exactly as before.
376+ """
377+ device_id = record ["device_id" ]
378+ if not any (self ._whole_token (serial ).search (device_id ) for serial in self ._declared_serials (record )):
379+ return False
380+ return not any (self ._whole_token (value ).search (device_id ) for value in self ._pseudonym_values (record ) if value )
381+
293382 def _misfiled (self , value , strict_numeric = False ):
294383 """Whether a value looks like an identifier rather than a measurement, a vendor code, or ordinary text.
295384
@@ -307,6 +396,10 @@ def _misfiled(self, value, strict_numeric=False):
307396 industry-standard shape: "HV2160123456") is that container's entire declared purpose, not
308397 a misfiling; a BARE all-digit string there is still genuinely suspicious, since an MPAN
309398 misfiled where a serial belongs looks exactly like one.
399+
400+ A serial some record declares is never a misfiling: it is removed (as a whole token - see
401+ _strip_serials) before any of the above is judged, so a declared all-digit serial and a
402+ device_id built from one stay readable while anything left beside them is still checked.
310403 """
311404 if isinstance (value , bool ):
312405 return False
@@ -315,8 +408,10 @@ def _misfiled(self, value, strict_numeric=False):
315408 integer_part = abs (int (value ))
316409 except (ValueError , OverflowError ):
317410 return False
411+ if not self ._strip_serials (str (integer_part )):
412+ return False
318413 return len (str (integer_part )) >= 10
319- text = str (value )
414+ text = self . _strip_serials ( str (value ) )
320415 if "@" in text :
321416 return True
322417 stripped = SEPARATOR_RE .sub ("" , text )
@@ -414,7 +509,7 @@ def _walk(self, node, container=None):
414509 CONTAINER_SPEC.
415510 """
416511 if isinstance (node , dict ):
417- if isinstance (node .get ("device_id" ), str ) and self ._has_pseudonym_container (node ):
512+ if isinstance (node .get ("device_id" ), str ) and self ._has_pseudonym_container (node ) and not self . _serial_derived ( node ) :
418513 self ._note (node ["device_id" ], substring = True )
419514 out = {}
420515 for key , value in node .items ():
@@ -527,6 +622,9 @@ def redact(self, catalogue):
527622 otherwise be free to corrupt it with.
528623 """
529624 generated = catalogue .get ("generated" )
625+ # Longest first, so a serial that contains a shorter one is removed whole
626+ serials = sorted (self ._collect_serials (catalogue ), key = len , reverse = True )
627+ self ._serial_patterns = [self ._whole_token (serial ) for serial in serials ]
530628 walked = self ._walk (catalogue )
531629 self ._substring_order = sorted (self .substring_ok , key = len , reverse = True )
532630 substituted = self ._substitute (walked )
@@ -916,8 +1014,9 @@ def inverter_record(
9161014 Tuples become lists because validate_report() keeps only a list for a structural or
9171015 vocabulary field: a tuple `serials` or `functions` - a module-level constant, say - would be
9181016 silently dropped from the catalogue. A set or frozenset becomes a sorted list for the same
919- reason, sorted because string hashing is randomised per process, so list(some_set) comes out
920- in a different order after a restart and two dumps of the same hardware would diff for nothing.
1017+ reason, sorted by string form so that a set mixing types cannot raise and withhold the whole
1018+ report, and sorted because string hashing is randomised per process, so list(some_set) comes
1019+ out in a different order after a restart and two dumps of the same hardware would diff for nothing.
9211020 """
9221021 fields = {
9231022 "inverter_type" : inverter_type ,
@@ -939,7 +1038,7 @@ def inverter_record(
9391038 record = {"device_id" : device_id }
9401039 for name , value in fields .items ():
9411040 if isinstance (value , (set , frozenset )):
942- value = sorted (value )
1041+ value = sorted (value , key = str )
9431042 elif isinstance (value , (list , tuple )):
9441043 value = list (value )
9451044 elif isinstance (value , dict ):
0 commit comments