Skip to content

Commit 74c29a2

Browse files
author
Maksym Shtembuliak
committed
fix(support): preserve binary reply evidence in exported archives
1 parent 56c5e41 commit 74c29a2

7 files changed

Lines changed: 269 additions & 6 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,12 @@ the GitHub release body should be rendered from the matching version section her
2828

2929
### Fixed
3030

31+
- Support archives preserve numeric-looking hex sequences in explicitly typed
32+
wire-evidence fields instead of replacing their bytes with identifier masks.
33+
Embedded ASCII identifiers remain masked. Short-ASCII archive creation and
34+
authenticated download are covered for successful, partial, malformed and
35+
offline reads through real Home Assistant.
36+
3137
- Retired collector readers can no longer deliver buffered replies, identity
3238
observations or read errors to a replacement connection during slow socket
3339
shutdown. Auxiliary read results also retain their original session owner.

‎custom_components/eybond_local/support/masking.py‎

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,15 @@
1818
_NUMERIC_IDENTIFIER_RE = re.compile(r"(?<!\d)(\d{10,})(?!\d)")
1919
_NUMERIC_IDENTIFIER_BYTES_RE = re.compile(rb"(?<!\d)(\d{10,})(?!\d)")
2020

21+
# These existing evidence fields explicitly contain encoded wire bytes, not
22+
# decimal serials. Keep this allowlist narrow: an arbitrary *_hex name is not
23+
# sufficient to bypass the conservative text policy.
24+
_WIRE_HEX_FIELDS = frozenset({
25+
"response_hex", "raw_response_hex", "payload_hex", "raw_payload_hex",
26+
"frame_hex", "chunk_hex", "remaining_hex", "buffer_hex", "raw_hex",
27+
"command_hex",
28+
})
29+
2130

2231
def mask_identifier_token(token: str) -> str:
2332
if len(token) <= 4:
@@ -31,16 +40,33 @@ def mask_numeric_identifiers(value):
3140
if isinstance(value, dict):
3241
# Keys are masked too: a PN/serial used as a mapping key would
3342
# otherwise leave the artifact unmasked.
34-
return {
35-
(mask_numeric_identifiers(key) if isinstance(key, str) else key): (
36-
mask_numeric_identifiers(item)
37-
)
38-
for key, item in value.items()
39-
}
43+
masked = {}
44+
for key, item in value.items():
45+
masked_key = mask_numeric_identifiers(key) if isinstance(key, str) else key
46+
if key in _WIRE_HEX_FIELDS and isinstance(item, str):
47+
masked[masked_key] = _mask_string(item, wire_hex=True)
48+
elif key == "responses_hex" and isinstance(item, dict):
49+
# A command -> hex reply map. Context applies to its scalar
50+
# replies only, never to keys or unexpected nested metadata.
51+
masked[masked_key] = {
52+
mask_numeric_identifiers(command): (
53+
_mask_string(reply, wire_hex=True) if isinstance(reply, str)
54+
else mask_numeric_identifiers(reply)
55+
)
56+
for command, reply in item.items()
57+
}
58+
else:
59+
masked[masked_key] = mask_numeric_identifiers(item)
60+
return masked
4061
if isinstance(value, list):
4162
return [mask_numeric_identifiers(item) for item in value]
4263
if not isinstance(value, str):
4364
return value
65+
return _mask_string(value)
66+
67+
68+
def _mask_string(value: str, *, wire_hex: bool = False) -> str:
69+
"""Mask decoded ASCII identifiers, not the digits encoding binary data."""
4470

4571
# Hex blobs FIRST: an ASCII-encoded identifier inside hex is itself a
4672
# long run of decimal characters ("E50..." becomes "4535303030..."), so
@@ -66,6 +92,10 @@ def mask_numeric_identifiers(value):
6692
)
6793
if redacted != raw:
6894
return redacted.hex()
95+
if wire_hex:
96+
# Bytes such as 00 01 02... become a long decimal-looking
97+
# string when hex-encoded. They are not a textual identifier.
98+
return value
6999
# No embedded ASCII identifier — fall through to text masking:
70100
# a bare decimal serial ("92632500000001") is also valid hex,
71101
# and returning it untouched would leak it.

‎docs/user/SUPPORT_ARCHIVE.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,14 @@ Depending on what is available for this entry, one archive can include:
6565
- an anonymized replay fixture when one can be built safely;
6666
- device-learning or cloud evidence that already belongs to this entry.
6767

68+
For the **EyeBond Short-ASCII family** in the test code, creating an archive
69+
also requests the basic `MP/Q1/MD` replies and optional `F/RB` replies. It keeps
70+
the returned bytes, including malformed responses, and records failed requests.
71+
It does not enable PV polling or inverter controls. This is a new diagnostic
72+
snapshot, not a continuous recording: if a reading jumps, also report when it
73+
happened and which sensor was affected. A later archive may no longer contain
74+
the response that caused that jump.
75+
6876
The archive helps answer questions such as:
6977

7078
- Which collector and inverter path was detected?

‎tests/test_support_masking.py‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
"""Binary evidence remains replayable without weakening identifier masking."""
2+
3+
from pathlib import Path
4+
import sys
5+
import unittest
6+
7+
ROOT = Path(__file__).resolve().parents[1]
8+
if str(ROOT) not in sys.path:
9+
sys.path.insert(0, str(ROOT))
10+
11+
from custom_components.eybond_local.support.masking import mask_numeric_identifiers
12+
13+
14+
class SupportMaskingTests(unittest.TestCase):
15+
def test_declared_binary_response_map_preserves_numeric_hex_runs(self):
16+
for raw in (bytes(range(36)), bytes(40), b"230.0 04 03 115.0\x00\r"):
17+
payload = {"responses_hex": {"MP": raw.hex()}}
18+
self.assertEqual(mask_numeric_identifiers(payload), payload)
19+
20+
def test_declared_wire_fields_preserve_numeric_hex_runs(self):
21+
for key in ("response_hex", "raw_response_hex", "payload_hex", "raw_payload_hex",
22+
"frame_hex", "chunk_hex", "remaining_hex", "buffer_hex", "raw_hex",
23+
"command_hex"):
24+
with self.subTest(key=key):
25+
payload = {key: bytes(range(36)).hex()}
26+
self.assertEqual(mask_numeric_identifiers(payload), payload)
27+
28+
def test_embedded_identifiers_still_masked_inside_declared_hex(self):
29+
identifier = "92632500000001"
30+
wire = ("AT+DTUPN:" + identifier).encode().hex()
31+
masked_wire = ("AT+DTUPN:9263******0001").encode().hex()
32+
payload = {"responses_hex": {"MP": wire}, "payload_hex": wire, "note": identifier}
33+
self.assertEqual(mask_numeric_identifiers(payload), {
34+
"responses_hex": {"MP": masked_wire}, "payload_hex": masked_wire,
35+
"note": "9263******0001",
36+
})
37+
self.assertEqual(mask_numeric_identifiers(mask_numeric_identifiers(payload)),
38+
mask_numeric_identifiers(payload))
39+
40+
def test_untyped_decimal_identifiers_and_mapping_keys_remain_masked(self):
41+
identifier = "92632500000001"
42+
payload = {identifier: identifier, "serial": identifier, "arbitrary_hex": identifier,
43+
"responses_hex": {identifier: bytes(10).hex()}}
44+
masked = mask_numeric_identifiers(payload)
45+
self.assertNotIn(identifier, repr(masked))
46+
self.assertEqual(masked["responses_hex"], {"9263******0001": bytes(10).hex()})
47+
self.assertEqual(mask_numeric_identifiers(identifier), "9263******0001")
48+
49+
def test_malformed_hex_does_not_bypass_text_masking(self):
50+
for value in ("92632500000001!", "12345678901", "PN=92632500000001"):
51+
self.assertIn("*", mask_numeric_identifiers({"payload_hex": value})["payload_hex"])
52+
53+
def test_hex_map_context_does_not_spread_to_nested_metadata(self):
54+
payload = {"responses_hex": {"unexpected": {"serial": "92632500000001"}}}
55+
self.assertEqual(mask_numeric_identifiers(payload), {
56+
"responses_hex": {"unexpected": {"serial": "9263******0001"}},
57+
})
58+
59+
def test_wire_whitespace_and_case_preserved_without_identifiers(self):
60+
payload = {"payload_hex": "00 01 02 03 04 05 06 07 08 09 AA BB"}
61+
self.assertEqual(mask_numeric_identifiers(payload), payload)
62+
63+
64+
if __name__ == "__main__":
65+
unittest.main()

‎tests/test_validation_workflow.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,14 @@
1414

1515

1616
class AffectedValidationSelectionTests(unittest.TestCase):
17+
def test_support_masking_selects_archive_and_wire_export_regressions(self) -> None:
18+
selected = self._selected("custom_components/eybond_local/support/masking.py")
19+
self.assertTrue({
20+
"test_support_masking.py", "test_support_package.py", "test_support_bundle.py",
21+
"test_proxy_trace.py", "test_diagnostic_export.py",
22+
"test_shadow_learning_support_package.py",
23+
}.issubset(selected))
24+
1725
def test_mppt_decoder_tool_and_wire_codec_select_offline_semantics_tests(self) -> None:
1826
for path in (
1927
"custom_components/eybond_local/payload/short_ascii_mppt.py",
Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
"""Real HA archive/download path with the real short-ASCII evidence collector."""
2+
3+
from __future__ import annotations
4+
5+
from io import BytesIO
6+
import json
7+
from pathlib import Path
8+
import zipfile
9+
10+
import pytest
11+
from pytest_homeassistant_custom_component.common import MockConfigEntry
12+
13+
from custom_components.eybond_local.const import DOMAIN
14+
from custom_components.eybond_local.drivers.eybond_short_ascii import EybondShortAsciiDriver
15+
from custom_components.eybond_local.link_models import EybondLinkRoute
16+
from custom_components.eybond_local.models import CollectorInfo, ProbeTarget, RuntimeSnapshot
17+
from custom_components.eybond_local.payload.short_ascii import READ_COMMANDS, parse_q1, parse_rb
18+
from synthetic import SYNTHETIC_COLLECTOR_IP, SYNTHETIC_COLLECTOR_PN, SYNTHETIC_SERVER_IP
19+
20+
21+
@pytest.mark.parametrize("outcome", ["ok", "partial", "invalid", "offline"])
22+
async def test_support_archive_preserves_read_evidence_through_ha_download(
23+
hass, hass_client_no_auth, fake_runtime, monkeypatch, outcome,
24+
) -> None:
25+
from conftest import FakeRuntimeManager
26+
27+
body = b"230.0 04 03 115.0 013 60.0 13.2 35.0 1000010" + bytes(3)
28+
rb = bytearray(37)
29+
rb[:2] = (520).to_bytes(2, "big")
30+
rb[2] = 80
31+
responses = {
32+
"MP": b"\x01" + bytes(range(36)) + b"\r",
33+
"Q1": b"\x01" + body + sum(body).to_bytes(2, "big") + b"\r",
34+
"MD": b"S2-8127-260101-V7.00 \x00\r",
35+
"F": b"#115.0 105 48.00 60.0\r",
36+
"RB": b"\x01" + rb + bytes([sum(rb) & 255]) + b"\r",
37+
}
38+
# Deliberately malformed wire evidence must survive export too. Do not
39+
# require successful semantic decoding just to retain a diagnostic sample.
40+
if outcome == "invalid":
41+
responses["RB"] = responses["RB"][:-2] + b"\xff\r"
42+
43+
class Transport:
44+
connected = True
45+
capturing = False
46+
capture_commands = []
47+
48+
async def async_send_payload(self, payload, *, route, request_timeout=None):
49+
assert type(route) is EybondLinkRoute
50+
assert (route.devcode, route.collector_addr) == (767, 255)
51+
command = payload[:-2].decode("ascii")
52+
assert command in READ_COMMANDS and payload[-2:] == b"\x01\r"
53+
if self.capturing:
54+
self.capture_commands.append(command)
55+
if outcome == "offline" or (outcome == "partial" and command == "F"):
56+
raise ConnectionError("synthetic_connection_lost")
57+
if outcome == "partial" and command == "RB":
58+
raise TimeoutError("synthetic_read_timeout")
59+
return bytes(responses[command])
60+
61+
driver, transport = EybondShortAsciiDriver(), Transport()
62+
inverter = await driver.async_probe(transport, ProbeTarget(767, 255, 1))
63+
assert inverter is not None
64+
65+
async def refresh(self, *, poll_interval=None):
66+
return RuntimeSnapshot(
67+
connected=outcome != "offline",
68+
collector=CollectorInfo(
69+
remote_ip=SYNTHETIC_COLLECTOR_IP, collector_pn=SYNTHETIC_COLLECTOR_PN,
70+
),
71+
inverter=inverter,
72+
values=parse_q1(responses["Q1"]) if outcome != "offline" else {},
73+
)
74+
75+
async def capture(self):
76+
transport.capturing = True
77+
try:
78+
return await driver.async_capture_support_evidence(transport, inverter)
79+
finally:
80+
transport.capturing = False
81+
82+
monkeypatch.setattr(FakeRuntimeManager, "async_refresh", refresh)
83+
monkeypatch.setattr(FakeRuntimeManager, "async_capture_support_evidence", capture)
84+
entry = MockConfigEntry(
85+
domain=DOMAIN, version=3, title="Short-ASCII support test",
86+
unique_id=f"collector:{SYNTHETIC_COLLECTOR_PN}",
87+
data={
88+
"connection_type": "eybond", "connection_mode": "known_ip",
89+
"server_ip": SYNTHETIC_SERVER_IP, "collector_ip": SYNTHETIC_COLLECTOR_IP,
90+
"collector_pn": SYNTHETIC_COLLECTOR_PN,
91+
"collector_operation_mode": "home_assistant_only",
92+
"tcp_port": 8899, "udp_port": 58899, "driver_hint": "auto",
93+
"control_mode": "read_only", "connection_strategy": "callback_on_demand",
94+
"endpoint_control_policy": "external", "proxy_enabled": False,
95+
"detected_driver": driver.key, "detected_model": inverter.model_name,
96+
"detected_serial": "", "detection_confidence": "high",
97+
},
98+
options={"poll_interval": 30, "poll_mode": "auto"},
99+
)
100+
entry.add_to_hass(hass)
101+
assert await hass.config_entries.async_setup(entry.entry_id)
102+
await hass.async_block_till_done()
103+
await entry.runtime_data.async_refresh()
104+
await hass.async_block_till_done()
105+
coordinator = entry.runtime_data
106+
path = Path(await coordinator.async_export_support_package())
107+
assert path.is_relative_to(Path(hass.config.config_dir) / "eybond_local/support_packages")
108+
assert not (Path(hass.config.config_dir) / "www/eybond_local/support_packages").exists()
109+
url = coordinator.data.values["support_package_download_url"]
110+
assert "authSig=" in url
111+
# A frontend link opens without an Authorization header; its signed URL
112+
# must work on its own, while the same unsigned private route must not.
113+
client = await hass_client_no_auth()
114+
denied = await client.get(url.split("?", 1)[0])
115+
assert denied.status == 401
116+
response = await client.get(url)
117+
assert response.status == 200
118+
downloaded = await response.read()
119+
assert downloaded == await hass.async_add_executor_job(path.read_bytes)
120+
121+
with zipfile.ZipFile(BytesIO(downloaded)) as archive:
122+
assert archive.testzip() is None
123+
manifest = json.loads(archive.read("manifest.json"))
124+
bundle = json.loads(archive.read("support_bundle.json"))
125+
raw = json.loads(archive.read("raw_capture.json"))
126+
assert manifest["sharing_guidance"]["recommended_artifact"] == path.name
127+
assert SYNTHETIC_COLLECTOR_PN not in archive.read("support_bundle.json").decode()
128+
assert bundle["runtime"]["connected"] is (outcome != "offline")
129+
assert raw["capture_kind"] == "short_ascii_read_only"
130+
assert transport.capture_commands == list(READ_COMMANDS)
131+
failed = set(READ_COMMANDS) if outcome == "offline" else {"F", "RB"} if outcome == "partial" else set()
132+
assert set(raw["failures"]) == failed
133+
assert set(raw["responses_hex"]) == set(READ_COMMANDS) - failed
134+
for command, payload in raw["responses_hex"].items():
135+
assert bytes.fromhex(payload) == responses[command], command
136+
if outcome == "partial":
137+
assert raw["failures"] == {"F": "ConnectionError", "RB": "TimeoutError"}
138+
if outcome == "ok":
139+
assert parse_rb(bytes.fromhex(raw["responses_hex"]["RB"]))["battery_soc"] == 80
140+
assert await hass.config_entries.async_unload(entry.entry_id)
141+
await hass.async_block_till_done()

‎tools/validate.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,11 @@
133133
# these entries prevent a cheap ``affected`` run from silently missing a typed
134134
# boundary or neutral wire contract.
135135
_EXACT_TESTS: dict[str, tuple[str, ...]] = {
136+
"custom_components/eybond_local/support/masking.py": (
137+
"test_support_masking.py", "test_support_package.py", "test_support_bundle.py",
138+
"test_proxy_trace.py", "test_diagnostic_export.py",
139+
"test_shadow_learning_support_package.py",
140+
),
136141
"custom_components/eybond_local/payload/short_ascii_mppt.py": (
137142
"test_short_ascii_mppt.py",
138143
),

0 commit comments

Comments
 (0)