Skip to content

Commit 92bc1be

Browse files
committed
fix: reuse service test connections to avoid Windows port exhaustion
The Windows service acceptance job opened a new TCP connection for every HTTP request. Sustained traffic exhausted ephemeral ports during rolling restart checks and failed with socket error 10048. Have check_services own one HTTPConnection per worker and reuse it for the initial reachability probe and concurrent traffic. Close all connections through ExitStack after stopping the traffic thread, including when a probe or audit fails. Preserve response validation, traffic failure reporting, and all runtime identity and restart assertions. Add loopback HTTP regressions proving that repeated requests share one connection, successful audits release it, and invalid responses still fail before inspection and release the connection. The reuse regression failed before the fix with 12 connections for 12 requests. Validation: - All 80 unittest regressions passed. - Core Ruff checks (E9,F63,F7,F82) passed. - Full darwin-arm64 service acceptance passed for mixed versions, stale workers after disk replacement, both rolling restarts, and a missing expected worker. - git diff --check passed.
1 parent be8e539 commit 92bc1be

2 files changed

Lines changed: 104 additions & 24 deletions

File tree

‎test_verify_service.py‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
import http.server
2+
import io
3+
import json
4+
import tempfile
5+
import threading
6+
import unittest
7+
from contextlib import redirect_stdout
8+
from pathlib import Path
9+
from types import SimpleNamespace
10+
from unittest.mock import patch
11+
12+
import verify_service
13+
14+
15+
class Handler(http.server.BaseHTTPRequestHandler):
16+
protocol_version = 'HTTP/1.1'
17+
18+
def do_POST(self):
19+
self.rfile.read(int(self.headers['Content-Length']))
20+
self.server.clients.add(self.client_address)
21+
self.server.requests += 1
22+
body = json.dumps({'pid': 123, 'text': self.server.text}).encode()
23+
self.send_response(200)
24+
self.send_header('Content-Length', str(len(body)))
25+
self.end_headers()
26+
self.wfile.write(body)
27+
self.wfile.flush()
28+
if self.server.requests >= 12:
29+
self.server.traffic_ready.set()
30+
31+
def finish(self):
32+
super().finish()
33+
self.server.connection_closed.set()
34+
35+
def log_message(self, *args):
36+
pass
37+
38+
39+
class ServiceTrafficTests(unittest.TestCase):
40+
def setUp(self):
41+
self.server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), Handler)
42+
self.addCleanup(self.server.server_close)
43+
self.server.clients = set()
44+
self.server.requests = 0
45+
self.server.text = 'node2shell benign service acceptance'
46+
self.server.traffic_ready = threading.Event()
47+
self.server.connection_closed = threading.Event()
48+
thread = threading.Thread(target=self.server.serve_forever, daemon=True)
49+
thread.start()
50+
self.addCleanup(thread.join, 5)
51+
self.addCleanup(self.server.shutdown)
52+
directory = tempfile.TemporaryDirectory()
53+
self.addCleanup(directory.cleanup)
54+
self.root = Path(directory.name)
55+
self.service = {'pid': 123, 'port': self.server.server_port,
56+
'payload': b'fixture', 'binary': str(self.root / 'node')}
57+
58+
def test_traffic_reuses_connection_and_closes_it(self):
59+
def audit_run(*args, **kwargs):
60+
self.assertTrue(self.server.traffic_ready.wait(5), 'traffic stopped early')
61+
report = {'exit_status': 0, 'verdict': 'UNKNOWN', 'runtimes': [],
62+
'issues': ['Audit mocked for traffic test']}
63+
return SimpleNamespace(returncode=0, stdout=json.dumps({'reports': [report]}))
64+
65+
with patch.object(verify_service.subprocess, 'run', side_effect=audit_run), redirect_stdout(io.StringIO()):
66+
verify_service.check_services(self.root, [self.service], {}, {}, 'Connection reuse')
67+
self.assertEqual(len(self.server.clients), 1, 'requests opened fresh TCP connections')
68+
self.assertTrue(self.server.connection_closed.wait(5), 'connection leaked after audit')
69+
70+
def test_invalid_response_fails_and_closes_connection(self):
71+
self.server.text = 'unexpected response'
72+
with patch.object(verify_service.subprocess, 'run') as audit_run, self.assertRaises(AssertionError):
73+
verify_service.check_services(self.root, [self.service], {}, {}, 'Invalid response')
74+
audit_run.assert_not_called()
75+
self.assertTrue(self.server.connection_closed.wait(5), 'connection leaked after failure')
76+
77+
78+
if __name__ == '__main__':
79+
unittest.main()

‎verify_service.py‎

Lines changed: 25 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -84,17 +84,13 @@ def start_service(binary, script, stack):
8484
return service
8585

8686

87-
def request(service):
88-
connection = http.client.HTTPConnection('127.0.0.1', service['port'], timeout=5)
89-
try:
90-
connection.request('POST', '/decode', body=service['payload'],
91-
headers={'Content-Encoding': 'br', 'Content-Type': 'application/octet-stream'})
92-
response = connection.getresponse()
93-
data = json.loads(response.read(2048))
94-
assert response.status == 200 and data == {
95-
'pid': service['pid'], 'text': 'node2shell benign service acceptance'}, data
96-
finally:
97-
connection.close()
87+
def request(service, connection):
88+
connection.request('POST', '/decode', body=service['payload'],
89+
headers={'Content-Encoding': 'br', 'Content-Type': 'application/octet-stream'})
90+
response = connection.getresponse()
91+
data = json.loads(response.read(2048))
92+
assert response.status == 200 and data == {
93+
'pid': service['pid'], 'text': 'node2shell benign service acceptance'}, data
9894

9995

10096
def check_services(root, services, proof, statuses, phase, stale=(), missing=False):
@@ -120,24 +116,29 @@ def traffic():
120116
deadline = time.monotonic() + 120
121117
while not stop.is_set() and time.monotonic() < deadline:
122118
for service in services:
123-
request(service)
119+
request(service, connections[service['pid']])
124120
counts[service['pid']] += 1
125121
stop.wait(0.01)
126122
except Exception as exc:
127123
failures.append(exc)
128124

129-
# Establish reachability before starting concurrent traffic and inspection.
130-
for service in services:
131-
request(service)
132-
worker = threading.Thread(target=traffic)
133-
worker.start()
134-
try:
135-
run = subprocess.run([sys.executable, str(Path(audit.__file__).resolve()),
136-
'--inventory', str(inv), '--evidence', str(evidence), '--json'],
137-
capture_output=True, text=True, timeout=120)
138-
finally:
139-
stop.set()
140-
worker.join(timeout=25)
125+
with ExitStack() as stack:
126+
# Reuse sockets so sustained traffic cannot exhaust Windows ephemeral ports.
127+
connections = {}
128+
for service in services:
129+
connection = http.client.HTTPConnection('127.0.0.1', service['port'], timeout=5)
130+
stack.callback(connection.close)
131+
connections[service['pid']] = connection
132+
request(service, connection)
133+
worker = threading.Thread(target=traffic)
134+
worker.start()
135+
try:
136+
run = subprocess.run([sys.executable, str(Path(audit.__file__).resolve()),
137+
'--inventory', str(inv), '--evidence', str(evidence), '--json'],
138+
capture_output=True, text=True, timeout=120)
139+
finally:
140+
stop.set()
141+
worker.join(timeout=25)
141142
assert not worker.is_alive() and not failures, failures
142143
assert all(counts.values()), counts
143144
report = json.loads(run.stdout)['reports'][0]

0 commit comments

Comments
 (0)