"""Watchdog contracts for absolute deadlines on streaming Git exchanges.""" from __future__ import annotations import threading import time import urllib.request from http.server import BaseHTTPRequestHandler, HTTPServer import pytest from testing.tests.test_hermes_scm_broker_support import Response, _load def test_stream_deadline_validates_bounds_and_counts_down(): module = _load("deadline_http") for timeout in (0, -1, module.MAX_STREAM_SECONDS + 1): with pytest.raises(module.PolicyError, match="bounds are invalid"): module.StreamDeadline(timeout) guard = module.StreamDeadline(30) try: first = guard.remaining() assert 0 < first <= 30 finally: guard.cancel() class _FakeTimer: def __init__(self, timeout, callback): self.timeout = timeout self.callback = callback self.daemon = False self.started = False self.cancelled = False def start(self): self.started = True def cancel(self): self.cancelled = True class _FakeSocket: def __init__(self, fail: bool = False): self.fail = fail self.shut = False def shutdown(self, _how): if self.fail: raise OSError("already gone") self.shut = True class _FakeConnection: def __init__(self, sock=None): self.sock = sock self.closed = False def close(self): self.closed = True def test_expiry_shuts_down_and_closes_every_tracked_connection(): module = _load("deadline_http") guard = module.StreamDeadline(30, timer=_FakeTimer) assert guard._timer.started and guard._timer.daemon healthy = _FakeConnection(_FakeSocket()) broken = _FakeConnection(_FakeSocket(fail=True)) bare = _FakeConnection() for connection in (healthy, broken, bare): assert guard._track(connection) is connection guard._expire() assert guard.expired assert healthy.sock.shut and healthy.closed assert broken.closed and not broken.sock.shut assert bare.closed with pytest.raises(module.PolicyError, match="deadline exceeded"): guard.remaining() late = _FakeConnection() with pytest.raises(module.PolicyError, match="deadline exceeded"): guard._track(late) assert late.closed guard.cancel() assert guard._timer.cancelled def test_handlers_build_guarded_connections_without_network(monkeypatch): module = _load("deadline_http") guard = module.StreamDeadline(30, timer=_FakeTimer) http_handler, https_handler = guard.handlers() assert isinstance(http_handler, urllib.request.HTTPHandler) assert isinstance(https_handler, urllib.request.HTTPSHandler) for handler, opener_name in ( (http_handler, "http_open"), (https_handler, "https_open"), ): seen = {} monkeypatch.setattr( handler, "do_open", lambda factory, req, seen=seen: seen.update(factory=factory, req=req), ) getattr(handler, opener_name)("request") connection = seen["factory"]("127.0.0.1", timeout=1) assert connection in guard._connections connection.close() guard.cancel() def test_dns_resolution_is_bounded_by_the_stream_deadline(monkeypatch): module = _load("deadline_http") def hang(*_args, **_kwargs): time.sleep(30) monkeypatch.setattr(module.socket, "getaddrinfo", hang) started = time.monotonic() with pytest.raises(module.PolicyError, match="DNS resolution deadline"): module._resolve_within("example.invalid", 443, 0.4) assert time.monotonic() - started < 3 def test_resolve_within_returns_on_success_and_propagates_errors(monkeypatch): module = _load("deadline_http") calls = [] monkeypatch.setattr( module.socket, "getaddrinfo", lambda *a, **k: calls.append((a, k)) ) module._resolve_within("127.0.0.1", 80, 5) assert calls def boom(*_a, **_k): raise OSError("name resolution failed") monkeypatch.setattr(module.socket, "getaddrinfo", boom) with pytest.raises(OSError, match="name resolution failed"): module._resolve_within("127.0.0.1", 80, 5) def test_bounded_connection_wraps_connect_to_resolve_first(monkeypatch): module = _load("deadline_http") guard = module.StreamDeadline(30, timer=_FakeTimer) order = [] monkeypatch.setattr( module, "_resolve_within", lambda host, port, timeout: order.append(("resolve", host, port)), ) class FakeConnection: host = "scm.bstein.dev" port = 443 def __init__(self): self.sock = None def connect(self): order.append(("connect", self.host)) def close(self): order.append(("close",)) connection = guard._bounded_connection(FakeConnection()) assert connection in guard._connections connection.connect() assert order == [("resolve", "scm.bstein.dev", 443), ("connect", "scm.bstein.dev")] guard.cancel() def test_bounded_connection_connect_fails_closed_once_expired(monkeypatch): module = _load("deadline_http") guard = module.StreamDeadline(30, timer=_FakeTimer) resolved = [] monkeypatch.setattr( module, "_resolve_within", lambda *a, **k: resolved.append(a) ) class FakeConnection: host = "scm.bstein.dev" port = 443 sock = None def connect(self): resolved.append("connected") def close(self): pass connection = guard._bounded_connection(FakeConnection()) guard._expire() with pytest.raises(module.PolicyError, match="deadline exceeded"): connection.connect() assert resolved == [] guard.cancel() class _TrickleHandler(BaseHTTPRequestHandler): def do_GET(self): time.sleep(10) def log_message(self, *_args): return def test_watchdog_interrupts_a_hung_streaming_exchange(): module = _load("deadline_http") server = HTTPServer(("127.0.0.1", 0), _TrickleHandler) threading.Thread(target=server.serve_forever, daemon=True).start() guard = module.StreamDeadline(1.0) opener = urllib.request.build_opener(*guard.handlers()) started = time.monotonic() try: with pytest.raises(OSError): opener.open( f"http://127.0.0.1:{server.server_port}/slow", timeout=30 ).read() assert time.monotonic() - started < 5 assert guard.expired finally: guard.cancel() server.shutdown() def test_upstream_git_request_runs_under_one_wall_clock_budget(monkeypatch): broker = _load("scm_broker") seen = {} class RecordingGuard: def __init__(self, timeout, **_kwargs): seen["timeout"] = timeout self.cancelled = False seen["guard"] = self def remaining(self): return 12.5 def cancel(self): self.cancelled = True def handlers(self): return () monkeypatch.setattr(broker.deadline_http, "StreamDeadline", RecordingGuard) def opener(request, timeout): seen["opener_timeout"] = timeout return Response(b"result", content_type="application/x-git-upload-pack-result") result = broker._upstream_git_request( "/titan/cassandra.git/git-upload-pack", method="POST", body=b"request", content_type="application/x-git-upload-pack-request", expected_type="application/x-git-upload-pack-result", token="sentinel", opener=opener, ) assert result == b"result" assert seen["timeout"] == broker.UPSTREAM_DEADLINE_SECONDS assert seen["opener_timeout"] == 12.5 assert seen["guard"].cancelled def test_upstream_git_request_cancels_guard_when_deadline_already_passed(monkeypatch): broker = _load("scm_broker") cancelled = [] class ExpiredGuard: def __init__(self, _timeout, **_kwargs): pass def remaining(self): raise broker.PolicyError("HTTP stream deadline exceeded") def cancel(self): cancelled.append(True) def handlers(self): return () monkeypatch.setattr(broker.deadline_http, "StreamDeadline", ExpiredGuard) with pytest.raises(broker.PolicyError, match="deadline exceeded"): broker._upstream_git_request( "/titan/cassandra.git/git-upload-pack", method="POST", body=b"request", content_type="application/x-git-upload-pack-request", expected_type="application/x-git-upload-pack-result", token="sentinel", opener=lambda *_a, **_k: None, ) assert cancelled == [True] def test_guarded_opener_wires_redirect_rejection_and_deadline_handlers(): broker = _load("scm_broker") module = _load("deadline_http") guard = module.StreamDeadline(30, timer=_FakeTimer) opener = broker._guarded_opener(guard) assert callable(opener) handlers = opener.__self__.handlers assert any(isinstance(item, broker.RejectRedirect) for item in handlers) assert any(isinstance(item, urllib.request.HTTPSHandler) for item in handlers) guard.cancel()