340ec58b68
Inflate every receive-pack object under strict pack, size, and checksum bounds, resolve deltas against in-pack bases only, and scan the real decompressed payloads for runtime-token forms, private keys, SSH key material, and known provider token formats. Thin packs are rejected so no pushed content escapes the scan, and upstream Git exchanges now run under one absolute stream deadline. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
456 lines
16 KiB
Python
456 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""Serve the credential-isolated Atlas API and Git smart-HTTP boundary."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import json
|
|
import re
|
|
import socket
|
|
import tempfile
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from http.server import BaseHTTPRequestHandler
|
|
from typing import BinaryIO
|
|
|
|
import deadline_http
|
|
from gitea_api import (
|
|
CANONICAL_BASE_URL,
|
|
PolicyError,
|
|
create_draft,
|
|
read,
|
|
read_token,
|
|
)
|
|
from gitea_api_policy import _reject_forbidden, _validate_repo
|
|
from receive_pack_scan import validate_receive_pack
|
|
from scm_broker_io import RejectRedirect, response_status as _status, spool_response
|
|
from scm_broker_server import AbsoluteHeaderDeadlineMixin, BoundedThreadingHTTPServer
|
|
|
|
BROKER_PORT = 9081
|
|
GIT_USER = "hermes-automation"
|
|
MAX_CONTROL_BODY = 64 * 1024
|
|
MAX_GIT_REQUEST = 128 * 1024 * 1024
|
|
MAX_GIT_RESPONSE = 128 * 1024 * 1024
|
|
MAX_HEADERS = 32
|
|
MAX_HEADER_BYTES = 16 * 1024
|
|
SPOOL_MEMORY_LIMIT = 1024 * 1024
|
|
STREAM_CHUNK = 64 * 1024
|
|
INBOUND_HEADER_TIMEOUT = 10.0
|
|
INBOUND_BODY_TIMEOUT = 120.0
|
|
UPSTREAM_DEADLINE_SECONDS = 300.0
|
|
GIT_PATH_RE = re.compile(
|
|
r"/git/atlas/(?P<repo>[A-Za-z0-9][A-Za-z0-9._-]{0,99})\.git/"
|
|
r"(?P<operation>info/refs|git-upload-pack|git-receive-pack)\Z"
|
|
)
|
|
|
|
|
|
def _read_bounded(
|
|
stream,
|
|
maximum: int,
|
|
length: int | None = None,
|
|
*,
|
|
deadline: float | None = None,
|
|
set_timeout=None,
|
|
) -> bytes:
|
|
if length is not None and not 0 <= length <= maximum:
|
|
raise PolicyError("SCM request exceeds the safe size limit")
|
|
wanted = maximum + 1 if length is None else length
|
|
chunks: list[bytes] = []
|
|
total = 0
|
|
while total < wanted:
|
|
if deadline is not None:
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise PolicyError("SCM request body deadline exceeded")
|
|
if set_timeout is not None:
|
|
set_timeout(remaining)
|
|
chunk = stream.read(min(STREAM_CHUNK, wanted - total))
|
|
if not chunk:
|
|
break
|
|
chunks.append(chunk)
|
|
total += len(chunk)
|
|
value = b"".join(chunks)
|
|
if len(value) > maximum or (length is not None and len(value) != length):
|
|
raise PolicyError("SCM request exceeds the safe size limit")
|
|
return value
|
|
|
|
|
|
def _spool_bounded(
|
|
stream,
|
|
maximum: int,
|
|
length: int,
|
|
*,
|
|
token: str,
|
|
context: str,
|
|
deadline: float,
|
|
set_timeout=None,
|
|
) -> tuple[BinaryIO, int]:
|
|
"""Copy a fixed-length exchange through a bounded-memory disk spool."""
|
|
if not 0 <= length <= maximum:
|
|
raise PolicyError("SCM request exceeds the safe size limit")
|
|
spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool
|
|
max_size=SPOOL_MEMORY_LIMIT, dir="/tmp"
|
|
)
|
|
remaining = length
|
|
forms = _credential_forms(token)
|
|
carry = b""
|
|
try:
|
|
while remaining:
|
|
remaining_time = deadline - time.monotonic()
|
|
if remaining_time <= 0:
|
|
raise PolicyError("SCM request body deadline exceeded")
|
|
if set_timeout is not None:
|
|
set_timeout(remaining_time)
|
|
chunk = stream.read(min(STREAM_CHUNK, remaining))
|
|
if not chunk:
|
|
raise PolicyError("SCM request body ended early")
|
|
candidate = carry + chunk
|
|
if any(form in candidate for form in forms):
|
|
raise PolicyError(f"{context} contains runtime credential material")
|
|
width = max(len(form) for form in forms) - 1
|
|
carry = candidate[-width:] if width else b""
|
|
spool.write(chunk)
|
|
remaining -= len(chunk)
|
|
spool.seek(0)
|
|
return spool, length
|
|
except Exception:
|
|
spool.close()
|
|
raise
|
|
|
|
|
|
def _spool_response(
|
|
response, maximum: int, token: str, deadline_seconds: float
|
|
) -> tuple[BinaryIO, int]:
|
|
return spool_response(
|
|
response,
|
|
maximum,
|
|
_credential_forms(token),
|
|
memory_limit=SPOOL_MEMORY_LIMIT,
|
|
chunk_size=STREAM_CHUNK,
|
|
deadline_seconds=deadline_seconds,
|
|
)
|
|
|
|
|
|
def _load_json(handler: BaseHTTPRequestHandler) -> dict[str, object]:
|
|
if handler.headers.get("Transfer-Encoding"):
|
|
raise PolicyError("chunked broker control requests are not allowed")
|
|
if handler.headers.get_content_type() != "application/json":
|
|
raise PolicyError("broker control request must be JSON")
|
|
length = _content_length(handler.headers, MAX_CONTROL_BODY)
|
|
deadline = time.monotonic() + INBOUND_BODY_TIMEOUT
|
|
value = json.loads(
|
|
_read_bounded(
|
|
handler.rfile,
|
|
MAX_CONTROL_BODY,
|
|
length,
|
|
deadline=deadline,
|
|
set_timeout=handler.connection.settimeout,
|
|
)
|
|
)
|
|
if not isinstance(value, dict):
|
|
raise PolicyError("broker control request must be an object")
|
|
return value
|
|
|
|
|
|
def _git_target(raw: str) -> tuple[str, str, str]:
|
|
if (
|
|
not raw.isascii()
|
|
or any(ord(character) < 32 or ord(character) == 127 for character in raw)
|
|
or "%" in raw
|
|
or "\\" in raw
|
|
or len(raw) > 512
|
|
):
|
|
raise PolicyError("Git target is not canonical ASCII")
|
|
target = urllib.parse.urlsplit(raw)
|
|
canonical = urllib.parse.urlunsplit(("", "", target.path, target.query, ""))
|
|
if canonical != raw:
|
|
raise PolicyError("Git target is not in exact canonical form")
|
|
if target.scheme or target.netloc or target.fragment:
|
|
raise PolicyError("Git target must be relative")
|
|
match = GIT_PATH_RE.fullmatch(target.path)
|
|
if not match:
|
|
raise PolicyError("Git target is outside the Atlas allowlist")
|
|
repo = _validate_repo(match.group("repo"))
|
|
operation = match.group("operation")
|
|
query = target.query
|
|
if operation == "info/refs":
|
|
if query not in {"service=git-upload-pack", "service=git-receive-pack"}:
|
|
raise PolicyError("Git discovery service is outside the allowlist")
|
|
service = query.removeprefix("service=")
|
|
elif query:
|
|
raise PolicyError("Git RPC query parameters are not allowed")
|
|
else:
|
|
service = operation
|
|
return repo, operation, service
|
|
|
|
|
|
def _content_length(headers: object, maximum: int) -> int:
|
|
"""Parse one short canonical bounded Content-Length header."""
|
|
raw = headers.get("Content-Length", "") # type: ignore[attr-defined]
|
|
if (
|
|
not isinstance(raw, str)
|
|
or not raw.isascii()
|
|
or len(raw) > 10
|
|
or not raw.isdigit()
|
|
):
|
|
raise PolicyError("SCM request length is invalid")
|
|
length = int(raw)
|
|
if raw != str(length) or not 0 <= length <= maximum:
|
|
raise PolicyError("SCM request length is invalid")
|
|
return length
|
|
|
|
|
|
def _credential_forms(token: str) -> tuple[bytes, ...]:
|
|
basic = base64.b64encode(f"{GIT_USER}:{token}".encode())
|
|
return token.encode("utf-8"), basic, b"Basic " + basic
|
|
|
|
|
|
def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None:
|
|
"""Permit only vetted new-feature-branch pushes with scanned contents."""
|
|
validate_receive_pack(body, token, _credential_forms(token))
|
|
|
|
|
|
def _guarded_opener(guard: deadline_http.StreamDeadline):
|
|
"""Build an opener whose connection obeys one absolute stream deadline."""
|
|
return urllib.request.build_opener(RejectRedirect(), *guard.handlers()).open
|
|
|
|
|
|
def _upstream_git_request(
|
|
target: str,
|
|
*,
|
|
method: str,
|
|
body: bytes | BinaryIO | None,
|
|
content_type: str | None,
|
|
expected_type: str,
|
|
token: str,
|
|
body_length: int | None = None,
|
|
stream_result: bool = False,
|
|
opener=None,
|
|
) -> bytes | tuple[BinaryIO, int]:
|
|
credentials = base64.b64encode(f"{GIT_USER}:{token}".encode()).decode("ascii")
|
|
headers = {"Accept": expected_type, "Authorization": f"Basic {credentials}"}
|
|
if content_type:
|
|
headers["Content-Type"] = content_type
|
|
if body is not None:
|
|
if body_length is None:
|
|
if not isinstance(body, bytes):
|
|
raise PolicyError("Git upstream request length is missing")
|
|
body_length = len(body)
|
|
headers["Content-Length"] = str(body_length)
|
|
request = urllib.request.Request(
|
|
CANONICAL_BASE_URL + target,
|
|
data=body,
|
|
method=method,
|
|
headers=headers,
|
|
)
|
|
# One wall-clock budget covers connect, send, headers, and body read; the
|
|
# watchdog closes the upstream socket if any phase overstays it.
|
|
guard = deadline_http.StreamDeadline(UPSTREAM_DEADLINE_SECONDS)
|
|
try:
|
|
open_upstream = opener if opener is not None else _guarded_opener(guard)
|
|
with open_upstream(request, timeout=guard.remaining()) as response:
|
|
if _status(response) != 200:
|
|
raise PolicyError("Git upstream returned an unexpected HTTP status")
|
|
if response.headers.get_content_type() != expected_type:
|
|
raise PolicyError("Git upstream returned an unexpected response type")
|
|
result, result_length = _spool_response(
|
|
response, MAX_GIT_RESPONSE, token, guard.remaining()
|
|
)
|
|
finally:
|
|
guard.cancel()
|
|
if stream_result:
|
|
return result, result_length
|
|
try:
|
|
return result.read()
|
|
finally:
|
|
result.close()
|
|
|
|
|
|
class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
|
|
"""Expose only bounded metadata, draft creation, and smart-HTTP Git."""
|
|
|
|
server_version = "HermesSCMBroker/1"
|
|
sys_version = ""
|
|
header_deadline_seconds = INBOUND_HEADER_TIMEOUT
|
|
|
|
def _validate_headers(self) -> None:
|
|
items = list(self.headers.items())
|
|
if len(items) > MAX_HEADERS:
|
|
raise PolicyError("SCM request has too many headers")
|
|
total = 0
|
|
for name, value in items:
|
|
if not name.isascii() or not value.isascii():
|
|
raise PolicyError("SCM request headers must be ASCII")
|
|
total += len(name) + len(value) + 4
|
|
if any(ord(character) < 32 and character != "\t" for character in value):
|
|
raise PolicyError("SCM request header contains controls")
|
|
if total > MAX_HEADER_BYTES:
|
|
raise PolicyError("SCM request headers exceed the safe limit")
|
|
if len(self.headers.get_all("Content-Length", [])) > 1:
|
|
raise PolicyError("SCM request has duplicate Content-Length")
|
|
|
|
def _stream(
|
|
self, status: int, content_type: str, body: BinaryIO, length: int
|
|
) -> None:
|
|
self.send_response(status)
|
|
self.send_header("Content-Type", content_type)
|
|
self.send_header("Content-Length", str(length))
|
|
self.end_headers()
|
|
deadline = time.monotonic() + INBOUND_BODY_TIMEOUT
|
|
while True:
|
|
chunk = body.read(STREAM_CHUNK)
|
|
if not chunk:
|
|
break
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise PolicyError("SCM response write deadline exceeded")
|
|
self.connection.settimeout(remaining)
|
|
self.wfile.write(chunk)
|
|
|
|
def log_message(self, _format: str, *_args: object) -> None:
|
|
return
|
|
|
|
def _json(self, status: int, body: bytes) -> None:
|
|
self.send_response(status)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.send_header("Cache-Control", "no-store")
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def _reject(self, status: int = 400) -> None:
|
|
self._json(status, b'{"error":"request rejected"}')
|
|
|
|
def do_GET(self) -> None:
|
|
try:
|
|
self._validate_headers()
|
|
if self.path == "/healthz":
|
|
self._json(200, b'{"status":"ok"}')
|
|
return
|
|
repo, operation, service = _git_target(self.path)
|
|
if operation != "info/refs":
|
|
raise PolicyError("Git RPC requires POST")
|
|
token = read_token()
|
|
_reject_forbidden(repo, "repository", (token,))
|
|
expected = f"application/x-{service}-advertisement"
|
|
streamed = _upstream_git_request(
|
|
f"/atlas/{repo}.git/info/refs?service={service}",
|
|
method="GET",
|
|
body=None,
|
|
content_type=None,
|
|
expected_type=expected,
|
|
token=token,
|
|
stream_result=True,
|
|
)
|
|
body, length = streamed
|
|
try:
|
|
self._stream(200, expected, body, length)
|
|
finally:
|
|
body.close()
|
|
except (
|
|
OSError,
|
|
PolicyError,
|
|
socket.timeout,
|
|
urllib.error.URLError,
|
|
ValueError,
|
|
):
|
|
self._reject()
|
|
|
|
def do_POST(self) -> None:
|
|
try:
|
|
self._validate_headers()
|
|
self.connection.settimeout(INBOUND_BODY_TIMEOUT)
|
|
if self.path in {"/v1/metadata", "/v1/drafts"}:
|
|
self._control()
|
|
else:
|
|
self._git_rpc()
|
|
except (
|
|
OSError,
|
|
PolicyError,
|
|
socket.timeout,
|
|
urllib.error.URLError,
|
|
ValueError,
|
|
json.JSONDecodeError,
|
|
):
|
|
self._reject()
|
|
|
|
def _control(self) -> None:
|
|
data = _load_json(self)
|
|
token = read_token()
|
|
if self.path == "/v1/metadata":
|
|
if set(data) != {"path"} or not isinstance(data["path"], str):
|
|
raise PolicyError("metadata request fields are invalid")
|
|
result = read(data["path"], token=token)
|
|
else:
|
|
expected = {"base", "body", "head", "head_sha", "repo", "title"}
|
|
if set(data) != expected or not all(
|
|
isinstance(data[key], str) for key in expected
|
|
):
|
|
raise PolicyError("draft request fields are invalid")
|
|
result = create_draft(token=token, **data) # type: ignore[arg-type]
|
|
if token.encode("utf-8") in result:
|
|
raise PolicyError("SCM upstream reflected credential material")
|
|
self._json(200, result)
|
|
|
|
def _git_rpc(self) -> None:
|
|
repo, operation, service = _git_target(self.path)
|
|
if (
|
|
operation not in {"git-upload-pack", "git-receive-pack"}
|
|
or service != operation
|
|
):
|
|
raise PolicyError("Git RPC operation is outside the allowlist")
|
|
expected_request = f"application/x-{service}-request"
|
|
if self.headers.get_content_type() != expected_request or self.headers.get(
|
|
"Transfer-Encoding"
|
|
):
|
|
raise PolicyError("Git RPC request type is invalid")
|
|
length = _content_length(self.headers, MAX_GIT_REQUEST)
|
|
token = read_token()
|
|
_reject_forbidden(repo, "repository", (token,))
|
|
body, body_length = _spool_bounded(
|
|
self.rfile,
|
|
MAX_GIT_REQUEST,
|
|
length,
|
|
token=token,
|
|
context="Git request",
|
|
deadline=time.monotonic() + INBOUND_BODY_TIMEOUT,
|
|
set_timeout=self.connection.settimeout,
|
|
)
|
|
try:
|
|
if service == "git-receive-pack":
|
|
_validate_receive_pack(body, token)
|
|
expected = f"application/x-{service}-result"
|
|
streamed = _upstream_git_request(
|
|
f"/atlas/{repo}.git/{service}",
|
|
method="POST",
|
|
body=body,
|
|
body_length=body_length,
|
|
content_type=expected_request,
|
|
expected_type=expected,
|
|
token=token,
|
|
stream_result=True,
|
|
)
|
|
result, result_length = streamed
|
|
try:
|
|
self._stream(200, expected, result, result_length)
|
|
finally:
|
|
result.close()
|
|
finally:
|
|
body.close()
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--listen", default="0.0.0.0")
|
|
parser.add_argument("--port", type=int, default=BROKER_PORT)
|
|
args = parser.parse_args()
|
|
BoundedThreadingHTTPServer((args.listen, args.port), BrokerHandler).serve_forever()
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|