456 lines
16 KiB
Python
456 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""Serve the credential-isolated Atlas API and Git smart-HTTP boundary."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import json
|
|
import re
|
|
import socket
|
|
import tempfile
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from http.server import BaseHTTPRequestHandler
|
|
from typing import BinaryIO
|
|
|
|
import deadline_http
|
|
from gitea_api import (
|
|
CANONICAL_BASE_URL,
|
|
PolicyError,
|
|
create_draft,
|
|
read,
|
|
read_token,
|
|
)
|
|
from gitea_api_policy import _reject_forbidden, _validate_repo
|
|
from receive_pack_scan import validate_receive_pack
|
|
from scm_broker_io import RejectRedirect, response_status as _status, spool_response
|
|
from scm_broker_server import AbsoluteHeaderDeadlineMixin, BoundedThreadingHTTPServer
|
|
|
|
BROKER_PORT = 9081
|
|
GIT_USER = "hermes-automation"
|
|
MAX_CONTROL_BODY = 64 * 1024
|
|
MAX_GIT_REQUEST = 128 * 1024 * 1024
|
|
MAX_GIT_RESPONSE = 128 * 1024 * 1024
|
|
MAX_HEADERS = 32
|
|
MAX_HEADER_BYTES = 16 * 1024
|
|
SPOOL_MEMORY_LIMIT = 1024 * 1024
|
|
STREAM_CHUNK = 64 * 1024
|
|
INBOUND_HEADER_TIMEOUT = 10.0
|
|
INBOUND_BODY_TIMEOUT = 120.0
|
|
UPSTREAM_DEADLINE_SECONDS = 300.0
|
|
GIT_PATH_RE = re.compile(
|
|
r"/git/atlas/(?P<repo>[A-Za-z0-9][A-Za-z0-9._-]{0,99})\.git/"
|
|
r"(?P<operation>info/refs|git-upload-pack|git-receive-pack)\Z"
|
|
)
|
|
|
|
|
|
def _read_bounded(
|
|
stream,
|
|
maximum: int,
|
|
length: int | None = None,
|
|
*,
|
|
deadline: float | None = None,
|
|
set_timeout=None,
|
|
) -> bytes:
|
|
if length is not None and not 0 <= length <= maximum:
|
|
raise PolicyError("SCM request exceeds the safe size limit")
|
|
wanted = maximum + 1 if length is None else length
|
|
chunks: list[bytes] = []
|
|
total = 0
|
|
while total < wanted:
|
|
if deadline is not None:
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise PolicyError("SCM request body deadline exceeded")
|
|
if set_timeout is not None:
|
|
set_timeout(remaining)
|
|
chunk = stream.read(min(STREAM_CHUNK, wanted - total))
|
|
if not chunk:
|
|
break
|
|
chunks.append(chunk)
|
|
total += len(chunk)
|
|
value = b"".join(chunks)
|
|
if len(value) > maximum or (length is not None and len(value) != length):
|
|
raise PolicyError("SCM request exceeds the safe size limit")
|
|
return value
|
|
|
|
|
|
def _spool_bounded(
|
|
stream,
|
|
maximum: int,
|
|
length: int,
|
|
*,
|
|
token: str,
|
|
context: str,
|
|
deadline: float,
|
|
set_timeout=None,
|
|
) -> tuple[BinaryIO, int]:
|
|
"""Copy a fixed-length exchange through a bounded-memory disk spool."""
|
|
if not 0 <= length <= maximum:
|
|
raise PolicyError("SCM request exceeds the safe size limit")
|
|
spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool
|
|
max_size=SPOOL_MEMORY_LIMIT, dir="/tmp"
|
|
)
|
|
remaining = length
|
|
forms = _credential_forms(token)
|
|
carry = b""
|
|
try:
|
|
while remaining:
|
|
remaining_time = deadline - time.monotonic()
|
|
if remaining_time <= 0:
|
|
raise PolicyError("SCM request body deadline exceeded")
|
|
if set_timeout is not None:
|
|
set_timeout(remaining_time)
|
|
chunk = stream.read(min(STREAM_CHUNK, remaining))
|
|
if not chunk:
|
|
raise PolicyError("SCM request body ended early")
|
|
candidate = carry + chunk
|
|
if any(form in candidate for form in forms):
|
|
raise PolicyError(f"{context} contains runtime credential material")
|
|
width = max(len(form) for form in forms) - 1
|
|
carry = candidate[-width:] if width else b""
|
|
spool.write(chunk)
|
|
remaining -= len(chunk)
|
|
spool.seek(0)
|
|
return spool, length
|
|
except Exception:
|
|
spool.close()
|
|
raise
|
|
|
|
|
|
def _spool_response(
|
|
response, maximum: int, token: str, deadline_seconds: float
|
|
) -> tuple[BinaryIO, int]:
|
|
return spool_response(
|
|
response,
|
|
maximum,
|
|
_credential_forms(token),
|
|
memory_limit=SPOOL_MEMORY_LIMIT,
|
|
chunk_size=STREAM_CHUNK,
|
|
deadline_seconds=deadline_seconds,
|
|
)
|
|
|
|
|
|
def _load_json(handler: BaseHTTPRequestHandler) -> dict[str, object]:
|
|
if handler.headers.get("Transfer-Encoding"):
|
|
raise PolicyError("chunked broker control requests are not allowed")
|
|
if handler.headers.get_content_type() != "application/json":
|
|
raise PolicyError("broker control request must be JSON")
|
|
length = _content_length(handler.headers, MAX_CONTROL_BODY)
|
|
deadline = time.monotonic() + INBOUND_BODY_TIMEOUT
|
|
value = json.loads(
|
|
_read_bounded(
|
|
handler.rfile,
|
|
MAX_CONTROL_BODY,
|
|
length,
|
|
deadline=deadline,
|
|
set_timeout=handler.connection.settimeout,
|
|
)
|
|
)
|
|
if not isinstance(value, dict):
|
|
raise PolicyError("broker control request must be an object")
|
|
return value
|
|
|
|
|
|
def _git_target(raw: str) -> tuple[str, str, str]:
|
|
if (
|
|
not raw.isascii()
|
|
or any(ord(character) < 32 or ord(character) == 127 for character in raw)
|
|
or "%" in raw
|
|
or "\\" in raw
|
|
or len(raw) > 512
|
|
):
|
|
raise PolicyError("Git target is not canonical ASCII")
|
|
target = urllib.parse.urlsplit(raw)
|
|
canonical = urllib.parse.urlunsplit(("", "", target.path, target.query, ""))
|
|
if canonical != raw:
|
|
raise PolicyError("Git target is not in exact canonical form")
|
|
if target.scheme or target.netloc or target.fragment:
|
|
raise PolicyError("Git target must be relative")
|
|
match = GIT_PATH_RE.fullmatch(target.path)
|
|
if not match:
|
|
raise PolicyError("Git target is outside the Atlas allowlist")
|
|
repo = _validate_repo(match.group("repo"))
|
|
operation = match.group("operation")
|
|
query = target.query
|
|
if operation == "info/refs":
|
|
if query not in {"service=git-upload-pack", "service=git-receive-pack"}:
|
|
raise PolicyError("Git discovery service is outside the allowlist")
|
|
service = query.removeprefix("service=")
|
|
elif query:
|
|
raise PolicyError("Git RPC query parameters are not allowed")
|
|
else:
|
|
service = operation
|
|
return repo, operation, service
|
|
|
|
|
|
def _content_length(headers: object, maximum: int) -> int:
|
|
"""Parse one short canonical bounded Content-Length header."""
|
|
raw = headers.get("Content-Length", "") # type: ignore[attr-defined]
|
|
if (
|
|
not isinstance(raw, str)
|
|
or not raw.isascii()
|
|
or len(raw) > 10
|
|
or not raw.isdigit()
|
|
):
|
|
raise PolicyError("SCM request length is invalid")
|
|
length = int(raw)
|
|
if raw != str(length) or not 0 <= length <= maximum:
|
|
raise PolicyError("SCM request length is invalid")
|
|
return length
|
|
|
|
|
|
def _credential_forms(token: str) -> tuple[bytes, ...]:
|
|
basic = base64.b64encode(f"{GIT_USER}:{token}".encode())
|
|
return token.encode("utf-8"), basic, b"Basic " + basic
|
|
|
|
|
|
def _validate_receive_pack(body: bytes | BinaryIO, token: str) -> None:
|
|
"""Permit only vetted new-feature-branch pushes with scanned contents."""
|
|
validate_receive_pack(body, token, _credential_forms(token))
|
|
|
|
|
|
def _guarded_opener(guard: deadline_http.StreamDeadline):
|
|
"""Build an opener whose connection obeys one absolute stream deadline."""
|
|
return urllib.request.build_opener(RejectRedirect(), *guard.handlers()).open
|
|
|
|
|
|
def _upstream_git_request(
|
|
target: str,
|
|
*,
|
|
method: str,
|
|
body: bytes | BinaryIO | None,
|
|
content_type: str | None,
|
|
expected_type: str,
|
|
token: str,
|
|
body_length: int | None = None,
|
|
stream_result: bool = False,
|
|
opener=None,
|
|
) -> bytes | tuple[BinaryIO, int]:
|
|
credentials = base64.b64encode(f"{GIT_USER}:{token}".encode()).decode("ascii")
|
|
headers = {"Accept": expected_type, "Authorization": f"Basic {credentials}"}
|
|
if content_type:
|
|
headers["Content-Type"] = content_type
|
|
if body is not None:
|
|
if body_length is None:
|
|
if not isinstance(body, bytes):
|
|
raise PolicyError("Git upstream request length is missing")
|
|
body_length = len(body)
|
|
headers["Content-Length"] = str(body_length)
|
|
request = urllib.request.Request(
|
|
CANONICAL_BASE_URL + target,
|
|
data=body,
|
|
method=method,
|
|
headers=headers,
|
|
)
|
|
# One wall-clock budget covers connect, send, headers, and body read; the
|
|
# watchdog closes the upstream socket if any phase overstays it.
|
|
guard = deadline_http.StreamDeadline(UPSTREAM_DEADLINE_SECONDS)
|
|
try:
|
|
open_upstream = opener if opener is not None else _guarded_opener(guard)
|
|
with open_upstream(request, timeout=guard.remaining()) as response:
|
|
if _status(response) != 200:
|
|
raise PolicyError("Git upstream returned an unexpected HTTP status")
|
|
if response.headers.get_content_type() != expected_type:
|
|
raise PolicyError("Git upstream returned an unexpected response type")
|
|
result, result_length = _spool_response(
|
|
response, MAX_GIT_RESPONSE, token, guard.remaining()
|
|
)
|
|
finally:
|
|
guard.cancel()
|
|
if stream_result:
|
|
return result, result_length
|
|
try:
|
|
return result.read()
|
|
finally:
|
|
result.close()
|
|
|
|
|
|
class BrokerHandler(AbsoluteHeaderDeadlineMixin, BaseHTTPRequestHandler):
|
|
"""Expose only bounded metadata, draft creation, and smart-HTTP Git."""
|
|
|
|
server_version = "HermesSCMBroker/1"
|
|
sys_version = ""
|
|
header_deadline_seconds = INBOUND_HEADER_TIMEOUT
|
|
|
|
def _validate_headers(self) -> None:
|
|
items = list(self.headers.items())
|
|
if len(items) > MAX_HEADERS:
|
|
raise PolicyError("SCM request has too many headers")
|
|
total = 0
|
|
for name, value in items:
|
|
if not name.isascii() or not value.isascii():
|
|
raise PolicyError("SCM request headers must be ASCII")
|
|
total += len(name) + len(value) + 4
|
|
if any(ord(character) < 32 and character != "\t" for character in value):
|
|
raise PolicyError("SCM request header contains controls")
|
|
if total > MAX_HEADER_BYTES:
|
|
raise PolicyError("SCM request headers exceed the safe limit")
|
|
if len(self.headers.get_all("Content-Length", [])) > 1:
|
|
raise PolicyError("SCM request has duplicate Content-Length")
|
|
|
|
def _stream(
|
|
self, status: int, content_type: str, body: BinaryIO, length: int
|
|
) -> None:
|
|
self.send_response(status)
|
|
self.send_header("Content-Type", content_type)
|
|
self.send_header("Content-Length", str(length))
|
|
self.end_headers()
|
|
deadline = time.monotonic() + INBOUND_BODY_TIMEOUT
|
|
while True:
|
|
chunk = body.read(STREAM_CHUNK)
|
|
if not chunk:
|
|
break
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise PolicyError("SCM response write deadline exceeded")
|
|
self.connection.settimeout(remaining)
|
|
self.wfile.write(chunk)
|
|
|
|
def log_message(self, _format: str, *_args: object) -> None:
|
|
return
|
|
|
|
def _json(self, status: int, body: bytes) -> None:
|
|
self.send_response(status)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.send_header("Cache-Control", "no-store")
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def _reject(self, status: int = 400) -> None:
|
|
self._json(status, b'{"error":"request rejected"}')
|
|
|
|
def do_GET(self) -> None:
|
|
try:
|
|
self._validate_headers()
|
|
if self.path == "/healthz":
|
|
self._json(200, b'{"status":"ok"}')
|
|
return
|
|
repo, operation, service = _git_target(self.path)
|
|
if operation != "info/refs":
|
|
raise PolicyError("Git RPC requires POST")
|
|
token = read_token()
|
|
_reject_forbidden(repo, "repository", (token,))
|
|
expected = f"application/x-{service}-advertisement"
|
|
streamed = _upstream_git_request(
|
|
f"/titan/{repo}.git/info/refs?service={service}",
|
|
method="GET",
|
|
body=None,
|
|
content_type=None,
|
|
expected_type=expected,
|
|
token=token,
|
|
stream_result=True,
|
|
)
|
|
body, length = streamed
|
|
try:
|
|
self._stream(200, expected, body, length)
|
|
finally:
|
|
body.close()
|
|
except (
|
|
OSError,
|
|
PolicyError,
|
|
socket.timeout,
|
|
urllib.error.URLError,
|
|
ValueError,
|
|
):
|
|
self._reject()
|
|
|
|
def do_POST(self) -> None:
|
|
try:
|
|
self._validate_headers()
|
|
self.connection.settimeout(INBOUND_BODY_TIMEOUT)
|
|
if self.path in {"/v1/metadata", "/v1/drafts"}:
|
|
self._control()
|
|
else:
|
|
self._git_rpc()
|
|
except (
|
|
OSError,
|
|
PolicyError,
|
|
socket.timeout,
|
|
urllib.error.URLError,
|
|
ValueError,
|
|
json.JSONDecodeError,
|
|
):
|
|
self._reject()
|
|
|
|
def _control(self) -> None:
|
|
data = _load_json(self)
|
|
token = read_token()
|
|
if self.path == "/v1/metadata":
|
|
if set(data) != {"path"} or not isinstance(data["path"], str):
|
|
raise PolicyError("metadata request fields are invalid")
|
|
result = read(data["path"], token=token)
|
|
else:
|
|
expected = {"base", "body", "head", "head_sha", "repo", "title"}
|
|
if set(data) != expected or not all(
|
|
isinstance(data[key], str) for key in expected
|
|
):
|
|
raise PolicyError("draft request fields are invalid")
|
|
result = create_draft(token=token, **data) # type: ignore[arg-type]
|
|
if token.encode("utf-8") in result:
|
|
raise PolicyError("SCM upstream reflected credential material")
|
|
self._json(200, result)
|
|
|
|
def _git_rpc(self) -> None:
|
|
repo, operation, service = _git_target(self.path)
|
|
if (
|
|
operation not in {"git-upload-pack", "git-receive-pack"}
|
|
or service != operation
|
|
):
|
|
raise PolicyError("Git RPC operation is outside the allowlist")
|
|
expected_request = f"application/x-{service}-request"
|
|
if self.headers.get_content_type() != expected_request or self.headers.get(
|
|
"Transfer-Encoding"
|
|
):
|
|
raise PolicyError("Git RPC request type is invalid")
|
|
length = _content_length(self.headers, MAX_GIT_REQUEST)
|
|
token = read_token()
|
|
_reject_forbidden(repo, "repository", (token,))
|
|
body, body_length = _spool_bounded(
|
|
self.rfile,
|
|
MAX_GIT_REQUEST,
|
|
length,
|
|
token=token,
|
|
context="Git request",
|
|
deadline=time.monotonic() + INBOUND_BODY_TIMEOUT,
|
|
set_timeout=self.connection.settimeout,
|
|
)
|
|
try:
|
|
if service == "git-receive-pack":
|
|
_validate_receive_pack(body, token)
|
|
expected = f"application/x-{service}-result"
|
|
streamed = _upstream_git_request(
|
|
f"/titan/{repo}.git/{service}",
|
|
method="POST",
|
|
body=body,
|
|
body_length=body_length,
|
|
content_type=expected_request,
|
|
expected_type=expected,
|
|
token=token,
|
|
stream_result=True,
|
|
)
|
|
result, result_length = streamed
|
|
try:
|
|
self._stream(200, expected, result, result_length)
|
|
finally:
|
|
result.close()
|
|
finally:
|
|
body.close()
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--listen", default="0.0.0.0")
|
|
parser.add_argument("--port", type=int, default=BROKER_PORT)
|
|
args = parser.parse_args()
|
|
BoundedThreadingHTTPServer((args.listen, args.port), BrokerHandler).serve_forever()
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|