340ec58b68
Inflate every receive-pack object under strict pack, size, and checksum bounds, resolve deltas against in-pack bases only, and scan the real decompressed payloads for runtime-token forms, private keys, SSH key material, and known provider token formats. Thin packs are rejected so no pushed content escapes the scan, and upstream Git exchanges now run under one absolute stream deadline. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
309 lines
12 KiB
Python
309 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Bounded Git pack inflation and delta resolution for quarantine scanning.
|
|
|
|
Pushed objects arrive zlib-compressed inside a packfile, so any content
|
|
screening must inflate the real payloads first. This module unpacks one
|
|
self-contained pack under explicit object, size, and checksum bounds and
|
|
resolves every delta against bases inside the same pack. Thin packs are
|
|
rejected outright: a delta against a base the broker cannot see would leave
|
|
part of the pushed content unscanned.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import tempfile
|
|
import zlib
|
|
from typing import BinaryIO
|
|
|
|
from gitea_api_policy import PolicyError
|
|
|
|
OBJECT_TYPE_NAMES = {1: b"commit", 2: b"tree", 3: b"blob", 4: b"tag"}
|
|
OFS_DELTA = 6
|
|
REF_DELTA = 7
|
|
MAX_PACK_OBJECTS = 50_000
|
|
MAX_OBJECT_BYTES = 32 * 1024 * 1024
|
|
MAX_TOTAL_BYTES = 256 * 1024 * 1024
|
|
MAX_RESOLVE_PASSES = 64
|
|
SPOOL_MEMORY_LIMIT = 256 * 1024
|
|
READ_CHUNK = 64 * 1024
|
|
|
|
|
|
def _read_exact(stream: BinaryIO, size: int) -> bytes:
|
|
value = stream.read(size)
|
|
if len(value) != size:
|
|
raise PolicyError("Git pack data ended early")
|
|
return value
|
|
|
|
|
|
def _object_header(stream: BinaryIO) -> tuple[int, int]:
|
|
"""Parse one object's type and declared inflated size."""
|
|
byte = _read_exact(stream, 1)[0]
|
|
type_code = (byte >> 4) & 0x7
|
|
size = byte & 0x0F
|
|
shift = 4
|
|
while byte & 0x80:
|
|
if shift > 60:
|
|
raise PolicyError("Git object header is invalid")
|
|
byte = _read_exact(stream, 1)[0]
|
|
size |= (byte & 0x7F) << shift
|
|
shift += 7
|
|
return type_code, size
|
|
|
|
|
|
def _delta_distance(stream: BinaryIO) -> int:
|
|
"""Parse one ofs-delta base distance."""
|
|
byte = _read_exact(stream, 1)[0]
|
|
value = byte & 0x7F
|
|
while byte & 0x80:
|
|
byte = _read_exact(stream, 1)[0]
|
|
value = ((value + 1) << 7) | (byte & 0x7F)
|
|
if value > 1 << 48:
|
|
raise PolicyError("Git delta base distance is invalid")
|
|
return value
|
|
|
|
|
|
def _inflate(stream: BinaryIO, declared: int) -> BinaryIO:
|
|
"""Inflate one object into a quarantine spool, bounded by its header."""
|
|
spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - caller owns returned spool
|
|
max_size=SPOOL_MEMORY_LIMIT, dir="/tmp"
|
|
)
|
|
decompressor = zlib.decompressobj()
|
|
produced = 0
|
|
try:
|
|
while not decompressor.eof:
|
|
data = stream.read(READ_CHUNK)
|
|
if not data:
|
|
raise PolicyError("Git pack object data ended early")
|
|
while True:
|
|
value = decompressor.decompress(data, READ_CHUNK)
|
|
produced += len(value)
|
|
if produced > declared:
|
|
raise PolicyError("Git object size does not match its header")
|
|
spool.write(value)
|
|
data = decompressor.unconsumed_tail
|
|
if decompressor.eof or not data:
|
|
break
|
|
if decompressor.unused_data:
|
|
stream.seek(-len(decompressor.unused_data), 1)
|
|
if produced != declared:
|
|
raise PolicyError("Git object size does not match its header")
|
|
spool.seek(0)
|
|
return spool
|
|
except zlib.error as exc:
|
|
spool.close()
|
|
raise PolicyError("Git pack object data is corrupt") from exc
|
|
except Exception:
|
|
spool.close()
|
|
raise
|
|
|
|
|
|
def _delta_size(delta: bytes, position: int) -> tuple[int, int]:
|
|
size = 0
|
|
shift = 0
|
|
while True:
|
|
if position >= len(delta) or shift > 60:
|
|
raise PolicyError("Git delta header is invalid")
|
|
byte = delta[position]
|
|
position += 1
|
|
size |= (byte & 0x7F) << shift
|
|
shift += 7
|
|
if not byte & 0x80:
|
|
return size, position
|
|
|
|
|
|
def _apply_delta(base: bytes, delta: bytes) -> bytes:
|
|
"""Apply one Git delta and require exact declared sizes throughout."""
|
|
source_size, position = _delta_size(delta, 0)
|
|
target_size, position = _delta_size(delta, position)
|
|
if source_size != len(base) or target_size > MAX_OBJECT_BYTES:
|
|
raise PolicyError("Git delta sizes are invalid")
|
|
parts: list[bytes] = []
|
|
produced = 0
|
|
while position < len(delta):
|
|
opcode = delta[position]
|
|
position += 1
|
|
if opcode & 0x80:
|
|
operands = [bit for bit in (1, 2, 4, 8, 16, 32, 64) if opcode & bit]
|
|
if position + len(operands) > len(delta):
|
|
raise PolicyError("Git delta copy is truncated")
|
|
offset = 0
|
|
size = 0
|
|
for shift, bit in enumerate((1, 2, 4, 8)):
|
|
if opcode & bit:
|
|
offset |= delta[position] << (8 * shift)
|
|
position += 1
|
|
for shift, bit in enumerate((16, 32, 64)):
|
|
if opcode & bit:
|
|
size |= delta[position] << (8 * shift)
|
|
position += 1
|
|
size = size or 0x10000
|
|
if offset + size > len(base):
|
|
raise PolicyError("Git delta copy is out of bounds")
|
|
parts.append(base[offset : offset + size])
|
|
produced += size
|
|
elif opcode:
|
|
if position + opcode > len(delta):
|
|
raise PolicyError("Git delta insert is truncated")
|
|
parts.append(delta[position : position + opcode])
|
|
position += opcode
|
|
produced += opcode
|
|
else:
|
|
raise PolicyError("Git delta instruction is invalid")
|
|
if produced > target_size:
|
|
raise PolicyError("Git delta result exceeds its declared size")
|
|
if produced != target_size:
|
|
raise PolicyError("Git delta result size is invalid")
|
|
return b"".join(parts)
|
|
|
|
|
|
def _payload_sha(type_code: int, payload: BinaryIO, size: int) -> bytes:
|
|
digest = hashlib.sha1(OBJECT_TYPE_NAMES[type_code] + b" %d\x00" % size)
|
|
payload.seek(0)
|
|
while True:
|
|
chunk = payload.read(READ_CHUNK)
|
|
if not chunk:
|
|
break
|
|
digest.update(chunk)
|
|
payload.seek(0)
|
|
return digest.digest()
|
|
|
|
|
|
def _verify_checksum(stream: BinaryIO, start: int, end: int) -> None:
|
|
trailer = _read_exact(stream, 20)
|
|
stream.seek(start)
|
|
digest = hashlib.sha1()
|
|
remaining = end - start
|
|
while remaining:
|
|
chunk = stream.read(min(READ_CHUNK, remaining))
|
|
digest.update(chunk)
|
|
remaining -= len(chunk)
|
|
if digest.digest() != trailer:
|
|
raise PolicyError("Git pack checksum is invalid")
|
|
stream.seek(end + 20)
|
|
|
|
|
|
class _Entry:
|
|
"""One pack entry: either resolved content or an unresolved delta."""
|
|
|
|
def __init__(self, type_code: int, payload: BinaryIO, size: int, base):
|
|
self.type_code = type_code
|
|
self.payload = payload
|
|
self.size = size
|
|
self.base = base
|
|
|
|
|
|
def _resolve(entries: dict[int, _Entry], total: list[int]) -> None:
|
|
"""Resolve every delta against in-pack bases only, to a fixed point."""
|
|
by_sha: dict[bytes, int] = {}
|
|
for offset, entry in entries.items():
|
|
if entry.base is None:
|
|
by_sha.setdefault(
|
|
_payload_sha(entry.type_code, entry.payload, entry.size), offset
|
|
)
|
|
for _ in range(MAX_RESOLVE_PASSES):
|
|
pending = [item for item in entries.items() if item[1].base is not None]
|
|
if not pending:
|
|
return
|
|
progressed = False
|
|
for offset, entry in pending:
|
|
base_key = entry.base
|
|
base_offset = base_key if isinstance(base_key, int) else by_sha.get(base_key)
|
|
base = entries.get(base_offset) if base_offset is not None else None
|
|
if base is None or base.base is not None:
|
|
if isinstance(base_key, int) and base is None:
|
|
raise PolicyError("Git delta base offset is invalid")
|
|
continue
|
|
base.payload.seek(0)
|
|
entry.payload.seek(0)
|
|
value = _apply_delta(base.payload.read(), entry.payload.read())
|
|
total[0] += len(value)
|
|
if total[0] > MAX_TOTAL_BYTES:
|
|
raise PolicyError("Git pack contents exceed the safe size limit")
|
|
entry.payload.close()
|
|
spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - entry owns spool
|
|
max_size=SPOOL_MEMORY_LIMIT, dir="/tmp"
|
|
)
|
|
spool.write(value)
|
|
spool.seek(0)
|
|
entry.type_code = base.type_code
|
|
entry.payload = spool
|
|
entry.size = len(value)
|
|
entry.base = None
|
|
by_sha.setdefault(_payload_sha(entry.type_code, spool, entry.size), offset)
|
|
progressed = True
|
|
if not progressed:
|
|
raise PolicyError("Git delta base is outside the pack; push full packs")
|
|
raise PolicyError("Git delta chains exceed the safe resolution limit")
|
|
|
|
|
|
def unpack_objects(stream: BinaryIO) -> list[tuple[int, BinaryIO]]:
|
|
"""Inflate one self-contained pack and return every resolved payload."""
|
|
start = stream.tell()
|
|
header = _read_exact(stream, 12)
|
|
version = int.from_bytes(header[4:8], "big")
|
|
count = int.from_bytes(header[8:12], "big")
|
|
if header[:4] != b"PACK" or version not in {2, 3}:
|
|
raise PolicyError("Git pack header is invalid")
|
|
if count > MAX_PACK_OBJECTS:
|
|
raise PolicyError("Git pack contains too many objects")
|
|
entries: dict[int, _Entry] = {}
|
|
total = [0]
|
|
try:
|
|
for _ in range(count):
|
|
offset = stream.tell() - start
|
|
type_code, declared = _object_header(stream)
|
|
if declared > MAX_OBJECT_BYTES:
|
|
raise PolicyError("Git object exceeds the safe size limit")
|
|
total[0] += declared
|
|
if total[0] > MAX_TOTAL_BYTES:
|
|
raise PolicyError("Git pack contents exceed the safe size limit")
|
|
base: int | bytes | None = None
|
|
if type_code == OFS_DELTA:
|
|
distance = _delta_distance(stream)
|
|
base = offset - distance
|
|
if base < 0:
|
|
raise PolicyError("Git delta base offset is invalid")
|
|
elif type_code == REF_DELTA:
|
|
base = _read_exact(stream, 20)
|
|
elif type_code not in OBJECT_TYPE_NAMES:
|
|
raise PolicyError("Git pack object type is invalid")
|
|
entries[offset] = _Entry(type_code, _inflate(stream, declared), declared, base)
|
|
_verify_checksum(stream, start, stream.tell())
|
|
_resolve(entries, total)
|
|
except Exception:
|
|
for entry in entries.values():
|
|
entry.payload.close()
|
|
raise
|
|
for entry in entries.values():
|
|
entry.payload.seek(0)
|
|
return [
|
|
(entries[offset].type_code, entries[offset].payload)
|
|
for offset in sorted(entries)
|
|
]
|
|
|
|
|
|
def parse_loose_object(value: bytes) -> tuple[int, bytes]:
|
|
"""Inflate one quarantined loose object and return its typed payload."""
|
|
if len(value) > MAX_OBJECT_BYTES:
|
|
raise PolicyError("Git object exceeds the safe size limit")
|
|
decompressor = zlib.decompressobj()
|
|
try:
|
|
inflated = decompressor.decompress(value, MAX_OBJECT_BYTES + 1)
|
|
except zlib.error as exc:
|
|
raise PolicyError("Git loose object data is corrupt") from exc
|
|
if not decompressor.eof or decompressor.unused_data or len(inflated) > MAX_OBJECT_BYTES:
|
|
raise PolicyError("Git loose object framing is invalid")
|
|
kind, separator, remainder = inflated.partition(b"\x00")
|
|
name, space, size = kind.partition(b" ")
|
|
types = {label: code for code, label in OBJECT_TYPE_NAMES.items()}
|
|
if (
|
|
not separator
|
|
or not space
|
|
or name not in types
|
|
or not size.isdigit()
|
|
or int(size) != len(remainder)
|
|
):
|
|
raise PolicyError("Git loose object header is invalid")
|
|
return types[name], remainder
|