494 lines
17 KiB
Python
494 lines
17 KiB
Python
"""Adversarial contracts for chat-only continuity and truthful rendering."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import runpy
|
|
import sqlite3
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
import yaml
|
|
|
|
|
|
ROOT = Path(__file__).parents[2]
|
|
HERMES = ROOT / "services" / "hermes"
|
|
|
|
|
|
def _load(name: str, path: Path):
|
|
spec = importlib.util.spec_from_file_location(name, path)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
@pytest.fixture
|
|
def continuity():
|
|
return _load(
|
|
"telegram_continuity",
|
|
HERMES / "scripts" / "migrate_telegram_api_sessions.py",
|
|
)
|
|
|
|
|
|
def test_context_is_deterministic_bounded_deduplicated_and_truthful(continuity):
|
|
turns = []
|
|
for index in range(30):
|
|
user = {"role": "user", "content": f"question {index}", "_db_persisted": True}
|
|
assistant = {
|
|
"role": "assistant",
|
|
"content": f"answer {index}",
|
|
"_db_persisted": True,
|
|
}
|
|
turns.extend((user, assistant))
|
|
if index == 4:
|
|
# Simulate one complete transcript replay.
|
|
turns.extend((user.copy(), assistant.copy()))
|
|
turns.extend(
|
|
[
|
|
{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call-1",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "browser",
|
|
"arguments": "x" * 40_000,
|
|
},
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"role": "tool",
|
|
"tool_call_id": "call-1",
|
|
"tool_name": "browser",
|
|
"content": "tool output " * 4_000,
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{"type": "input_text", "text": "edit this"},
|
|
{
|
|
"type": "input_image",
|
|
"image_url": "data:image/png;base64," + "a" * 80_000,
|
|
},
|
|
],
|
|
},
|
|
{
|
|
"role": "assistant",
|
|
"content": "Created it MEDIA:/opt/data/cache/images/private.png",
|
|
},
|
|
]
|
|
)
|
|
|
|
compacted = continuity.compact_telegram_history(turns)
|
|
|
|
assert compacted == continuity.compact_telegram_history(turns)
|
|
assert compacted[0]["role"] == "system"
|
|
assert compacted[0]["_db_persisted"] is True
|
|
assert compacted[0]["content"].startswith(continuity.SUMMARY_PREFIX)
|
|
assert sum(item["role"] == "user" for item in compacted[1:]) <= 12
|
|
assert len(compacted[1:]) <= continuity.MAX_HISTORY_ITEMS
|
|
assert len(json.dumps(compacted).encode()) <= (
|
|
continuity.MAX_HISTORY_BYTES + continuity.MAX_SUMMARY_CHARS
|
|
)
|
|
encoded = json.dumps(compacted)
|
|
assert "data:image" not in encoded
|
|
assert "MEDIA:" not in encoded
|
|
assert "/opt/data/" not in encoded
|
|
assert continuity.ATTACHMENT_NOTE in encoded
|
|
assert "question 29" in encoded
|
|
assert (
|
|
next(item for item in compacted if item.get("content") == "question 29")[
|
|
"_db_persisted"
|
|
]
|
|
is True
|
|
)
|
|
assert "x" * 3_000 not in encoded
|
|
fingerprints = [continuity._fingerprint(item) for item in compacted]
|
|
assert all(
|
|
left != right
|
|
for left, right in zip(fingerprints, fingerprints[1:], strict=False)
|
|
)
|
|
|
|
|
|
def test_context_rejects_malformed_history_and_bounds_single_large_item(continuity):
|
|
assert continuity.compact_telegram_history(None) == []
|
|
assert continuity.compact_telegram_history({"role": "user"}) == []
|
|
malformed = [
|
|
None,
|
|
"text",
|
|
{"role": "unknown", "content": "skip"},
|
|
{"role": "tool", "content": ""},
|
|
{"role": "user", "content": {"nested": "value"}},
|
|
{"role": "assistant", "content": "z" * 2_000_000},
|
|
]
|
|
compacted = continuity.compact_telegram_history(malformed)
|
|
assert len(compacted) == 2
|
|
assert compacted[0]["content"] == '{"nested": "value"}'
|
|
assert len(compacted[1]["content"]) == continuity.MAX_ITEM_CHARS
|
|
|
|
|
|
def test_context_defensive_normalization_and_replay_edges(continuity, monkeypatch):
|
|
assert continuity._text(["skip", {"type": "text", "text": "keep"}]) == "keep"
|
|
assert continuity._text({"not-json-serializable"}) == "{'not-json-serializable'}"
|
|
normalized = continuity._normalized_item(
|
|
{
|
|
"role": "assistant",
|
|
"tool_calls": [
|
|
None,
|
|
{"id": "call", "function": {"name": "browser", "arguments": {1}}},
|
|
],
|
|
}
|
|
)
|
|
assert normalized and normalized["tool_calls"][0]["id"] == "call"
|
|
assert continuity._fingerprint({"role": "user", "content": {1}}).startswith(
|
|
"{'role':"
|
|
)
|
|
|
|
user = {"role": "user", "content": "same"}
|
|
assistant = {"role": "assistant", "content": "reply"}
|
|
assert continuity._deduplicate(
|
|
[user, assistant, user.copy(), assistant.copy()]
|
|
) == [
|
|
user,
|
|
assistant,
|
|
]
|
|
assert continuity._deduplicate(
|
|
[
|
|
{"role": "tool", "content": "first", "id": "stable"},
|
|
{"role": "tool", "content": "replay", "id": "stable"},
|
|
]
|
|
) == [{"role": "tool", "content": "first", "id": "stable"}]
|
|
|
|
summary = continuity._summary(
|
|
[
|
|
{
|
|
"role": "system",
|
|
"content": continuity.SUMMARY_PREFIX + "\n- existing fact",
|
|
},
|
|
{"role": "tool", "content": "not durable prose"},
|
|
]
|
|
)
|
|
assert summary and "existing fact" in summary["content"]
|
|
monkeypatch.setattr(
|
|
continuity, "MAX_SUMMARY_CHARS", len(continuity.SUMMARY_PREFIX) + 1
|
|
)
|
|
assert (
|
|
continuity._summary([{"role": "user", "content": "too long"}])["content"]
|
|
== continuity.SUMMARY_PREFIX
|
|
)
|
|
|
|
|
|
def test_context_final_serialized_size_guard(continuity, monkeypatch):
|
|
monkeypatch.setattr(continuity, "MAX_HISTORY_ITEMS", 2)
|
|
monkeypatch.setattr(continuity, "MAX_HISTORY_BYTES", 1_000)
|
|
monkeypatch.setattr(
|
|
continuity,
|
|
"_summary",
|
|
lambda _older: {
|
|
"role": "system",
|
|
"content": "s" * 20_000,
|
|
"_db_persisted": True,
|
|
},
|
|
)
|
|
compacted = continuity.compact_telegram_history(
|
|
[
|
|
{"role": "user", "content": "older"},
|
|
{"role": "assistant", "content": "recent one"},
|
|
{"role": "assistant", "content": "recent two"},
|
|
]
|
|
)
|
|
assert [item["content"] for item in compacted[1:]] == ["recent two"]
|
|
|
|
|
|
def _create_session_databases(tmp_path: Path):
|
|
state = tmp_path / "state.db"
|
|
responses = tmp_path / "response_store.db"
|
|
with sqlite3.connect(state) as connection:
|
|
connection.execute(
|
|
"""CREATE TABLE sessions (
|
|
id TEXT PRIMARY KEY, source TEXT, session_key TEXT, chat_type TEXT,
|
|
display_name TEXT, origin_json TEXT, title TEXT
|
|
)"""
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO sessions VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
(
|
|
(
|
|
"legacy",
|
|
"api_server",
|
|
"wrong",
|
|
"group",
|
|
"Unassigned",
|
|
"{}",
|
|
"Unassigned",
|
|
),
|
|
("named", "api_server", None, None, None, None, "Keep this title"),
|
|
("other", "cli", None, None, "Unassigned", None, None),
|
|
),
|
|
)
|
|
connection.execute(
|
|
"CREATE TABLE messages (id INTEGER PRIMARY KEY, session_id TEXT, role TEXT, content TEXT, timestamp REAL)"
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO messages VALUES (?, 'legacy', 'user', 'same', 1)", ((1,), (2,))
|
|
)
|
|
with sqlite3.connect(responses) as connection:
|
|
connection.execute(
|
|
"CREATE TABLE conversations (name TEXT PRIMARY KEY, response_id TEXT NOT NULL)"
|
|
)
|
|
connection.execute(
|
|
"CREATE TABLE responses (response_id TEXT PRIMARY KEY, data TEXT NOT NULL, accessed_at REAL NOT NULL)"
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO responses VALUES (?, ?, 0)",
|
|
(
|
|
("general", json.dumps({"session_id": "legacy"})),
|
|
("topic", json.dumps({"session_id": "named"})),
|
|
("bad-json", "{"),
|
|
("bad-id", json.dumps({"session_id": "../unsafe"})),
|
|
(
|
|
"oversized",
|
|
json.dumps({"session_id": "other", "padding": "x" * 500}),
|
|
),
|
|
),
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO conversations VALUES (?, ?)",
|
|
(
|
|
("telegram", "general"),
|
|
("telegram-topic-plans-g1", "topic"),
|
|
("telegram-topic-malformed", "bad-json"),
|
|
("telegram-topic-unsafe", "bad-id"),
|
|
("telegram-topic-oversized", "oversized"),
|
|
),
|
|
)
|
|
return state, responses
|
|
|
|
|
|
def test_existing_session_migration_repairs_labels_and_is_idempotent(
|
|
tmp_path: Path, continuity, monkeypatch
|
|
):
|
|
state, responses = _create_session_databases(tmp_path)
|
|
monkeypatch.setattr(continuity, "MAX_RESPONSE_BYTES", 300)
|
|
|
|
assert continuity.migrate(state, responses) == 2
|
|
assert continuity.migrate(state, responses) == 0
|
|
|
|
with sqlite3.connect(state) as connection:
|
|
rows = {
|
|
row[0]: row[1:]
|
|
for row in connection.execute(
|
|
"SELECT id, session_key, chat_type, display_name, origin_json, title "
|
|
"FROM sessions ORDER BY id"
|
|
)
|
|
}
|
|
assert rows["legacy"][:3] == ("telegram", "private", "Telegram")
|
|
assert json.loads(rows["legacy"][3]) == {
|
|
"platform": "telegram",
|
|
"session_key": "telegram",
|
|
}
|
|
assert rows["legacy"][4] == "Telegram · General"
|
|
assert rows["named"][:3] == (
|
|
"telegram-topic-plans-g1",
|
|
"private",
|
|
"Telegram",
|
|
)
|
|
assert rows["named"][4] == "Keep this title"
|
|
assert rows["other"] == (None, None, "Unassigned", None, None)
|
|
with sqlite3.connect(state) as connection:
|
|
assert connection.execute("SELECT COUNT(*) FROM messages").fetchone() == (1,)
|
|
|
|
|
|
def test_migration_fails_closed_on_missing_or_malformed_databases(
|
|
tmp_path: Path, continuity
|
|
):
|
|
assert continuity.telegram_sessions(tmp_path / "missing.db") == {}
|
|
assert (
|
|
continuity.migrate(tmp_path / "missing-state.db", tmp_path / "missing.db") == 0
|
|
)
|
|
broken = tmp_path / "broken.db"
|
|
broken.write_text("not sqlite", encoding="utf-8")
|
|
assert continuity.telegram_sessions(broken) == {}
|
|
|
|
state = tmp_path / "state.db"
|
|
responses = tmp_path / "responses.db"
|
|
with sqlite3.connect(state) as connection:
|
|
connection.execute("CREATE TABLE sessions (id TEXT PRIMARY KEY)")
|
|
with sqlite3.connect(responses) as connection:
|
|
connection.execute("CREATE TABLE conversations (name TEXT, response_id TEXT)")
|
|
connection.execute("CREATE TABLE responses (response_id TEXT, data TEXT)")
|
|
connection.execute(
|
|
"INSERT INTO responses VALUES ('valid', '{\"session_id\": \"known\"}')"
|
|
)
|
|
connection.execute("INSERT INTO conversations VALUES ('telegram', 'valid')")
|
|
assert continuity.migrate(state, responses) == 0
|
|
|
|
|
|
def test_migration_skips_invalid_conversation_and_rolls_back_database_errors(
|
|
tmp_path: Path, continuity
|
|
):
|
|
state = tmp_path / "state.db"
|
|
responses = tmp_path / "responses.db"
|
|
with sqlite3.connect(state) as connection:
|
|
connection.execute(
|
|
"""CREATE TABLE sessions (
|
|
id TEXT PRIMARY KEY, source TEXT, session_key TEXT, chat_type TEXT,
|
|
display_name TEXT, origin_json TEXT, title TEXT
|
|
)"""
|
|
)
|
|
connection.execute(
|
|
"INSERT INTO sessions VALUES ('known', 'api_server', NULL, NULL, NULL, NULL, NULL)"
|
|
)
|
|
connection.execute(
|
|
"""CREATE TRIGGER reject_updates BEFORE UPDATE ON sessions
|
|
BEGIN SELECT RAISE(FAIL, 'read only for test'); END"""
|
|
)
|
|
with sqlite3.connect(responses) as connection:
|
|
connection.execute(
|
|
"CREATE TABLE conversations (name TEXT PRIMARY KEY, response_id TEXT NOT NULL)"
|
|
)
|
|
connection.execute(
|
|
"CREATE TABLE responses (response_id TEXT PRIMARY KEY, data TEXT NOT NULL)"
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO responses VALUES (?, ?)",
|
|
(
|
|
("valid", json.dumps({"session_id": "known"})),
|
|
("invalid", json.dumps({"session_id": "ignored"})),
|
|
),
|
|
)
|
|
connection.executemany(
|
|
"INSERT INTO conversations VALUES (?, ?)",
|
|
(("telegram", "valid"), ("telegram-topic-BAD", "invalid")),
|
|
)
|
|
|
|
assert continuity.telegram_sessions(responses) == {"known": "telegram"}
|
|
assert continuity.migrate(state, responses) == 0
|
|
with sqlite3.connect(state) as connection:
|
|
assert connection.execute(
|
|
"SELECT session_key FROM sessions WHERE id = 'known'"
|
|
).fetchone() == (None,)
|
|
|
|
|
|
def test_api_patch_compacts_input_batch_and_disconnect_snapshots(tmp_path: Path):
|
|
patcher = _load(
|
|
"chat_api_patch",
|
|
HERMES / "scripts" / "patch_api_server_sessions.py",
|
|
)
|
|
source = tmp_path / "api_server.py"
|
|
destination = tmp_path / "patched.py"
|
|
source.write_text(
|
|
patcher.BEFORE
|
|
+ patcher.RUNS_BEFORE
|
|
+ patcher.RUN_CLOSE_BEFORE
|
|
+ patcher.TRUNCATION_BEFORE
|
|
+ patcher.RESPONSES_SESSION_BEFORE
|
|
+ patcher.EVENT_CALLBACK_SIGNATURE_BEFORE
|
|
+ patcher.EVENT_CALLBACK_BODY_BEFORE
|
|
+ patcher.EVENT_CALLBACK_END_BEFORE
|
|
+ patcher.EVENT_CALLBACK_CALL_BEFORE
|
|
+ patcher.RUN_SWEEP_BEFORE
|
|
+ patcher.SSE_STORE_BEFORE
|
|
+ patcher.HISTORY_STORE_BEFORE,
|
|
encoding="utf-8",
|
|
)
|
|
|
|
patcher.patch(source, destination)
|
|
patched = destination.read_text(encoding="utf-8")
|
|
|
|
assert (
|
|
"conversation_history = compact_telegram_history(conversation_history)"
|
|
in patched
|
|
)
|
|
assert "conversation_history_snapshot = compact_telegram_history(" in patched
|
|
assert "full_history = compact_telegram_history(full_history)" in patched
|
|
assert patched.count("from gateway.platforms.telegram_continuity import") == 2
|
|
|
|
|
|
def _patch_source(patcher) -> str:
|
|
return (
|
|
patcher.BEFORE
|
|
+ patcher.RUNS_BEFORE
|
|
+ patcher.RUN_CLOSE_BEFORE
|
|
+ patcher.RESPONSES_SESSION_BEFORE
|
|
+ patcher.EVENT_CALLBACK_SIGNATURE_BEFORE
|
|
+ patcher.EVENT_CALLBACK_BODY_BEFORE
|
|
+ patcher.EVENT_CALLBACK_END_BEFORE
|
|
+ patcher.EVENT_CALLBACK_CALL_BEFORE
|
|
+ patcher.RUN_SWEEP_BEFORE
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"marker_name",
|
|
[
|
|
"BEFORE",
|
|
"RUNS_BEFORE",
|
|
"RUN_CLOSE_BEFORE",
|
|
"RESPONSES_SESSION_BEFORE",
|
|
"EVENT_CALLBACK_SIGNATURE_BEFORE",
|
|
],
|
|
)
|
|
def test_api_patch_fails_closed_on_upstream_drift(tmp_path: Path, marker_name: str):
|
|
patcher = _load(
|
|
"chat_api_patch_drift",
|
|
HERMES / "scripts" / "patch_api_server_sessions.py",
|
|
)
|
|
source = tmp_path / "api_server.py"
|
|
marker = getattr(patcher, marker_name)
|
|
source.write_text(_patch_source(patcher).replace(marker, "", 1), encoding="utf-8")
|
|
with pytest.raises(RuntimeError, match="patch context changed"):
|
|
patcher.patch(source, tmp_path / "patched.py")
|
|
|
|
|
|
def test_script_entrypoints(tmp_path: Path, monkeypatch):
|
|
continuity_path = HERMES / "scripts" / "migrate_telegram_api_sessions.py"
|
|
monkeypatch.setattr(
|
|
sys,
|
|
"argv",
|
|
[
|
|
str(continuity_path),
|
|
str(tmp_path / "state.db"),
|
|
str(tmp_path / "responses.db"),
|
|
],
|
|
)
|
|
with pytest.raises(SystemExit, match="0"):
|
|
runpy.run_path(str(continuity_path), run_name="__main__")
|
|
|
|
patch_path = HERMES / "scripts" / "patch_api_server_sessions.py"
|
|
patcher = _load("chat_api_patch_cli", patch_path)
|
|
source = tmp_path / "api_server.py"
|
|
destination = tmp_path / "patched.py"
|
|
source.write_text(_patch_source(patcher), encoding="utf-8")
|
|
monkeypatch.setattr(sys, "argv", [str(patch_path), str(source), str(destination)])
|
|
with pytest.raises(SystemExit, match="0"):
|
|
runpy.run_path(str(patch_path), run_name="__main__")
|
|
assert destination.exists()
|
|
|
|
|
|
def test_chat_mounts_continuity_only_into_isolated_tenants():
|
|
statefulset = yaml.safe_load((HERMES / "chat-statefulset.yaml").read_text())
|
|
pod = statefulset["spec"]["template"]["spec"]
|
|
hermes = next(item for item in pod["containers"] if item["name"] == "hermes")
|
|
assert {
|
|
"name": "coordinator",
|
|
"mountPath": "/opt/hermes/gateway/platforms/telegram_continuity.py",
|
|
"subPath": "migrate_telegram_api_sessions.py",
|
|
"readOnly": True,
|
|
} in hermes["volumeMounts"]
|
|
patch_init = next(
|
|
item
|
|
for item in pod["initContainers"]
|
|
if item["name"] == "patch-api-server-sessions"
|
|
)
|
|
command = patch_init["args"][0]
|
|
assert command.count("grep -Fq") == 3
|
|
assert "compact_telegram_history" in command
|
|
assert "agent-deployment" not in (HERMES / "chat-statefulset.yaml").read_text()
|