feat(database): store gameplay replays in compact form

- Add compact storage package: codec, records, hydrate, replay blobs, cache.
- Running games write game_turns and snake_states keyed by integer ids.
- Encode moves as 1-byte codes and bodies as head plus step codes.
- Store food and hazards only when they changed since the previous turn.
- Strip reasoning fields that duplicate stored values and restore on read.
- Pack finished games into one zstd blob per game in game_replays.
- Merge late turns into packed blobs during the stale-game sweep.
- Extract SQLite and PostgreSQL schemas into dedicated modules.
- Add read/import game bundle and list_game_ids to the backend template.
- Add legacy readers and converter for the old turns/snake_turns layout.
- Add migrate_gameplay_to_compact.py with in-place or cross-DB migration.
- Verify every migrated game and keep legacy rows on failure.
- Support --since, --minimum-quality and guarded --drop-legacy.
- Add just recipe migrate-gameplay-postgres.
- Rewrite merge script to copy game by game via backend bundles.
- Move benchmark state loading into benchmark_states for both backends.
- Update PostgreSQL cleanup script to the new replay tables.
- Drop sqlite-zstd build stages from the Dockerfile.
- Remove old migration scripts, normalized_turn and their tests.
- Add tests for compact storage, schema, legacy migration and PostgreSQL.
- Document the compact layout and migration workflow in the README.

tests/test_LegacyMigration.py

@@ -0,0 +1,176 @@
+import asyncio
+import json
+import sqlite3
+import subprocess
+import sys
+import tempfile
+import unittest
+from contextlib import closing
+from pathlib import Path
+
+from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend
+from server.database.legacy.convert import LegacyDataError, build_legacy_bundle
+from server.database.legacy.migrate import migrate_legacy
+from server.database.legacy.reader_sqlite import LegacySqliteReader
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+
+LEGACY_SCHEMA = """
+CREATE TABLE games (
+  game_id TEXT PRIMARY KEY, started_at TEXT NOT NULL, ended_at TEXT,
+  width INTEGER, height INTEGER, source TEXT, map_name TEXT,
+  ruleset_name TEXT, ruleset_version TEXT, your_snake_id TEXT,
+  your_snake_name TEXT, your_snake_type TEXT, your_snake_version TEXT,
+  game_type TEXT, winner_name TEXT, winner_you INTEGER NOT NULL DEFAULT 0,
+  final_turn INTEGER NOT NULL DEFAULT 0, status TEXT NOT NULL DEFAULT 'running'
+);
+CREATE TABLE game_snakes (
+  game_id TEXT NOT NULL, snake_id TEXT NOT NULL, snake_name TEXT,
+  is_you INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (game_id, snake_id)
+);
+CREATE TABLE turns (
+  id INTEGER PRIMARY KEY AUTOINCREMENT, game_id TEXT NOT NULL, turn INTEGER NOT NULL,
+  observed_at TEXT NOT NULL, my_move TEXT, my_thinking_json TEXT,
+  board_state_json TEXT NOT NULL, snakes_json TEXT NOT NULL, you_json TEXT NOT NULL,
+  food_json TEXT NOT NULL, hazards_json TEXT NOT NULL, UNIQUE (game_id, turn)
+);
+CREATE TABLE snake_turns (
+  id INTEGER PRIMARY KEY AUTOINCREMENT, game_id TEXT NOT NULL, turn INTEGER NOT NULL,
+  snake_id TEXT NOT NULL, snake_name TEXT, health INTEGER, length INTEGER,
+  head_x INTEGER, head_y INTEGER, body_json TEXT NOT NULL, is_you INTEGER NOT NULL DEFAULT 0,
+  inferred_move TEXT, latency TEXT, UNIQUE (game_id, turn, snake_id)
+);
+"""
+
+def _body(*points):
+  return [{"x": x, "y": y} for x, y in points]
+
+THINKING = {"turn": 1, "health": 99, "length": 3, "food_count": 1, "board_size": "11x11", "scores": {"up": 2.5}}
+
+def create_legacy_database(path:Path, game_snakes:bool=True) -> None:
+  with closing(sqlite3.connect(path)) as connection, connection:
+    connection.executescript(LEGACY_SCHEMA)
+    connection.execute("""
+      INSERT INTO games (game_id, started_at, ended_at, width, height, source, map_name,
+        ruleset_name, ruleset_version, your_snake_id, your_snake_name, game_type,
+        winner_name, winner_you, final_turn, status)
+      VALUES ('g1', '2026-08-01T10:00:00+00:00', '2026-08-01T10:01:00+00:00', 11, 11,
+        'league', 'standard', 'standard', 'v1', 'me', 'Me', 'duel', 'Me', 1, 2, 'finished')
+    """)
+    if game_snakes:
+      connection.executemany(
+        "INSERT INTO game_snakes VALUES (?, ?, ?, ?)",
+        [("g1", "me", "Me", 1), ("g1", "enemy", "Enemy", 0)],
+      )
+    food = json.dumps(_body((5, 5)))
+    for turn, move, thinking in ((0, None, None), (1, "up", THINKING), (2, "left", {"scores": {"left": 1}})):
+      connection.execute("""
+        INSERT INTO turns (game_id, turn, observed_at, my_move, my_thinking_json,
+          board_state_json, snakes_json, you_json, food_json, hazards_json)
+        VALUES ('g1', ?, ?, ?, ?, '{}', '[]', '{}', ?, '[]')
+      """, (turn, f"2026-08-01T10:00:0{turn}.123456+00:00", move, json.dumps(thinking) if thinking else None, food))
+    me = [_body((1, 1), (1, 1), (1, 1)), _body((1, 2), (1, 1), (1, 1)), _body((0, 2), (1, 2), (1, 1))]
+    enemy = [_body((9, 9), (9, 9), (9, 9)), _body((9, 8), (9, 9), (9, 9)), _body((9, 7), (9, 8), (9, 9))]
+    for turn in range(3):
+      for snake_id, bodies, latency in (("me", me, "12"), ("enemy", enemy, "")):
+        body = bodies[turn]
+        connection.execute("""
+          INSERT INTO snake_turns (game_id, turn, snake_id, snake_name, health, length,
+            head_x, head_y, body_json, is_you, latency)
+          VALUES ('g1', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        """, (
+          turn, snake_id, None if game_snakes else snake_id.title(), 100 - turn, len(body),
+          body[0]["x"], body[0]["y"], json.dumps(body), int(snake_id == "me" and not game_snakes), latency,
+        ))
+
+class TestLegacyConvert(unittest.TestCase):
+  def test_rejects_inconsistent_rows_instead_of_losing_data(self):
+    with self.assertRaises(LegacyDataError):
+      build_legacy_bundle(
+        {"game_id": "g", "started_at": "2026-01-01"}, [], [],
+        [{"snake_id": "s", "turn": 0, "length": 5, "body": json.dumps(_body((1, 1)))}],
+      )
+
+  def test_invalid_json_aborts(self):
+    with self.assertRaises(LegacyDataError):
+      build_legacy_bundle(
+        {"game_id": "g", "started_at": "2026-01-01"}, [],
+        [{"turn": 0, "observed_at": "2026-01-01", "my_thinking": "{invalid"}], [],
+      )
+
+class TestLegacyMigration(unittest.TestCase):
+  def _migrate(self, source:Path, destination:Path, **options):
+    async def run():
+      backend = SqliteGameplayBackend(str(destination))
+      with LegacySqliteReader(source) as reader:
+        stats = await migrate_legacy(reader, backend, **options)
+      return stats, await backend.get_game_replay("g1")
+    return asyncio.run(run())
+
+  def test_migrates_finished_game_into_identical_replay(self):
+    with tempfile.TemporaryDirectory() as temp_dir:
+      source, destination = Path(temp_dir) / "legacy.sqlite3", Path(temp_dir) / "compact.sqlite3"
+      create_legacy_database(source)
+
+      stats, replay = self._migrate(source, destination)
+
+      self.assertEqual((stats.games, stats.turns, stats.states, stats.failed), (1, 3, 6, 0))
+      turns = replay["turns"]
+      self.assertEqual([turn["my_move"] for turn in turns], [None, "up", "left"])
+      self.assertEqual(turns[1]["my_thinking"], THINKING)
+      self.assertEqual(turns[2]["food"], _body((5, 5)))
+      self.assertEqual(turns[1]["observed_at"], "2026-08-01T10:00:01.123456+00:00")
+      me = turns[2]["snakes"][0]
+      self.assertEqual((me["snake_id"], me["snake_name"], me["is_you"]), ("me", "Me", True))
+      self.assertEqual((me["inferred_move"], me["latency"], me["health"], me["length"]), ("left", "12", 98, 3))
+      enemy = turns[2]["snakes"][1]
+      self.assertEqual((enemy["inferred_move"], enemy["latency"]), ("down", None))
+      with closing(sqlite3.connect(destination)) as connection:
+        self.assertEqual(connection.execute("SELECT turn_count FROM game_replays").fetchone()[0], 3)
+
+  def test_synthesizes_missing_game_snakes_from_snake_rows(self):
+    with tempfile.TemporaryDirectory() as temp_dir:
+      source, destination = Path(temp_dir) / "legacy.sqlite3", Path(temp_dir) / "compact.sqlite3"
+      create_legacy_database(source, game_snakes=False)
+
+      _, replay = self._migrate(source, destination)
+
+      snakes = replay["turns"][0]["snakes"]
+      self.assertEqual([(s["snake_id"], s["snake_name"], s["is_you"]) for s in snakes], [
+        ("me", "Me", True), ("enemy", "Enemy", False),
+      ])
+      self.assertEqual(snakes[0]["customizations"], {})
+
+  def test_quality_filter_keeps_result_but_drops_replay(self):
+    with tempfile.TemporaryDirectory() as temp_dir:
+      source, destination = Path(temp_dir) / "legacy.sqlite3", Path(temp_dir) / "compact.sqlite3"
+      create_legacy_database(source)
+
+      stats, replay = self._migrate(source, destination, minimum_quality="high")
+
+      self.assertEqual(stats.dropped_replays, 1)
+      self.assertIsNone(replay)
+      with closing(sqlite3.connect(destination)) as connection:
+        row = connection.execute("SELECT has_replay, quality_status, winner_you FROM games").fetchone()
+        self.assertEqual(row, (0, "low_quality", 1))
+        self.assertEqual(connection.execute("SELECT COUNT(*) FROM game_replays").fetchone()[0], 0)
+
+  def test_script_migrates_in_place_and_drops_legacy_tables(self):
+    with tempfile.TemporaryDirectory() as temp_dir:
+      database = Path(temp_dir) / "gameplay.sqlite3"
+      create_legacy_database(database)
+      script = [sys.executable, "scripts/migrate_gameplay_to_compact.py", "--source", str(database)]
+
+      migrated = subprocess.run(script, cwd=REPO_ROOT, text=True, capture_output=True)
+      dropped = subprocess.run(script + ["--drop-legacy"], cwd=REPO_ROOT, text=True, capture_output=True)
+
+      self.assertEqual(migrated.returncode, 0, migrated.stderr)
+      self.assertEqual(dropped.returncode, 0, dropped.stderr)
+      with closing(sqlite3.connect(database)) as connection:
+        tables = {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type = 'table'")}
+      self.assertFalse({"turns", "snake_turns"} & tables)
+      replay = asyncio.run(SqliteGameplayBackend(str(database)).get_game_replay("g1"))
+      self.assertEqual(len(replay["turns"]), 3)
+
+if __name__ == "__main__":
+  unittest.main()