feat(database): store gameplay replays in compact form

- Add compact storage package: codec, records, hydrate, replay blobs, cache.
- Running games write game_turns and snake_states keyed by integer ids.
- Encode moves as 1-byte codes and bodies as head plus step codes.
- Store food and hazards only when they changed since the previous turn.
- Strip reasoning fields that duplicate stored values and restore on read.
- Pack finished games into one zstd blob per game in game_replays.
- Merge late turns into packed blobs during the stale-game sweep.
- Extract SQLite and PostgreSQL schemas into dedicated modules.
- Add read/import game bundle and list_game_ids to the backend template.
- Add legacy readers and converter for the old turns/snake_turns layout.
- Add migrate_gameplay_to_compact.py with in-place or cross-DB migration.
- Verify every migrated game and keep legacy rows on failure.
- Support --since, --minimum-quality and guarded --drop-legacy.
- Add just recipe migrate-gameplay-postgres.
- Rewrite merge script to copy game by game via backend bundles.
- Move benchmark state loading into benchmark_states for both backends.
- Update PostgreSQL cleanup script to the new replay tables.
- Drop sqlite-zstd build stages from the Dockerfile.
- Remove old migration scripts, normalized_turn and their tests.
- Add tests for compact storage, schema, legacy migration and PostgreSQL.
- Document the compact layout and migration workflow in the README.

tests/test_BenchmarkStates.py

@@ -1,6 +1,21 @@
+import asyncio
 import unittest
+from unittest.mock import AsyncMock
 
-from server.database.benchmark_states import _build_state, is_postgresql_source
+from server.database.benchmark_states import is_postgresql_source, replay_turn_to_state, sample_states
+
+GAME = {
+  "game_id": "game-id", "source": "league", "map": "standard",
+  "ruleset_name": "standard", "ruleset_version": "v1",
+}
+
+def _turn(number:int) -> dict:
+  you = {"id": "you-id", "head": {"x": 1, "y": 2}, "body": [{"x": 1, "y": 2}]}
+  return {
+    "turn": number,
+    "board": {"width": 11, "height": 11, "food": [{"x": 5, "y": 5}], "hazards": [], "snakes": [you]},
+    "you": you,
+  }
 
 class TestBenchmarkStates(unittest.TestCase):
   def test_detects_postgresql_dsn(self):
@@ -8,23 +23,27 @@ class TestBenchmarkStates(unittest.TestCase):
     self.assertTrue(is_postgresql_source("postgres://user:pass@example.com/db"))
     self.assertFalse(is_postgresql_source("/tmp/gameplay.sqlite3"))
 
-  def test_builds_normalized_state_from_postgresql_json_values(self):
-    row = (
-      1, {}, {}, [{"x": 5, "y": 5}], [], "you-id", "You", 11, 11,
-      "game-id", "league", "standard", "standard", "v1", 7,
-    )
-    snake_rows = [
-      ("you-id", "You", 90, 3, 1, 2, [{"x": 1, "y": 2}], {"head": "default"}),
-    ]
-
-    state = _build_state(row, snake_rows)
-
-    self.assertIsNotNone(state)
-    board, metadata = state
+  def test_builds_state_from_replay_turn(self):
+    board, metadata = replay_turn_to_state(GAME, _turn(7))
     self.assertEqual(board["food"], [{"x": 5, "y": 5}])
-    self.assertEqual(board["snakes"][0]["id"], "you-id")
     self.assertEqual(metadata["you"]["id"], "you-id")
     self.assertEqual(metadata["turn"], 7)
+    self.assertEqual(metadata["ruleset"]["version"], "v1")
+
+  def test_turn_without_snakes_is_skipped(self):
+    self.assertIsNone(replay_turn_to_state(GAME, {"turn": 0, "board": {"snakes": []}, "you": {}}))
+
+  def test_samples_every_stride_turn_from_newest_games(self):
+    backend = AsyncMock()
+    backend.list_game_ids.return_value = ["old", "new"]
+    backend.get_game_replay.side_effect = lambda game_id: {
+      "game": {**GAME, "game_id": game_id},
+      "turns": [_turn(number) for number in range(4)],
+    }
+
+    states = asyncio.run(sample_states(backend, samples=3, stride=3))
+
+    self.assertEqual([(m["game_id"], m["turn"]) for _, m in states], [("new", 3), ("new", 0), ("old", 1)])
 
 if __name__ == "__main__":
   unittest.main()

tests/test_CompactStorage.py

@@ -0,0 +1,124 @@
+import unittest
+from datetime import datetime, timezone
+
+from server.database.compact.codec import (
+  decode_body, decode_latency, decode_move, decode_points, encode_body,
+  encode_latency, encode_move, encode_points, from_epoch_us, to_epoch_us,
+)
+from server.database.compact.records import PointsState, SnakeState, TurnRecord, compact_turn
+from server.database.compact.replay_blob import pack_replay, unpack_replay
+from server.database.compact.thinking import derived_values, restore_thinking, slim_thinking
+
+def _body(*points):
+  return [{"x": x, "y": y} for x, y in points]
+
+class TestCodec(unittest.TestCase):
+  def test_body_roundtrip_including_stacked_tail(self):
+    body = _body((5, 5), (5, 6), (4, 6), (4, 6))
+    encoded = encode_body(body)
+    self.assertEqual(decode_body(encoded), body)
+    # 1 format byte + head pair + 3 step codes
+    self.assertEqual(len(encoded), 6)
+
+  def test_body_roundtrip_with_wrapped_jump(self):
+    body = _body((0, 5), (10, 5), (9, 5))
+    self.assertEqual(decode_body(encode_body(body)), body)
+
+  def test_body_out_of_byte_range_falls_back_to_json(self):
+    body = _body((-1, 3), (300, 3))
+    self.assertEqual(decode_body(encode_body(body)), body)
+
+  def test_empty_body(self):
+    self.assertEqual(decode_body(encode_body([])), [])
+
+  def test_points_keep_none_as_unchanged(self):
+    self.assertIsNone(encode_points(None))
+    self.assertIsNone(decode_points(None))
+    self.assertEqual(decode_points(encode_points([])), [])
+    food = _body((1, 2), (10, 0))
+    self.assertEqual(decode_points(encode_points(food)), food)
+
+  def test_moves_and_latency(self):
+    for move in ("up", "down", "left", "right", None):
+      self.assertEqual(decode_move(encode_move(move)), move)
+    self.assertEqual(decode_latency(encode_latency("77")), "77")
+    self.assertIsNone(encode_latency(""))
+    self.assertIsNone(encode_latency("timeout"))
+
+  def test_epoch_microseconds_roundtrip(self):
+    value = datetime(2026, 11, 4, 21, 41, 48, 123456, tzinfo=timezone.utc)
+    self.assertEqual(from_epoch_us(to_epoch_us(value)), value)
+    self.assertEqual(from_epoch_us(to_epoch_us(value.isoformat())), value)
+
+class TestThinking(unittest.TestCase):
+  def setUp(self):
+    self.derived = derived_values(
+      turn=12, move="up", you={"head": {"x": 1, "y": 2}, "health": 90, "length": 4},
+      food=[{"x": 0, "y": 0}], width=11, height=11, snake_count=2,
+    )
+
+  def test_strips_and_restores_derived_fields(self):
+    thinking = {
+      "turn": 12, "move": "up", "head": {"x": 1, "y": 2}, "health": 90,
+      "length": 4, "food_count": 1, "board_size": "11x11", "snakes": 2,
+      "scores": {"up": 1.5}, "mode": "duel",
+    }
+    slim = slim_thinking(thinking, self.derived)
+    self.assertEqual(set(slim), {"scores", "mode", "_derived"})
+    self.assertEqual(restore_thinking(slim, self.derived), thinking)
+
+  def test_keeps_values_that_differ_from_derived(self):
+    thinking = {"turn": 11, "health": 89, "snakes": True}
+    slim = slim_thinking(thinking, self.derived)
+    self.assertEqual(slim, thinking)
+    self.assertEqual(restore_thinking(slim, self.derived), thinking)
+
+  def test_payload_using_marker_key_is_stored_verbatim(self):
+    thinking = {"_derived": "mine", "turn": 12}
+    self.assertEqual(restore_thinking(slim_thinking(thinking, self.derived), self.derived), thinking)
+
+  def test_non_dict_payload_passes_through(self):
+    self.assertIsNone(slim_thinking(None, self.derived))
+    self.assertEqual(slim_thinking(["a"], self.derived), ["a"])
+
+class TestCompactTurn(unittest.TestCase):
+  def _compact(self, turn, food, previous):
+    states = [SnakeState("me", turn, 90, 5, _body((1, 1)))]
+    return compact_turn(turn, None, "up", None, food, [], states, "me", 11, 11, previous)
+
+  def test_unchanged_food_is_stored_as_none_only_after_consecutive_turn(self):
+    food = _body((3, 3))
+    first, points = self._compact(1, food, None)
+    second, points = self._compact(2, list(food), points)
+    after_gap, _ = self._compact(4, list(food), points)
+    self.assertEqual(first.food, food)
+    self.assertIsNone(second.food)
+    self.assertEqual(second.hazards, None)
+    self.assertEqual(after_gap.food, food)
+
+  def test_previous_state_from_other_turn_is_not_used(self):
+    record, _ = self._compact(5, [], PointsState(5, [], []))
+    self.assertEqual(record.food, [])
+
+class TestReplayBlob(unittest.TestCase):
+  def test_pack_roundtrip(self):
+    observed = datetime(2026, 11, 4, 12, 0, 0, 5, tzinfo=timezone.utc)
+    turns = [
+      TurnRecord(0, observed, None, None, _body((1, 1)), []),
+      TurnRecord(1, observed, "left", {"scores": {"left": 1.0}, "_derived": 3}, None, None),
+    ]
+    states = [
+      SnakeState("me", 0, 100, None, _body((5, 5), (5, 5), (5, 5))),
+      SnakeState("me", 1, 99, 42, _body((4, 5), (5, 5), (5, 5))),
+      SnakeState("enemy", 1, 99, 7, _body((-1, 2))),
+    ]
+    unpacked_turns, unpacked_states = unpack_replay(pack_replay(turns, states))
+    self.assertEqual(unpacked_turns, turns)
+    self.assertEqual(sorted(unpacked_states, key=lambda s: (s.snake_id, s.turn)), sorted(states, key=lambda s: (s.snake_id, s.turn)))
+
+  def test_unknown_format_is_rejected(self):
+    with self.assertRaises(ValueError):
+      unpack_replay(b"", 99)
+
+if __name__ == "__main__":
+  unittest.main()

tests/test_GameplayDatabase.py

@@ -2,6 +2,7 @@ import unittest
 
 from pathlib import Path
 import tempfile, sqlite3
+from contextlib import closing
 
 from server.database import GameplayDatabase, GameplayBackendBuilder
 
@@ -59,93 +60,127 @@ class TestGameplayDatabase(unittest.IsolatedAsyncioTestCase):
       "you": snakes[0],
     }
 
-  async def test_records_gameplay_with_wal_and_inferred_moves(self):
+  async def _play_game(self, database) -> None:
+    await database.record_game_start(self._build_state(turn=0, me_head=(1, 1), enemy_head=(5, 5)))
+    await database.record_turn(
+      self._build_state(turn=1, me_head=(2, 1), enemy_head=(5, 4)),
+      my_move="right",
+      my_thinking={
+        "turn": 1,
+        "reason": "safe_space",
+        "health": 90,
+        "length": 3,
+        "scores": {"right": 1.0},
+      },
+    )
+    await database.record_turn(
+      self._build_state(turn=2, me_head=(2, 2), enemy_head=(4, 4)),
+      my_move="up",
+      my_thinking={"turn": 2, "reason": "food", "scores": {"up": 1.4}},
+    )
+
+  async def test_running_game_uses_compact_rows(self):
     with tempfile.TemporaryDirectory() as temp_dir:
       db_path = Path(temp_dir) / "gameplay.sqlite3"
       database = GameplayDatabase(GameplayBackendBuilder.build(db_path=str(db_path), busy_timeout_ms=4000))
+      await self._play_game(database)
 
-      await database.record_game_start(self._build_state(turn=0, me_head=(1, 1), enemy_head=(5, 5)))
-      await database.record_turn(
-        self._build_state(turn=1, me_head=(2, 1), enemy_head=(5, 4)),
-        my_move="right",
-        my_thinking={
-          "turn": 1,
-          "reason": "safe_space",
-          "scores": {"right": 1.0},
-        },
-      )
-      await database.record_turn(
-        self._build_state(turn=2, me_head=(2, 2), enemy_head=(4, 4)),
-        my_move="up",
-        my_thinking={"turn": 2, "reason": "food", "scores": {"up": 1.4}},
-      )
+      with closing(sqlite3.connect(str(db_path))) as connection:
+        self.assertEqual(str(connection.execute("PRAGMA journal_mode").fetchone()[0]).lower(), "wal")
+        rows = connection.execute("""
+          SELECT t.turn, t.my_move, t.thinking_json, t.food
+          FROM game_turns AS t JOIN games AS g ON g.gid = t.gid
+          WHERE g.game_id = ? ORDER BY t.turn
+        """, ("game-abc",)).fetchall()
+        self.assertEqual([row[0] for row in rows], [1, 2])
+        self.assertEqual([row[1] for row in rows], [4, 1])  # right, up
+        # Derivable thinking fields are stripped; unchanged food is NULL.
+        self.assertNotIn('"turn"', rows[0][2])
+        self.assertNotIn('"health"', rows[0][2])
+        self.assertIsNotNone(rows[0][3])
+        self.assertIsNone(rows[1][3])
+        states = connection.execute("SELECT COUNT(*) FROM snake_states").fetchone()[0]
+        self.assertEqual(states, 4)
+        self.assertEqual(connection.execute("SELECT COUNT(*) FROM game_replays").fetchone()[0], 0)
+
+      replay = await database.get_game_replay("game-abc")
+      self.assertEqual(replay["game"]["status"], "running")
+      self.assertEqual(replay["turns"][1]["food"], [{"x": 2, "y": 2}])
+
+  async def test_finished_game_is_packed_and_replays_identically(self):
+    with tempfile.TemporaryDirectory() as temp_dir:
+      db_path = Path(temp_dir) / "gameplay.sqlite3"
+      database = GameplayDatabase(GameplayBackendBuilder.build(db_path=str(db_path), busy_timeout_ms=4000))
+      await self._play_game(database)
+      running_replay = await database.get_game_replay("game-abc")
       await database.record_game_end(self._build_state(turn=2, me_head=(2, 2), enemy_head=(4, 4), include_enemy=False))
 
-      connection = sqlite3.connect(str(db_path))
-      journal_mode = connection.execute("PRAGMA journal_mode").fetchone()[0]
-      self.assertEqual(str(journal_mode).lower(), "wal")
-
-      games = connection.execute("SELECT status, winner_you, final_turn FROM games WHERE game_id = ?", ("game-abc",)).fetchone()
-      self.assertEqual(games[0], "finished")
-      self.assertEqual(games[1], 1)
-      self.assertEqual(games[2], 2)
-
-      turns_count = connection.execute("SELECT COUNT(*) FROM turns WHERE game_id = ?", ("game-abc",)).fetchone()[0]
-      self.assertEqual(turns_count, 2)
-
-      compact_turn = connection.execute("""
-        SELECT snakes_json, you_json, food_json, hazards_json
-        FROM turns WHERE game_id = ? AND turn = ?
-      """, ("game-abc", 2)).fetchone()
-      self.assertEqual(compact_turn, ("[]", "{}", '[{"x":2,"y":2}]', "[]"))
-      stored_body = connection.execute("""
-        SELECT body_json FROM snake_turns
-        WHERE game_id = ? AND turn = ? AND snake_id = ?
-      """, ("game-abc", 2, "me")).fetchone()[0]
-      self.assertNotEqual(stored_body, "[]")
-      identities = connection.execute("""
-        SELECT snake_id, snake_name, is_you, customizations_json FROM game_snakes
-        WHERE game_id = ? ORDER BY snake_id
-      """, ("game-abc",)).fetchall()
-      self.assertEqual(identities, [
-        ("enemy", "Enemy", 0, "{}"),
-        ("me", "Me", 1, '{"color":"#00ff00","head":"ferret","tail":"swirl"}'),
-      ])
-      repeated_identity = connection.execute("""
-        SELECT snake_name, is_you FROM snake_turns
-        WHERE game_id = ? AND turn = ? AND snake_id = ?
-      """, ("game-abc", 2, "me")).fetchone()
-      self.assertEqual(repeated_identity, (None, 0))
-
-      me_inferred = connection.execute("SELECT inferred_move FROM snake_turns WHERE game_id = ? AND turn = ? AND snake_id = ?", ("game-abc", 2, "me")).fetchone()[0]
-      enemy_inferred = connection.execute("SELECT inferred_move FROM snake_turns WHERE game_id = ? AND turn = ? AND snake_id = ?", ("game-abc", 2, "enemy")).fetchone()[0]
-      self.assertEqual(me_inferred, "up")
-      self.assertEqual(enemy_inferred, "left")
+      with closing(sqlite3.connect(str(db_path))) as connection:
+        games = connection.execute("SELECT status, winner_you, final_turn FROM games WHERE game_id = ?", ("game-abc",)).fetchone()
+        self.assertEqual(games, ("finished", 1, 2))
+        self.assertEqual(connection.execute("SELECT COUNT(*) FROM game_turns").fetchone()[0], 0)
+        self.assertEqual(connection.execute("SELECT COUNT(*) FROM snake_states").fetchone()[0], 0)
+        self.assertEqual(connection.execute("SELECT turn_count FROM game_replays").fetchone()[0], 2)
+        identities = connection.execute("""
+          SELECT snake_id, snake_name, is_you, customizations_json FROM game_snakes
+          WHERE game_id = ? ORDER BY snake_id
+        """, ("game-abc",)).fetchall()
+        self.assertEqual(identities, [
+          ("enemy", "Enemy", 0, "{}"),
+          ("me", "Me", 1, '{"color":"#00ff00","head":"ferret","tail":"swirl"}'),
+        ])
 
       summary = await database.get_summary()
       self.assertEqual(summary["finished_games"], 1)
       self.assertEqual(summary["wins"], 1)
 
       replay = await database.get_game_replay("game-abc")
-      self.assertIsNotNone(replay)
-      replay = replay or {}
+      self.assertEqual(replay["turns"], running_replay["turns"])
       self.assertEqual(replay["game"]["final_turn"], 2)
-      self.assertEqual(len(replay["turns"]), 2)
-      self.assertEqual(replay["turns"][1]["my_move"], "up")
-      self.assertEqual(replay["turns"][1]["my_thinking"]["reason"], "food")
-      self.assertEqual(replay["turns"][1]["food"], [{"x": 2, "y": 2}])
-      self.assertEqual(replay["turns"][1]["you"]["id"], "me")
-      me = next(snake for snake in replay["turns"][1]["snakes"] if snake["snake_id"] == "me")
-      self.assertEqual(me["customizations"], {
-        "color": "#00ff00",
-        "head": "ferret",
-        "tail": "swirl",
-      })
+      turn = replay["turns"][1]
+      self.assertEqual(turn["my_move"], "up")
+      self.assertEqual(turn["my_thinking"], {"turn": 2, "reason": "food", "scores": {"up": 1.4}})
+      self.assertEqual(replay["turns"][0]["my_thinking"]["health"], 90)
+      self.assertEqual(turn["food"], [{"x": 2, "y": 2}])
+      self.assertEqual(turn["you"]["id"], "me")
+      me = next(snake for snake in turn["snakes"] if snake["snake_id"] == "me")
+      enemy = next(snake for snake in turn["snakes"] if snake["snake_id"] == "enemy")
+      self.assertEqual(me["inferred_move"], "up")
+      self.assertEqual(enemy["inferred_move"], "left")
+      self.assertEqual(me["customizations"], {"color": "#00ff00", "head": "ferret", "tail": "swirl"})
       self.assertEqual(len(me["body"]), 3)
-      api_me = next(snake for snake in replay["turns"][1]["board"]["snakes"] if snake["id"] == "me")
+      self.assertEqual(turn["snakes"][0]["snake_id"], "me")
+      api_me = next(snake for snake in turn["board"]["snakes"] if snake["id"] == "me")
       self.assertEqual(api_me["customizations"], me["customizations"])
 
-      connection.close()
+  async def test_turn_after_restart_stores_full_food(self):
+    with tempfile.TemporaryDirectory() as temp_dir:
+      db_path = Path(temp_dir) / "gameplay.sqlite3"
+      first = GameplayDatabase(GameplayBackendBuilder.build(db_path=str(db_path)))
+      await first.record_game_start(self._build_state(turn=0, me_head=(1, 1), enemy_head=(5, 5)))
+      await first.record_turn(self._build_state(turn=1, me_head=(2, 1), enemy_head=(5, 4)), my_move="right")
+      second = GameplayDatabase(GameplayBackendBuilder.build(db_path=str(db_path)))
+      await second.record_turn(self._build_state(turn=2, me_head=(2, 2), enemy_head=(4, 4)), my_move="up")
+
+      replay = await second.get_game_replay("game-abc")
+      self.assertEqual([turn["food"] for turn in replay["turns"]], [[{"x": 2, "y": 2}]] * 2)
+
+  async def test_stale_running_game_is_finalized_and_packed(self):
+    with tempfile.TemporaryDirectory() as temp_dir:
+      db_path = Path(temp_dir) / "gameplay.sqlite3"
+      database = GameplayDatabase(GameplayBackendBuilder.build(db_path=str(db_path)))
+      await self._play_game(database)
+      with closing(sqlite3.connect(str(db_path))) as connection, connection:
+        connection.execute("UPDATE games SET started_at = '2020-01-01T00:00:00+00:00'")
+
+      self.assertEqual(await database.finalize_stale_running_games(600), 1)
+
+      replay = await database.get_game_replay("game-abc")
+      self.assertEqual(replay["game"]["status"], "finished")
+      self.assertFalse(replay["game"]["winner_you"])
+      self.assertEqual(len(replay["turns"]), 2)
+      with closing(sqlite3.connect(str(db_path))) as connection:
+        self.assertEqual(connection.execute("SELECT COUNT(*) FROM game_turns").fetchone()[0], 0)
 
 if __name__ == "__main__":
   unittest.main()

tests/test_GameplaySchema.py

@@ -0,0 +1,120 @@
+import sqlite3
+import tempfile
+import unittest
+from contextlib import closing
+from pathlib import Path
+
+from server.database.backend import postgresql_schema
+from server.database.backend.PostgresqlGameplayBackend import PostgresqlGameplayBackend
+from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend
+
+REDUNDANT = {"idx_turns_game_turn", "idx_snake_turns_game_turn"}
+
+class _RecordingConnection:
+  def __init__(self) -> None:
+    self.statements:list[str] = []
+
+  async def execute(self, statement:str, *args) -> str:
+    self.statements.append(statement)
+    return "OK"
+
+class _AcquireContext:
+  def __init__(self, connection) -> None:
+    self._connection = connection
+
+  async def __aenter__(self):
+    return self._connection
+
+  async def __aexit__(self, *exc) -> None:
+    return None
+
+class _Pool:
+  def __init__(self, connection) -> None:
+    self._connection = connection
+
+  def acquire(self):
+    return _AcquireContext(self._connection)
+
+class TestPostgresqlSchema(unittest.IsolatedAsyncioTestCase):
+  def test_ddl_defines_compact_tables_without_legacy_indexes(self):
+    ddl = postgresql_schema.TABLES_DDL
+    for table in ("game_turns", "snake_states", "game_replays"):
+      self.assertIn(f"CREATE TABLE IF NOT EXISTS {table}", ddl)
+    for name in REDUNDANT:
+      self.assertNotIn(f"CREATE INDEX IF NOT EXISTS {name}", ddl)
+    self.assertIn("SET STORAGE EXTERNAL", ddl)
+
+  async def test_schema_init_is_locked_and_drops_redundant_indexes_separately(self):
+    connection = _RecordingConnection()
+    backend = PostgresqlGameplayBackend("postgresql://example")
+    backend._pool = _Pool(connection)
+
+    await backend._initialize_schema()
+
+    self.assertIn("pg_advisory_lock", connection.statements[0])
+    drops = [s for s in connection.statements if "DROP INDEX" in s]
+    self.assertEqual({s.split()[-1] for s in drops}, REDUNDANT)
+    for statement in drops:
+      # CONCURRENTLY only works as a standalone statement outside a transaction.
+      self.assertIn("CONCURRENTLY IF EXISTS", statement)
+      self.assertNotIn(";", statement)
+
+class TestSqliteSchema(unittest.TestCase):
+  def _objects(self, db_path:Path, kind:str) -> set[str]:
+    with closing(sqlite3.connect(db_path)) as connection:
+      return {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type = ?", (kind,))}
+
+  def test_new_database_has_compact_tables(self):
+    with tempfile.TemporaryDirectory() as temp_dir:
+      db_path = Path(temp_dir) / "gameplay.sqlite3"
+      SqliteGameplayBackend(str(db_path))
+      tables = self._objects(db_path, "table")
+      self.assertTrue({"games", "game_snakes", "game_turns", "snake_states", "game_replays"} <= tables)
+      self.assertFalse({"turns", "snake_turns"} & tables)
+      self.assertFalse(self._objects(db_path, "index") & REDUNDANT)
+
+  def test_legacy_database_gets_ids_and_keeps_legacy_tables(self):
+    with tempfile.TemporaryDirectory() as temp_dir:
+      db_path = Path(temp_dir) / "gameplay.sqlite3"
+      with closing(sqlite3.connect(db_path)) as connection, connection:
+        connection.executescript("""
+          CREATE TABLE games (game_id TEXT PRIMARY KEY, started_at TEXT NOT NULL,
+            winner_you INTEGER NOT NULL DEFAULT 0, final_turn INTEGER NOT NULL DEFAULT 0,
+            status TEXT NOT NULL DEFAULT 'running', width INTEGER, height INTEGER,
+            source TEXT, map_name TEXT, ruleset_name TEXT, ruleset_version TEXT,
+            your_snake_id TEXT, your_snake_name TEXT);
+          CREATE TABLE game_snakes (game_id TEXT NOT NULL, snake_id TEXT NOT NULL,
+            snake_name TEXT, is_you INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (game_id, snake_id));
+          CREATE TABLE turns (game_id TEXT, turn INTEGER);
+          CREATE TABLE snake_turns (game_id TEXT, turn INTEGER, snake_id TEXT);
+          CREATE INDEX idx_turns_game_turn ON turns(game_id, turn);
+          INSERT INTO games (game_id, started_at) VALUES ('a', '2026-01-01'), ('b', '2026-01-02');
+          INSERT INTO game_snakes (game_id, snake_id) VALUES ('a', 's1'), ('a', 's2');
+        """)
+
+      SqliteGameplayBackend(str(db_path))
+
+      with closing(sqlite3.connect(db_path)) as connection:
+        gids = [row[0] for row in connection.execute("SELECT gid FROM games ORDER BY game_id")]
+        gsids = [row[0] for row in connection.execute("SELECT gsid FROM game_snakes ORDER BY snake_id")]
+      self.assertEqual(len(set(gids)), 2)
+      self.assertNotIn(None, gids)
+      self.assertEqual(len(set(gsids)), 2)
+      self.assertNotIn(None, gsids)
+      self.assertTrue({"turns", "snake_turns"} <= self._objects(db_path, "table"))
+      self.assertFalse(self._objects(db_path, "index") & REDUNDANT)
+
+  def test_running_game_lookups_use_primary_keys(self):
+    with tempfile.TemporaryDirectory() as temp_dir:
+      db_path = Path(temp_dir) / "gameplay.sqlite3"
+      SqliteGameplayBackend(str(db_path))
+      with closing(sqlite3.connect(db_path)) as connection:
+        for query in (
+          "SELECT * FROM game_turns WHERE gid = 1 ORDER BY turn",
+          "SELECT * FROM snake_states WHERE gsid = 1 AND turn = 3",
+        ):
+          plan = " ".join(str(row[-1]) for row in connection.execute(f"EXPLAIN QUERY PLAN {query}"))
+          self.assertIn("PRIMARY KEY", plan, plan)
+
+if __name__ == "__main__":
+  unittest.main()

tests/test_LegacyMigration.py

@@ -0,0 +1,176 @@
+import asyncio
+import json
+import sqlite3
+import subprocess
+import sys
+import tempfile
+import unittest
+from contextlib import closing
+from pathlib import Path
+
+from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend
+from server.database.legacy.convert import LegacyDataError, build_legacy_bundle
+from server.database.legacy.migrate import migrate_legacy
+from server.database.legacy.reader_sqlite import LegacySqliteReader
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+
+LEGACY_SCHEMA = """
+CREATE TABLE games (
+  game_id TEXT PRIMARY KEY, started_at TEXT NOT NULL, ended_at TEXT,
+  width INTEGER, height INTEGER, source TEXT, map_name TEXT,
+  ruleset_name TEXT, ruleset_version TEXT, your_snake_id TEXT,
+  your_snake_name TEXT, your_snake_type TEXT, your_snake_version TEXT,
+  game_type TEXT, winner_name TEXT, winner_you INTEGER NOT NULL DEFAULT 0,
+  final_turn INTEGER NOT NULL DEFAULT 0, status TEXT NOT NULL DEFAULT 'running'
+);
+CREATE TABLE game_snakes (
+  game_id TEXT NOT NULL, snake_id TEXT NOT NULL, snake_name TEXT,
+  is_you INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (game_id, snake_id)
+);
+CREATE TABLE turns (
+  id INTEGER PRIMARY KEY AUTOINCREMENT, game_id TEXT NOT NULL, turn INTEGER NOT NULL,
+  observed_at TEXT NOT NULL, my_move TEXT, my_thinking_json TEXT,
+  board_state_json TEXT NOT NULL, snakes_json TEXT NOT NULL, you_json TEXT NOT NULL,
+  food_json TEXT NOT NULL, hazards_json TEXT NOT NULL, UNIQUE (game_id, turn)
+);
+CREATE TABLE snake_turns (
+  id INTEGER PRIMARY KEY AUTOINCREMENT, game_id TEXT NOT NULL, turn INTEGER NOT NULL,
+  snake_id TEXT NOT NULL, snake_name TEXT, health INTEGER, length INTEGER,
+  head_x INTEGER, head_y INTEGER, body_json TEXT NOT NULL, is_you INTEGER NOT NULL DEFAULT 0,
+  inferred_move TEXT, latency TEXT, UNIQUE (game_id, turn, snake_id)
+);
+"""
+
+def _body(*points):
+  return [{"x": x, "y": y} for x, y in points]
+
+THINKING = {"turn": 1, "health": 99, "length": 3, "food_count": 1, "board_size": "11x11", "scores": {"up": 2.5}}
+
+def create_legacy_database(path:Path, game_snakes:bool=True) -> None:
+  with closing(sqlite3.connect(path)) as connection, connection:
+    connection.executescript(LEGACY_SCHEMA)
+    connection.execute("""
+      INSERT INTO games (game_id, started_at, ended_at, width, height, source, map_name,
+        ruleset_name, ruleset_version, your_snake_id, your_snake_name, game_type,
+        winner_name, winner_you, final_turn, status)
+      VALUES ('g1', '2026-08-01T10:00:00+00:00', '2026-08-01T10:01:00+00:00', 11, 11,
+        'league', 'standard', 'standard', 'v1', 'me', 'Me', 'duel', 'Me', 1, 2, 'finished')
+    """)
+    if game_snakes:
+      connection.executemany(
+        "INSERT INTO game_snakes VALUES (?, ?, ?, ?)",
+        [("g1", "me", "Me", 1), ("g1", "enemy", "Enemy", 0)],
+      )
+    food = json.dumps(_body((5, 5)))
+    for turn, move, thinking in ((0, None, None), (1, "up", THINKING), (2, "left", {"scores": {"left": 1}})):
+      connection.execute("""
+        INSERT INTO turns (game_id, turn, observed_at, my_move, my_thinking_json,
+          board_state_json, snakes_json, you_json, food_json, hazards_json)
+        VALUES ('g1', ?, ?, ?, ?, '{}', '[]', '{}', ?, '[]')
+      """, (turn, f"2026-08-01T10:00:0{turn}.123456+00:00", move, json.dumps(thinking) if thinking else None, food))
+    me = [_body((1, 1), (1, 1), (1, 1)), _body((1, 2), (1, 1), (1, 1)), _body((0, 2), (1, 2), (1, 1))]
+    enemy = [_body((9, 9), (9, 9), (9, 9)), _body((9, 8), (9, 9), (9, 9)), _body((9, 7), (9, 8), (9, 9))]
+    for turn in range(3):
+      for snake_id, bodies, latency in (("me", me, "12"), ("enemy", enemy, "")):
+        body = bodies[turn]
+        connection.execute("""
+          INSERT INTO snake_turns (game_id, turn, snake_id, snake_name, health, length,
+            head_x, head_y, body_json, is_you, latency)
+          VALUES ('g1', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        """, (
+          turn, snake_id, None if game_snakes else snake_id.title(), 100 - turn, len(body),
+          body[0]["x"], body[0]["y"], json.dumps(body), int(snake_id == "me" and not game_snakes), latency,
+        ))
+
+class TestLegacyConvert(unittest.TestCase):
+  def test_rejects_inconsistent_rows_instead_of_losing_data(self):
+    with self.assertRaises(LegacyDataError):
+      build_legacy_bundle(
+        {"game_id": "g", "started_at": "2026-01-01"}, [], [],
+        [{"snake_id": "s", "turn": 0, "length": 5, "body": json.dumps(_body((1, 1)))}],
+      )
+
+  def test_invalid_json_aborts(self):
+    with self.assertRaises(LegacyDataError):
+      build_legacy_bundle(
+        {"game_id": "g", "started_at": "2026-01-01"}, [],
+        [{"turn": 0, "observed_at": "2026-01-01", "my_thinking": "{invalid"}], [],
+      )
+
+class TestLegacyMigration(unittest.TestCase):
+  def _migrate(self, source:Path, destination:Path, **options):
+    async def run():
+      backend = SqliteGameplayBackend(str(destination))
+      with LegacySqliteReader(source) as reader:
+        stats = await migrate_legacy(reader, backend, **options)
+      return stats, await backend.get_game_replay("g1")
+    return asyncio.run(run())
+
+  def test_migrates_finished_game_into_identical_replay(self):
+    with tempfile.TemporaryDirectory() as temp_dir:
+      source, destination = Path(temp_dir) / "legacy.sqlite3", Path(temp_dir) / "compact.sqlite3"
+      create_legacy_database(source)
+
+      stats, replay = self._migrate(source, destination)
+
+      self.assertEqual((stats.games, stats.turns, stats.states, stats.failed), (1, 3, 6, 0))
+      turns = replay["turns"]
+      self.assertEqual([turn["my_move"] for turn in turns], [None, "up", "left"])
+      self.assertEqual(turns[1]["my_thinking"], THINKING)
+      self.assertEqual(turns[2]["food"], _body((5, 5)))
+      self.assertEqual(turns[1]["observed_at"], "2026-08-01T10:00:01.123456+00:00")
+      me = turns[2]["snakes"][0]
+      self.assertEqual((me["snake_id"], me["snake_name"], me["is_you"]), ("me", "Me", True))
+      self.assertEqual((me["inferred_move"], me["latency"], me["health"], me["length"]), ("left", "12", 98, 3))
+      enemy = turns[2]["snakes"][1]
+      self.assertEqual((enemy["inferred_move"], enemy["latency"]), ("down", None))
+      with closing(sqlite3.connect(destination)) as connection:
+        self.assertEqual(connection.execute("SELECT turn_count FROM game_replays").fetchone()[0], 3)
+
+  def test_synthesizes_missing_game_snakes_from_snake_rows(self):
+    with tempfile.TemporaryDirectory() as temp_dir:
+      source, destination = Path(temp_dir) / "legacy.sqlite3", Path(temp_dir) / "compact.sqlite3"
+      create_legacy_database(source, game_snakes=False)
+
+      _, replay = self._migrate(source, destination)
+
+      snakes = replay["turns"][0]["snakes"]
+      self.assertEqual([(s["snake_id"], s["snake_name"], s["is_you"]) for s in snakes], [
+        ("me", "Me", True), ("enemy", "Enemy", False),
+      ])
+      self.assertEqual(snakes[0]["customizations"], {})
+
+  def test_quality_filter_keeps_result_but_drops_replay(self):
+    with tempfile.TemporaryDirectory() as temp_dir:
+      source, destination = Path(temp_dir) / "legacy.sqlite3", Path(temp_dir) / "compact.sqlite3"
+      create_legacy_database(source)
+
+      stats, replay = self._migrate(source, destination, minimum_quality="high")
+
+      self.assertEqual(stats.dropped_replays, 1)
+      self.assertIsNone(replay)
+      with closing(sqlite3.connect(destination)) as connection:
+        row = connection.execute("SELECT has_replay, quality_status, winner_you FROM games").fetchone()
+        self.assertEqual(row, (0, "low_quality", 1))
+        self.assertEqual(connection.execute("SELECT COUNT(*) FROM game_replays").fetchone()[0], 0)
+
+  def test_script_migrates_in_place_and_drops_legacy_tables(self):
+    with tempfile.TemporaryDirectory() as temp_dir:
+      database = Path(temp_dir) / "gameplay.sqlite3"
+      create_legacy_database(database)
+      script = [sys.executable, "scripts/migrate_gameplay_to_compact.py", "--source", str(database)]
+
+      migrated = subprocess.run(script, cwd=REPO_ROOT, text=True, capture_output=True)
+      dropped = subprocess.run(script + ["--drop-legacy"], cwd=REPO_ROOT, text=True, capture_output=True)
+
+      self.assertEqual(migrated.returncode, 0, migrated.stderr)
+      self.assertEqual(dropped.returncode, 0, dropped.stderr)
+      with closing(sqlite3.connect(database)) as connection:
+        tables = {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type = 'table'")}
+      self.assertFalse({"turns", "snake_turns"} & tables)
+      replay = asyncio.run(SqliteGameplayBackend(str(database)).get_game_replay("g1"))
+      self.assertEqual(len(replay["turns"]), 3)
+
+if __name__ == "__main__":
+  unittest.main()

tests/test_MergeGameplayDatabases.py

@@ -1,104 +1,98 @@
+import json
 import sqlite3
 import subprocess
 import sys
 import tempfile
-from pathlib import Path
 import unittest
+from contextlib import closing
+from datetime import datetime, timezone
+from pathlib import Path
 
 from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend
+from server.database.compact.records import GameBundle, SnakeIdentity, SnakeState, TurnRecord
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+CUSTOMIZATIONS = {"color": "#663399", "head": "ferret", "tail": "swirl"}
+
+def _bundle(game_id:str, winner_you:bool, cleaned:bool=False) -> GameBundle:
+  turns = [
+    TurnRecord(turn, datetime(2026, 1, 1, 0, 0, turn, tzinfo=timezone.utc), "up", {"score": turn}, [] if turn == 1 else None, [] if turn == 1 else None)
+    for turn in range(1, 6)
+  ]
+  states = [
+    SnakeState(snake_id, turn, 90, 10, [{"x": x, "y": turn}])
+    for turn in range(1, 6)
+    for snake_id, x in (("me", 1), ("a", 3), ("b", 5))
+  ]
+  return GameBundle(
+    game={
+      "game_id": game_id, "started_at": "2026-01-01T00:00:00+00:00",
+      "ended_at": "2026-01-01T00:01:00+00:00", "width": 11, "height": 11,
+      "source": "league", "map_name": "standard", "ruleset_name": "standard",
+      "ruleset_version": "v1", "your_snake_id": "me", "your_snake_name": "PrismBattleSnake",
+      "your_snake_type": "PrismBattleSnake_GPT_5_6_Sol", "your_snake_version": "1.0.0",
+      "game_type": "standard", "winner_name": "PrismBattleSnake", "winner_you": winner_you,
+      "final_turn": 5, "status": "finished", "has_replay": True, "quality_status": "retained",
+      "quality_score": 90 if cleaned else None, "quality_tier": "high" if cleaned else None,
+      "quality_reasons": ["already_scored"] if cleaned else None,
+    },
+    snakes=[
+      SnakeIdentity("me", "PrismBattleSnake", True, CUSTOMIZATIONS),
+      SnakeIdentity("a", "A", False, {}),
+      SnakeIdentity("b", "B", False, {}),
+    ],
+    turns=turns,
+    states=states,
+  )
 
 class TestMergeGameplayDatabases(unittest.TestCase):
-  def _create_game(self, path:Path, game_id:str, winner_you:int, cleaned:bool=False) -> None:
-    SqliteGameplayBackend(str(path))
-    with sqlite3.connect(path) as connection:
-      connection.execute("""
-        INSERT INTO games (
-          game_id, started_at, ended_at, width, height, source, map_name,
-          ruleset_name, ruleset_version, your_snake_id, your_snake_name,
-          your_snake_type, your_snake_version, game_type, winner_name,
-          winner_you, final_turn, status, has_replay, quality_status,
-          quality_score, quality_tier, quality_reasons_json
-        ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
-      """, (
-        game_id, "2026-01-01T00:00:00Z", "2026-01-01T00:01:00Z", 11, 11,
-        "league", "standard", "standard", "v1", "me", "PrismBattleSnake",
-        "PrismBattleSnake_GPT_5_6_Sol", "1.0.0", "duel", "PrismBattleSnake",
-        winner_you, 1, "finished", 1, "retained",
-        90 if cleaned else None, "high" if cleaned else None,
-        '["already_scored"]' if cleaned else None,
-      ))
-      connection.execute("""
-        INSERT INTO game_snakes (
-          game_id,snake_id,snake_name,is_you,customizations_json
-        ) VALUES (?,?,?,?,?)
-      """, (
-        game_id, "me", "PrismBattleSnake", 1,
-        '{"color":"#663399","head":"ferret","tail":"swirl"}',
-      ))
-      connection.execute("""
-        INSERT INTO turns (
-          game_id,turn,observed_at,my_move,my_thinking_json,
-          board_state_json,snakes_json,you_json,food_json,hazards_json
-        ) VALUES (?,?,?,?,?,?,?,?,?,?)
-      """, (game_id, 1, "2026-01-01T00:00:01Z", "up", '{"score":1}', '{}', '[]', '{}', '[]', '[]'))
-      connection.execute("""
-        INSERT INTO snake_turns (
-          game_id,turn,snake_id,health,length,head_x,head_y,body_json,is_you,inferred_move
-        ) VALUES (?,?,?,?,?,?,?,?,?,?)
-      """, (game_id, 1, "me", 90, 3, 1, 1, '[{"x":1,"y":1}]', 0, "up"))
+  def _create(self, path:Path, bundle:GameBundle) -> None:
+    SqliteGameplayBackend(str(path)).import_game_bundle_sync(bundle)
 
-  def test_merges_base_and_delta_and_regenerates_row_ids(self):
+  def _merge(self, root:Path, *extra:str):
+    return subprocess.run([
+      sys.executable, "scripts/merge_gameplay_databases.py",
+      "--base", str(root / "base.sqlite3"), "--delta", str(root / "delta.sqlite3"),
+      "--destination", str(root / "merged.sqlite3"), *extra,
+    ], cwd=REPO_ROOT, text=True, capture_output=True)
+
+  def test_merges_base_and_delta_with_fresh_ids(self):
     with tempfile.TemporaryDirectory() as temp_dir:
       root = Path(temp_dir)
-      base = root / "base.sqlite3"
-      delta = root / "delta.sqlite3"
-      merged = root / "merged.sqlite3"
-      self._create_game(base, "base-game", 1, cleaned=True)
-      self._create_game(delta, "delta-game", 0)
+      self._create(root / "base.sqlite3", _bundle("base-game", True, cleaned=True))
+      self._create(root / "delta.sqlite3", _bundle("delta-game", False))
 
-      result = subprocess.run([
-        sys.executable, "scripts/merge_gameplay_databases.py",
-        "--base", str(base), "--delta", str(delta),
-        "--destination", str(merged), "--minimum-quality", "medium",
-      ], cwd=Path(__file__).resolve().parents[1], text=True, capture_output=True)
+      result = self._merge(root, "--minimum-quality", "medium")
 
       self.assertEqual(result.returncode, 0, result.stderr)
-      with sqlite3.connect(merged) as connection:
+      merged = SqliteGameplayBackend(str(root / "merged.sqlite3"))
+      with closing(sqlite3.connect(root / "merged.sqlite3")) as connection:
         games = connection.execute(
-          "SELECT game_id, winner_you, has_replay, quality_tier FROM games ORDER BY game_id"
+          "SELECT game_id, gid, winner_you, has_replay, quality_tier FROM games ORDER BY game_id"
         ).fetchall()
-        self.assertEqual(games, [
-          ("base-game", 1, 1, "high"),
-          ("delta-game", 0, 1, "medium"),
-        ])
-        self.assertEqual(connection.execute("SELECT COUNT(*) FROM turns").fetchone()[0], 2)
-        self.assertEqual(connection.execute("SELECT COUNT(*) FROM snake_turns").fetchone()[0], 2)
-        customizations = connection.execute("""
-          SELECT game_id, customizations_json FROM game_snakes ORDER BY game_id
-        """).fetchall()
-        self.assertEqual(customizations, [
-          ("base-game", '{"color":"#663399","head":"ferret","tail":"swirl"}'),
-          ("delta-game", '{"color":"#663399","head":"ferret","tail":"swirl"}'),
-        ])
+        self.assertEqual(connection.execute("SELECT COUNT(*) FROM game_replays").fetchone()[0], 2)
         self.assertEqual(connection.execute("PRAGMA foreign_key_check").fetchall(), [])
+      self.assertEqual([(g[0], g[2], g[3]) for g in games], [("base-game", 1, 1), ("delta-game", 0, 1)])
+      self.assertEqual(games[0][4], "high")
+      self.assertEqual(len({g[1] for g in games}), 2)
+      for game_id in ("base-game", "delta-game"):
+        bundle = merged.read_game_bundle_sync(game_id)
+        self.assertEqual(len(bundle.turns), 5)
+        self.assertEqual(len(bundle.states), 15)
+        me = next(snake for snake in bundle.snakes if snake.snake_id == "me")
+        self.assertEqual(me.customizations, CUSTOMIZATIONS)
 
   def test_conflicting_duplicate_aborts_without_destination(self):
     with tempfile.TemporaryDirectory() as temp_dir:
       root = Path(temp_dir)
-      base = root / "base.sqlite3"
-      delta = root / "delta.sqlite3"
-      merged = root / "merged.sqlite3"
-      self._create_game(base, "same-game", 1, cleaned=True)
-      self._create_game(delta, "same-game", 0)
+      self._create(root / "base.sqlite3", _bundle("same-game", True, cleaned=True))
+      self._create(root / "delta.sqlite3", _bundle("same-game", False))
 
-      result = subprocess.run([
-        sys.executable, "scripts/merge_gameplay_databases.py",
-        "--base", str(base), "--delta", str(delta), "--destination", str(merged),
-      ], cwd=Path(__file__).resolve().parents[1], text=True, capture_output=True)
+      result = self._merge(root)
 
       self.assertNotEqual(result.returncode, 0)
       self.assertIn("Conflicting duplicate game_id", result.stderr)
-      self.assertFalse(merged.exists())
+      self.assertFalse((root / "merged.sqlite3").exists())
 
 if __name__ == "__main__":
   unittest.main()

tests/test_MigrateGameplayDatabase.py

@@ -1,137 +0,0 @@
-import sqlite3
-import tempfile
-import unittest
-from pathlib import Path
-
-from scripts.migrate_gameplay_database import copy_game_snakes
-from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend
-
-class TestMigrateGameplayDatabase(unittest.TestCase):
-  def test_copy_game_snakes_preserves_customizations(self):
-    with tempfile.TemporaryDirectory() as temp_dir:
-      root = Path(temp_dir)
-      source_path = root / "source.sqlite3"
-      destination_path = root / "destination.sqlite3"
-      SqliteGameplayBackend(str(source_path))
-      SqliteGameplayBackend(str(destination_path))
-
-      with sqlite3.connect(source_path) as source:
-        source.execute("PRAGMA foreign_keys = OFF")
-        source.execute("""
-          INSERT INTO game_snakes (
-            game_id, snake_id, snake_name, is_you, customizations_json
-          ) VALUES (?, ?, ?, ?, ?)
-        """, (
-          "game-1", "snake-1", "PrismBattleSnake", 1,
-          '{"color":"#663399","head":"ferret","tail":"swirl"}',
-        ))
-
-      source = sqlite3.connect(source_path)
-      destination = sqlite3.connect(destination_path)
-      try:
-        copied = copy_game_snakes(
-          source, destination, batch_size=10, retained_ids={"game-1"},
-        )
-        destination.commit()
-        row = destination.execute("""
-          SELECT snake_name, is_you, customizations_json
-          FROM game_snakes WHERE game_id = ? AND snake_id = ?
-        """, ("game-1", "snake-1")).fetchone()
-      finally:
-        source.close()
-        destination.close()
-
-      self.assertEqual(copied, 1)
-      self.assertEqual(row, (
-        "PrismBattleSnake", 1,
-        '{"color":"#663399","head":"ferret","tail":"swirl"}',
-      ))
-
-  def test_copy_game_snakes_synthesizes_rows_when_table_is_empty(self):
-    source = sqlite3.connect(":memory:")
-    destination = sqlite3.connect(":memory:")
-    try:
-      source.execute("""
-        CREATE TABLE game_snakes (
-          game_id TEXT, snake_id TEXT, snake_name TEXT, is_you INTEGER,
-          customizations_json TEXT NOT NULL DEFAULT '{}'
-        )
-      """)
-      source.execute("""
-        CREATE TABLE snake_turns (
-          game_id TEXT, snake_id TEXT, snake_name TEXT, is_you INTEGER
-        )
-      """)
-      source.executemany(
-        "INSERT INTO snake_turns VALUES (?, ?, ?, ?)",
-        [
-          ("game-1", "snake-1", "PrismBattleSnake", 1),
-          ("game-1", "snake-1", "PrismBattleSnake", 1),
-          ("game-1", "snake-2", "Enemy", 0),
-        ],
-      )
-      destination.execute("""
-        CREATE TABLE game_snakes (
-          game_id TEXT, snake_id TEXT, snake_name TEXT, is_you INTEGER,
-          customizations_json TEXT NOT NULL DEFAULT '{}',
-          PRIMARY KEY (game_id, snake_id)
-        )
-      """)
-
-      copied = copy_game_snakes(
-        source, destination, batch_size=10, retained_ids={"game-1"},
-      )
-      rows = destination.execute("""
-        SELECT snake_id, snake_name, is_you, customizations_json
-        FROM game_snakes ORDER BY snake_id
-      """).fetchall()
-    finally:
-      source.close()
-      destination.close()
-
-    self.assertEqual(copied, 2)
-    self.assertEqual(rows, [
-      ("snake-1", "PrismBattleSnake", 1, "{}"),
-      ("snake-2", "Enemy", 0, "{}"),
-    ])
-
-  def test_copy_game_snakes_defaults_legacy_schema_to_empty_customizations(self):
-    source = sqlite3.connect(":memory:")
-    destination = sqlite3.connect(":memory:")
-    try:
-      source.execute("""
-        CREATE TABLE game_snakes (
-          game_id TEXT, snake_id TEXT, snake_name TEXT, is_you INTEGER
-        )
-      """)
-      source.execute(
-        "INSERT INTO game_snakes VALUES (?, ?, ?, ?)",
-        ("game-1", "snake-1", "LegacySnake", 0),
-      )
-      source.execute("""
-        CREATE TABLE snake_turns (
-          game_id TEXT, snake_id TEXT, snake_name TEXT, is_you INTEGER
-        )
-      """)
-      destination.execute("""
-        CREATE TABLE game_snakes (
-          game_id TEXT, snake_id TEXT, snake_name TEXT, is_you INTEGER,
-          customizations_json TEXT NOT NULL DEFAULT '{}'
-        )
-      """)
-
-      copied = copy_game_snakes(
-        source, destination, batch_size=10, retained_ids={"game-1"},
-      )
-      row = destination.execute(
-        "SELECT customizations_json FROM game_snakes"
-      ).fetchone()
-    finally:
-      source.close()
-      destination.close()
-
-    self.assertEqual(copied, 1)
-    self.assertEqual(row, ("{}",))
-
-if __name__ == "__main__":
-  unittest.main()

tests/test_MigrateSqliteToPostgresql.py

@@ -1,34 +0,0 @@
-import unittest
-
-from scripts.migrate_sqlite_to_postgresql import parse_json, parse_timestamp, transform_row
-
-class TestMigrateSqliteToPostgresql(unittest.TestCase):
-  def test_transform_row_converts_timestamp_boolean_and_json(self):
-    columns = ("started_at", "winner_you", "quality_reasons", "game_id")
-    row = ("2026-08-01T10:20:30+00:00", 1, '["complete"]', "game-1")
-
-    transformed = transform_row(columns, row)
-
-    self.assertEqual(transformed[0].isoformat(), "2026-08-01T10:20:30+00:00")
-    self.assertIs(transformed[1], True)
-    self.assertEqual(transformed[2], '["complete"]')
-    self.assertEqual(transformed[3], "game-1")
-
-  def test_json_defaults_match_not_null_postgresql_columns(self):
-    columns = ("customizations", "board_state", "snakes", "you", "food", "hazards", "body")
-    self.assertEqual(
-      transform_row(columns, (None,) * len(columns)),
-      ("{}", "{}", "[]", "{}", "[]", "[]", "[]"),
-    )
-
-  def test_invalid_json_aborts_instead_of_silently_losing_data(self):
-    with self.assertRaisesRegex(ValueError, "Invalid JSON"):
-      parse_json("{invalid")
-
-  def test_z_suffix_timestamp_is_timezone_aware(self):
-    parsed = parse_timestamp("2026-08-01T10:20:30Z")
-    self.assertIsNotNone(parsed.tzinfo)
-    self.assertEqual(parsed.utcoffset().total_seconds(), 0)
-
-if __name__ == "__main__":
-  unittest.main()

tests/test_PostgresqlGameplayBackend.py

@@ -0,0 +1,90 @@
+"""Integration tests against a disposable PostgreSQL database.
+
+Set GAMEPLAY_TEST_PG_DSN to a database that may be wiped, e.g.
+postgresql://user:pass@127.0.0.1:5432/battlesnake_test
+"""
+import os
+import unittest
+
+from server.database.backend.PostgresqlGameplayBackend import PostgresqlGameplayBackend
+import test_GameplayDatabase as sqlite_tests
+
+DSN = os.environ.get("GAMEPLAY_TEST_PG_DSN")
+
+@unittest.skipUnless(DSN, "GAMEPLAY_TEST_PG_DSN not set")
+class TestPostgresqlGameplayBackend(unittest.IsolatedAsyncioTestCase):
+  _build_state = sqlite_tests.TestGameplayDatabase._build_state
+  _play_game = sqlite_tests.TestGameplayDatabase._play_game
+
+  async def asyncSetUp(self):
+    self.backend = PostgresqlGameplayBackend(DSN)
+    pool = await self.backend._get_pool()
+    async with pool.acquire() as conn:
+      await conn.execute("TRUNCATE games CASCADE")
+
+  async def asyncTearDown(self):
+    await self.backend.close()
+
+  async def _count(self, table:str) -> int:
+    pool = await self.backend._get_pool()
+    async with pool.acquire() as conn:
+      return int(await conn.fetchval(f"SELECT COUNT(*) FROM {table}"))
+
+  async def test_running_rows_then_packed_replay_is_identical(self):
+    await self._play_game(self.backend)
+    self.assertEqual(await self._count("game_turns"), 2)
+    self.assertEqual(await self._count("snake_states"), 4)
+    running = await self.backend.get_game_replay("game-abc")
+
+    await self.backend.record_game_end(
+      self._build_state(turn=2, me_head=(2, 2), enemy_head=(4, 4), include_enemy=False)
+    )
+
+    self.assertEqual(await self._count("game_turns"), 0)
+    self.assertEqual(await self._count("snake_states"), 0)
+    self.assertEqual(await self._count("game_replays"), 1)
+    finished = await self.backend.get_game_replay("game-abc")
+    self.assertEqual(finished["turns"], running["turns"])
+    self.assertEqual(finished["game"]["status"], "finished")
+    turn = finished["turns"][1]
+    self.assertEqual(turn["my_thinking"], {"turn": 2, "reason": "food", "scores": {"up": 1.4}})
+    self.assertEqual(finished["turns"][0]["my_thinking"]["health"], 90)
+    me = next(snake for snake in turn["snakes"] if snake["snake_id"] == "me")
+    self.assertEqual(me["inferred_move"], "up")
+    self.assertEqual(me["customizations"]["head"], "ferret")
+
+  async def test_stale_game_is_finalized_and_packed(self):
+    await self._play_game(self.backend)
+    pool = await self.backend._get_pool()
+    async with pool.acquire() as conn:
+      await conn.execute("UPDATE games SET started_at = now() - interval '1 day'")
+
+    self.assertEqual(await self.backend.finalize_stale_running_games(600), 1)
+
+    self.assertEqual(await self._count("game_turns"), 0)
+    replay = await self.backend.get_game_replay("game-abc")
+    self.assertEqual(len(replay["turns"]), 2)
+
+  async def test_late_turn_after_end_is_merged_and_repacked(self):
+    await self._play_game(self.backend)
+    await self.backend.record_game_end(self._build_state(turn=2, me_head=(2, 2), enemy_head=(4, 4)))
+    await self.backend.record_turn(self._build_state(turn=3, me_head=(3, 2), enemy_head=(3, 4)), my_move="right")
+
+    self.assertEqual(len((await self.backend.get_game_replay("game-abc"))["turns"]), 3)
+    await self.backend.finalize_stale_running_games(600)
+    self.assertEqual(await self._count("game_turns"), 0)
+    self.assertEqual(len((await self.backend.get_game_replay("game-abc"))["turns"]), 3)
+
+  async def test_bundle_roundtrip(self):
+    await self._play_game(self.backend)
+    await self.backend.record_game_end(self._build_state(turn=2, me_head=(2, 2), enemy_head=(4, 4)))
+    bundle = await self.backend.read_game_bundle("game-abc")
+    bundle.game["game_id"] = "copy"
+    await self.backend.import_game_bundle(bundle)
+
+    copy = await self.backend.read_game_bundle("copy")
+    self.assertEqual(copy.turns, bundle.turns)
+    self.assertEqual(sorted(copy.states, key=lambda s: (s.turn, s.snake_id)), sorted(bundle.states, key=lambda s: (s.turn, s.snake_id)))
+
+if __name__ == "__main__":
+  unittest.main()

tests/test_PostgresqlGameplayMigration.py

@@ -1,65 +0,0 @@
-import sqlite3
-import tempfile
-import unittest
-from pathlib import Path
-
-from server.database.backend.PostgresqlGameplayBackend import PostgresqlGameplayBackend
-
-class TestPostgresqlGameplayMigration(unittest.TestCase):
-  def _create_source(self, path:Path, winner_column:str, winner_value:str|None) -> None:
-    with sqlite3.connect(path) as connection:
-      connection.executescript(f"""
-        CREATE TABLE games (
-          game_id TEXT PRIMARY KEY, started_at TEXT NOT NULL, ended_at TEXT,
-          width INTEGER, height INTEGER, source TEXT, map_name TEXT,
-          ruleset_name TEXT, ruleset_version TEXT, your_snake_id TEXT,
-          your_snake_name TEXT, your_snake_type TEXT, your_snake_version TEXT,
-          game_type TEXT, {winner_column} TEXT, winner_you INTEGER,
-          final_turn INTEGER, status TEXT
-        );
-        CREATE TABLE turns (
-          game_id TEXT, turn INTEGER, observed_at TEXT, my_move TEXT,
-          my_thinking_json TEXT, board_state_json TEXT, snakes_json TEXT,
-          you_json TEXT, food_json TEXT, hazards_json TEXT
-        );
-        CREATE TABLE snake_turns (
-          game_id TEXT, turn INTEGER, snake_id TEXT, snake_name TEXT,
-          health INTEGER, length INTEGER, head_x INTEGER, head_y INTEGER,
-          body_json TEXT, is_you INTEGER, inferred_move TEXT, latency TEXT
-        );
-      """)
-      connection.execute(
-        f"""INSERT INTO games (
-          game_id, started_at, {winner_column}, winner_you, final_turn, status
-        ) VALUES (?, ?, ?, ?, ?, ?)""",
-        ("game-1", "2026-08-01T10:00:00+00:00", winner_value, 1, 4, "finished"),
-      )
-
-  def test_reads_current_winner_name_schema(self):
-    with tempfile.TemporaryDirectory() as temp_dir:
-      path = Path(temp_dir) / "gameplay.sqlite3"
-      self._create_source(path, "winner_name", "Prism")
-      backend = PostgresqlGameplayBackend("postgresql://example", sqlite_migration_path=str(path))
-
-      games, turns, snake_turns = backend._read_sqlite_data_sync(str(path))
-
-      self.assertEqual(games[0]["winner_name"], "Prism")
-      self.assertEqual(turns, [])
-      self.assertEqual(snake_turns, [])
-      self.assertEqual(backend._migrated_winner_name(games[0]["winner_name"]), "Prism")
-
-  def test_reads_legacy_winner_names_json_schema(self):
-    with tempfile.TemporaryDirectory() as temp_dir:
-      path = Path(temp_dir) / "gameplay.sqlite3"
-      self._create_source(path, "winner_names_json", '["Prism"]')
-      backend = PostgresqlGameplayBackend("postgresql://example", sqlite_migration_path=str(path))
-
-      games, _, _ = backend._read_sqlite_data_sync(str(path))
-
-      self.assertEqual(
-        backend._migrated_winner_name(games[0]["winner_name"]),
-        "Prism",
-      )
-
-if __name__ == "__main__":
-  unittest.main()

tests/test_ZstdCompression.py

@@ -1,92 +0,0 @@
-import unittest, sqlite3, os
-from pathlib import Path
-
-from server.database import GameplayDatabase
-
-EXT_PATH = Path(os.environ.get("SQLITE_ZSTD_EXT", "lib/libsqlite_zstd.so")).expanduser().resolve()
-
-def _open(db_path: str) -> sqlite3.Connection:
-  conn = sqlite3.connect(db_path)
-  conn.row_factory = sqlite3.Row
-  if EXT_PATH.exists():
-    conn.enable_load_extension(True)
-    conn.load_extension(str(EXT_PATH))
-    conn.enable_load_extension(False)
-  return conn
-
-@unittest.skipUnless(EXT_PATH.exists(), f"sqlite-zstd extension not found at {EXT_PATH}")
-class TestZstdCompression(unittest.TestCase):
-
-  def test_extension_loads(self):
-    conn = sqlite3.connect(":memory:")
-    conn.enable_load_extension(True)
-    conn.load_extension(str(EXT_PATH))
-    conn.enable_load_extension(False)
-    result = conn.execute("SELECT zstd_compress('hello world', 6)").fetchone()[0]
-    self.assertIsInstance(result, bytes)
-    self.assertGreater(len(result), 0)
-    conn.close()
-
-  def test_compress_decompress_roundtrip(self):
-    # Use a long repetitive string so compression actually reduces size
-    original = '{"board": {"width": 11, "height": 11, "snakes": []}, "turn": 42} ' * 20
-    conn = sqlite3.connect(":memory:")
-    conn.enable_load_extension(True)
-    conn.load_extension(str(EXT_PATH))
-    conn.enable_load_extension(False)
-    compressed = conn.execute("SELECT zstd_compress(?, 6)", (original,)).fetchone()[0]
-    self.assertIsInstance(compressed, bytes)
-    self.assertLess(len(compressed), len(original))
-    conn.close()
-
-  def test_transparent_compression_enabled(self):
-    db_path = os.environ.get("GAMEPLAY_DB_PATH", "data/database/gameplay.sqlite3")
-    if not Path(db_path).exists():
-      self.skipTest(f"Database not found at {db_path}")
-
-    conn = _open(db_path)
-    tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master").fetchall()}
-    self.assertIn("_zstd_configs", tables, "_zstd_configs table missing — compression not enabled")
-
-    configs = conn.execute("SELECT config FROM _zstd_configs").fetchall()
-    compressed_columns = [row["config"] for row in configs]
-    self.assertEqual(len(compressed_columns), 6)
-
-    expected = ["board_state_json", "snakes_json", "you_json", "food_json", "hazards_json", "body_json"]
-    for col in expected:
-      self.assertTrue(
-        any(col in cfg for cfg in compressed_columns),
-        f"{col} not found in _zstd_configs",
-      )
-    conn.close()
-
-  def test_turns_and_snake_turns_are_views(self):
-    db_path = os.environ.get("GAMEPLAY_DB_PATH", "data/database/gameplay.sqlite3")
-    if not Path(db_path).exists():
-      self.skipTest(f"Database not found at {db_path}")
-
-    conn = _open(db_path)
-    views = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='view'").fetchall()}
-    self.assertIn("turns", views)
-    self.assertIn("snake_turns", views)
-    conn.close()
-
-  def test_board_state_json_is_stored_compressed(self):
-    db_path = os.environ.get("GAMEPLAY_DB_PATH", "data/database/gameplay.sqlite3")
-    if not Path(db_path).exists():
-      self.skipTest(f"Database not found at {db_path}")
-
-    conn = _open(db_path)
-    tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()}
-    if "_turns_zstd" not in tables:
-      self.skipTest("_turns_zstd table not found")
-
-    row = conn.execute("SELECT board_state_json FROM _turns_zstd LIMIT 1").fetchone()
-    if row is None:
-      self.skipTest("No rows in _turns_zstd")
-
-    self.assertIsInstance(row[0], (bytes, bytearray), "board_state_json should be stored as compressed bytes")
-    conn.close()
-
-if __name__ == "__main__":
-  unittest.main()