feat(database): store gameplay replays in compact form

- Add compact storage package: codec, records, hydrate, replay blobs, cache. - Running games write game_turns and snake_states keyed by integer ids. - Encode moves as 1-byte codes and bodies as head plus step codes. - Store food and hazards only when they changed since the previous turn. - Strip reasoning fields that duplicate stored values and restore on read. - Pack finished games into one zstd blob per game in game_replays. - Merge late turns into packed blobs during the stale-game sweep. - Extract SQLite and PostgreSQL schemas into dedicated modules. - Add read/import game bundle and list_game_ids to the backend template. - Add legacy readers and converter for the old turns/snake_turns layout. - Add migrate_gameplay_to_compact.py with in-place or cross-DB migration. - Verify every migrated game and keep legacy rows on failure. - Support --since, --minimum-quality and guarded --drop-legacy. - Add just recipe migrate-gameplay-postgres. - Rewrite merge script to copy game by game via backend bundles. - Move benchmark state loading into benchmark_states for both backends. - Update PostgreSQL cleanup script to the new replay tables. - Drop sqlite-zstd build stages from the Dockerfile. - Remove old migration scripts, normalized_turn and their tests. - Add tests for compact storage, schema, legacy migration and PostgreSQL. - Document the compact layout and migration workflow in the README.
2026-11-05 00:02:56 +01:00
parent 4403978537
commit 39a7a53873
41 changed files with 2779 additions and 2493 deletions
@@ -1,104 +1,98 @@
+import json
 import sqlite3
 import subprocess
 import sys
 import tempfile
-from pathlib import Path
 import unittest
+from contextlib import closing
+from datetime import datetime, timezone
+from pathlib import Path

 from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend
+from server.database.compact.records import GameBundle, SnakeIdentity, SnakeState, TurnRecord
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+CUSTOMIZATIONS = {"color": "#663399", "head": "ferret", "tail": "swirl"}
+
+def _bundle(game_id:str, winner_you:bool, cleaned:bool=False) -> GameBundle:
+  turns = [
+    TurnRecord(turn, datetime(2026, 1, 1, 0, 0, turn, tzinfo=timezone.utc), "up", {"score": turn}, [] if turn == 1 else None, [] if turn == 1 else None)
+    for turn in range(1, 6)
+  ]
+  states = [
+    SnakeState(snake_id, turn, 90, 10, [{"x": x, "y": turn}])
+    for turn in range(1, 6)
+    for snake_id, x in (("me", 1), ("a", 3), ("b", 5))
+  ]
+  return GameBundle(
+    game={
+      "game_id": game_id, "started_at": "2026-01-01T00:00:00+00:00",
+      "ended_at": "2026-01-01T00:01:00+00:00", "width": 11, "height": 11,
+      "source": "league", "map_name": "standard", "ruleset_name": "standard",
+      "ruleset_version": "v1", "your_snake_id": "me", "your_snake_name": "PrismBattleSnake",
+      "your_snake_type": "PrismBattleSnake_GPT_5_6_Sol", "your_snake_version": "1.0.0",
+      "game_type": "standard", "winner_name": "PrismBattleSnake", "winner_you": winner_you,
+      "final_turn": 5, "status": "finished", "has_replay": True, "quality_status": "retained",
+      "quality_score": 90 if cleaned else None, "quality_tier": "high" if cleaned else None,
+      "quality_reasons": ["already_scored"] if cleaned else None,
+    },
+    snakes=[
+      SnakeIdentity("me", "PrismBattleSnake", True, CUSTOMIZATIONS),
+      SnakeIdentity("a", "A", False, {}),
+      SnakeIdentity("b", "B", False, {}),
+    ],
+    turns=turns,
+    states=states,
+  )

 class TestMergeGameplayDatabases(unittest.TestCase):
-  def _create_game(self, path:Path, game_id:str, winner_you:int, cleaned:bool=False) -> None:
-    SqliteGameplayBackend(str(path))
-    with sqlite3.connect(path) as connection:
-      connection.execute("""
-        INSERT INTO games (
-          game_id, started_at, ended_at, width, height, source, map_name,
-          ruleset_name, ruleset_version, your_snake_id, your_snake_name,
-          your_snake_type, your_snake_version, game_type, winner_name,
-          winner_you, final_turn, status, has_replay, quality_status,
-          quality_score, quality_tier, quality_reasons_json
-        ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
-      """, (
-        game_id, "2026-01-01T00:00:00Z", "2026-01-01T00:01:00Z", 11, 11,
-        "league", "standard", "standard", "v1", "me", "PrismBattleSnake",
-        "PrismBattleSnake_GPT_5_6_Sol", "1.0.0", "duel", "PrismBattleSnake",
-        winner_you, 1, "finished", 1, "retained",
-        90 if cleaned else None, "high" if cleaned else None,
-        '["already_scored"]' if cleaned else None,
-      ))
-      connection.execute("""
-        INSERT INTO game_snakes (
-          game_id,snake_id,snake_name,is_you,customizations_json
-        ) VALUES (?,?,?,?,?)
-      """, (
-        game_id, "me", "PrismBattleSnake", 1,
-        '{"color":"#663399","head":"ferret","tail":"swirl"}',
-      ))
-      connection.execute("""
-        INSERT INTO turns (
-          game_id,turn,observed_at,my_move,my_thinking_json,
-          board_state_json,snakes_json,you_json,food_json,hazards_json
-        ) VALUES (?,?,?,?,?,?,?,?,?,?)
-      """, (game_id, 1, "2026-01-01T00:00:01Z", "up", '{"score":1}', '{}', '[]', '{}', '[]', '[]'))
-      connection.execute("""
-        INSERT INTO snake_turns (
-          game_id,turn,snake_id,health,length,head_x,head_y,body_json,is_you,inferred_move
-        ) VALUES (?,?,?,?,?,?,?,?,?,?)
-      """, (game_id, 1, "me", 90, 3, 1, 1, '[{"x":1,"y":1}]', 0, "up"))
+  def _create(self, path:Path, bundle:GameBundle) -> None:
+    SqliteGameplayBackend(str(path)).import_game_bundle_sync(bundle)

-  def test_merges_base_and_delta_and_regenerates_row_ids(self):
+  def _merge(self, root:Path, *extra:str):
+    return subprocess.run([
+      sys.executable, "scripts/merge_gameplay_databases.py",
+      "--base", str(root / "base.sqlite3"), "--delta", str(root / "delta.sqlite3"),
+      "--destination", str(root / "merged.sqlite3"), *extra,
+    ], cwd=REPO_ROOT, text=True, capture_output=True)
+
+  def test_merges_base_and_delta_with_fresh_ids(self):
    with tempfile.TemporaryDirectory() as temp_dir:
      root = Path(temp_dir)
-      base = root / "base.sqlite3"
-      delta = root / "delta.sqlite3"
-      merged = root / "merged.sqlite3"
-      self._create_game(base, "base-game", 1, cleaned=True)
-      self._create_game(delta, "delta-game", 0)
+      self._create(root / "base.sqlite3", _bundle("base-game", True, cleaned=True))
+      self._create(root / "delta.sqlite3", _bundle("delta-game", False))

-      result = subprocess.run([
-        sys.executable, "scripts/merge_gameplay_databases.py",
-        "--base", str(base), "--delta", str(delta),
-        "--destination", str(merged), "--minimum-quality", "medium",
-      ], cwd=Path(__file__).resolve().parents[1], text=True, capture_output=True)
+      result = self._merge(root, "--minimum-quality", "medium")

      self.assertEqual(result.returncode, 0, result.stderr)
-      with sqlite3.connect(merged) as connection:
+      merged = SqliteGameplayBackend(str(root / "merged.sqlite3"))
+      with closing(sqlite3.connect(root / "merged.sqlite3")) as connection:
        games = connection.execute(
-          "SELECT game_id, winner_you, has_replay, quality_tier FROM games ORDER BY game_id"
+          "SELECT game_id, gid, winner_you, has_replay, quality_tier FROM games ORDER BY game_id"
        ).fetchall()
-        self.assertEqual(games, [
-          ("base-game", 1, 1, "high"),
-          ("delta-game", 0, 1, "medium"),
-        ])
-        self.assertEqual(connection.execute("SELECT COUNT(*) FROM turns").fetchone()[0], 2)
-        self.assertEqual(connection.execute("SELECT COUNT(*) FROM snake_turns").fetchone()[0], 2)
-        customizations = connection.execute("""
-          SELECT game_id, customizations_json FROM game_snakes ORDER BY game_id
-        """).fetchall()
-        self.assertEqual(customizations, [
-          ("base-game", '{"color":"#663399","head":"ferret","tail":"swirl"}'),
-          ("delta-game", '{"color":"#663399","head":"ferret","tail":"swirl"}'),
-        ])
+        self.assertEqual(connection.execute("SELECT COUNT(*) FROM game_replays").fetchone()[0], 2)
        self.assertEqual(connection.execute("PRAGMA foreign_key_check").fetchall(), [])
+      self.assertEqual([(g[0], g[2], g[3]) for g in games], [("base-game", 1, 1), ("delta-game", 0, 1)])
+      self.assertEqual(games[0][4], "high")
+      self.assertEqual(len({g[1] for g in games}), 2)
+      for game_id in ("base-game", "delta-game"):
+        bundle = merged.read_game_bundle_sync(game_id)
+        self.assertEqual(len(bundle.turns), 5)
+        self.assertEqual(len(bundle.states), 15)
+        me = next(snake for snake in bundle.snakes if snake.snake_id == "me")
+        self.assertEqual(me.customizations, CUSTOMIZATIONS)

  def test_conflicting_duplicate_aborts_without_destination(self):
    with tempfile.TemporaryDirectory() as temp_dir:
      root = Path(temp_dir)
-      base = root / "base.sqlite3"
-      delta = root / "delta.sqlite3"
-      merged = root / "merged.sqlite3"
-      self._create_game(base, "same-game", 1, cleaned=True)
-      self._create_game(delta, "same-game", 0)
+      self._create(root / "base.sqlite3", _bundle("same-game", True, cleaned=True))
+      self._create(root / "delta.sqlite3", _bundle("same-game", False))

-      result = subprocess.run([
-        sys.executable, "scripts/merge_gameplay_databases.py",
-        "--base", str(base), "--delta", str(delta), "--destination", str(merged),
-      ], cwd=Path(__file__).resolve().parents[1], text=True, capture_output=True)
+      result = self._merge(root)

      self.assertNotEqual(result.returncode, 0)
      self.assertIn("Conflicting duplicate game_id", result.stderr)
-      self.assertFalse(merged.exists())
+      self.assertFalse((root / "merged.sqlite3").exists())

 if __name__ == "__main__":
  unittest.main()