feat(database): store gameplay replays in compact form

- Add compact storage package: codec, records, hydrate, replay blobs, cache. - Running games write game_turns and snake_states keyed by integer ids. - Encode moves as 1-byte codes and bodies as head plus step codes. - Store food and hazards only when they changed since the previous turn. - Strip reasoning fields that duplicate stored values and restore on read. - Pack finished games into one zstd blob per game in game_replays. - Merge late turns into packed blobs during the stale-game sweep. - Extract SQLite and PostgreSQL schemas into dedicated modules. - Add read/import game bundle and list_game_ids to the backend template. - Add legacy readers and converter for the old turns/snake_turns layout. - Add migrate_gameplay_to_compact.py with in-place or cross-DB migration. - Verify every migrated game and keep legacy rows on failure. - Support --since, --minimum-quality and guarded --drop-legacy. - Add just recipe migrate-gameplay-postgres. - Rewrite merge script to copy game by game via backend bundles. - Move benchmark state loading into benchmark_states for both backends. - Update PostgreSQL cleanup script to the new replay tables. - Drop sqlite-zstd build stages from the Dockerfile. - Remove old migration scripts, normalized_turn and their tests. - Add tests for compact storage, schema, legacy migration and PostgreSQL. - Document the compact layout and migration workflow in the README.
2026-11-05 00:02:56 +01:00
parent 4403978537
commit 39a7a53873
41 changed files with 2779 additions and 2493 deletions
@@ -0,0 +1 @@
+"""Read the legacy turns/snake_turns layout and migrate it to compact storage."""
@@ -0,0 +1,98 @@
+"""Convert rows of the legacy layout (turns/snake_turns) into a compact bundle."""
+
+import json
+
+from server.database.compact.codec import encode_latency
+from server.database.compact.games import normalize_game
+from server.database.compact.records import (
+  GameBundle, PointsState, SnakeIdentity, SnakeState, compact_turn,
+)
+
+class LegacyDataError(ValueError):
+  """Legacy rows that cannot be converted without losing data."""
+
+def parse_json(value:object, default:object=None) -> object:
+  if value is None or value == "":
+    return default
+  if not isinstance(value, str):
+    return value
+  try:
+    return json.loads(value)
+  except (json.JSONDecodeError, TypeError) as exc:
+    raise LegacyDataError(f"Invalid JSON value: {value[:120]}") from exc
+
+def _state_from_row(row:dict) -> SnakeState:
+  body = parse_json(row.get("body"), []) or []
+  head_x, head_y, length = row.get("head_x"), row.get("head_y"), row.get("length")
+  if length is not None and int(length) != len(body):
+    raise LegacyDataError(f"snake {row['snake_id']} turn {row['turn']}: length {length} != body {len(body)}")
+  if body and head_x is not None and (head_x, head_y) != (body[0].get("x"), body[0].get("y")):
+    raise LegacyDataError(f"snake {row['snake_id']} turn {row['turn']}: head differs from body[0]")
+  return SnakeState(
+    snake_id=row["snake_id"], turn=int(row["turn"]), health=row.get("health"),
+    latency=encode_latency(row.get("latency")), body=body,
+  )
+
+def _states_from_board(board:dict, turn:int) -> list[SnakeState]:
+  return [
+    SnakeState(
+      snake_id=snake["id"], turn=turn, health=snake.get("health"),
+      latency=encode_latency(snake.get("latency")), body=list(snake.get("body", [])),
+    )
+    for snake in board.get("snakes", []) if snake.get("id") is not None
+  ]
+
+def _identities(game:dict, identity_rows:list[dict], snake_rows:list[dict], board_snakes:list[dict]) -> list[SnakeIdentity]:
+  identities:dict[str, SnakeIdentity] = {}
+  for row in identity_rows:
+    identities[row["snake_id"]] = SnakeIdentity(
+      row["snake_id"], row.get("snake_name"), bool(row.get("is_you")),
+      parse_json(row.get("customizations"), {}) or {},
+    )
+  # Older databases have an empty or partial game_snakes table.
+  for row in snake_rows + board_snakes:
+    snake_id = row.get("snake_id") or row.get("id")
+    if snake_id is None or snake_id in identities:
+      continue
+    name = row.get("snake_name") or row.get("name")
+    is_you = bool(row.get("is_you")) or snake_id == game.get("your_snake_id")
+    identities[snake_id] = SnakeIdentity(snake_id, name, is_you, row.get("customizations") or {})
+  for snake_id, identity in list(identities.items()):
+    if identity.name is None:
+      name = next((row.get("snake_name") for row in snake_rows if row.get("snake_id") == snake_id and row.get("snake_name")), None)
+      if name:
+        identities[snake_id] = SnakeIdentity(snake_id, name, identity.is_you, identity.customizations)
+  return list(identities.values())
+
+def build_legacy_bundle(game_row:dict, identity_rows:list[dict], turn_rows:list[dict], snake_rows:list[dict]) -> GameBundle:
+  game = normalize_game(game_row)
+  states_by_turn:dict[int, list[SnakeState]] = {}
+  for row in snake_rows:
+    state = _state_from_row(row)
+    states_by_turn.setdefault(state.turn, []).append(state)
+
+  turns = []
+  states = []
+  board_snakes = []
+  previous:PointsState|None = None
+  for row in sorted(turn_rows, key=lambda item: int(item["turn"])):
+    turn = int(row["turn"])
+    board = parse_json(row.get("board_state"), {}) or {}
+    food = board["food"] if "food" in board else parse_json(row.get("food"), []) or []
+    hazards = board["hazards"] if "hazards" in board else parse_json(row.get("hazards"), []) or []
+    turn_states = states_by_turn.pop(turn, None)
+    if turn_states is None:
+      turn_states = _states_from_board(board, turn)
+      board_snakes.extend(board.get("snakes", []))
+    states.extend(turn_states)
+    record, previous = compact_turn(
+      turn, row.get("observed_at"), row.get("my_move"), parse_json(row.get("my_thinking")),
+      food, hazards, turn_states, game.get("your_snake_id"), game.get("width"), game.get("height"),
+      previous,
+    )
+    turns.append(record)
+  # Snake rows without a turn row are kept as well.
+  for orphan in states_by_turn.values():
+    states.extend(orphan)
+
+  return GameBundle(game, _identities(game, identity_rows, snake_rows, board_snakes), turns, states)
@@ -0,0 +1,84 @@
+"""Copy legacy-layout games into the compact layout, verifying each game."""
+
+from dataclasses import dataclass, field
+import inspect
+import json
+
+from server.database.compact.quality import rate_bundle
+from server.database.compact.records import GameBundle
+from server.database.game_quality import quality_meets_minimum
+
+from .convert import LegacyDataError
+
+@dataclass
+class MigrationStats:
+  games:int = 0
+  replays:int = 0
+  turns:int = 0
+  states:int = 0
+  dropped_replays:int = 0
+  failed:int = 0
+  failures:list[str] = field(default_factory=list)
+
+async def _resolve(value):
+  return await value if inspect.isawaitable(value) else value
+
+def _signature(bundle:GameBundle) -> tuple:
+  turns = sorted(
+    (record.turn, record.move, json.dumps(record.thinking, sort_keys=True), json.dumps(record.food), json.dumps(record.hazards))
+    for record in bundle.turns
+  )
+  states = sorted(
+    (state.snake_id, state.turn, state.health, state.latency, json.dumps(state.body))
+    for state in bundle.states
+  )
+  return turns, states
+
+def apply_quality(bundle:GameBundle, minimum_quality:str) -> bool:
+  """Rate the replay; drop its turns below the minimum. Returns whether it is kept."""
+  quality = rate_bundle(bundle)
+  keep = quality_meets_minimum(quality.tier, minimum_quality)
+  bundle.game.update({
+    "has_replay": keep,
+    "quality_status": "retained" if keep else "low_quality",
+    "quality_score": quality.score,
+    "quality_tier": quality.tier,
+    "quality_reasons": list(quality.reasons),
+  })
+  if not keep:
+    bundle.turns = []
+    bundle.states = []
+  return keep
+
+async def migrate_legacy(
+  reader,
+  backend,
+  since:str|None=None,
+  minimum_quality:str|None=None,
+  verify:bool=True,
+  progress=None,
+) -> MigrationStats:
+  stats = MigrationStats()
+  game_ids = await _resolve(reader.game_ids(since))
+  for index, game_id in enumerate(game_ids, start=1):
+    try:
+      bundle = await _resolve(reader.read_bundle(game_id))
+      if bundle is None:
+        continue
+      if minimum_quality and not apply_quality(bundle, minimum_quality):
+        stats.dropped_replays += 1
+      await backend.import_game_bundle(bundle)
+      if verify and bundle.turns and bundle.game["has_replay"]:
+        stored = await backend.read_game_bundle(game_id)
+        if stored is None or _signature(stored) != _signature(bundle):
+          raise LegacyDataError("stored replay differs from source")
+      stats.games += 1
+      stats.replays += int(bool(bundle.turns))
+      stats.turns += len(bundle.turns)
+      stats.states += len(bundle.states)
+    except LegacyDataError as error:
+      stats.failed += 1
+      stats.failures.append(f"{game_id}: {error}")
+    if progress is not None and (index % 500 == 0 or index == len(game_ids)):
+      progress(index, len(game_ids), stats)
+  return stats
@@ -0,0 +1,69 @@
+"""Read legacy-layout games (turns/snake_turns) from PostgreSQL."""
+
+import json
+
+from server.database.compact.games import GAME_COLUMNS
+from server.database.compact.records import GameBundle
+
+from .convert import build_legacy_bundle
+
+class LegacyPostgresqlReader:
+  def __init__(self, pool):
+    self._pool = pool
+
+  @classmethod
+  async def connect(cls, dsn:str) -> "LegacyPostgresqlReader":
+    import asyncpg  # noqa: PLC0415
+
+    async def _init(conn) -> None:
+      await conn.set_type_codec("jsonb", encoder=json.dumps, decoder=json.loads, schema="pg_catalog")
+      await conn.set_type_codec("json", encoder=json.dumps, decoder=json.loads, schema="pg_catalog")
+
+    return cls(await asyncpg.create_pool(dsn=dsn, min_size=1, max_size=4, init=_init))
+
+  async def close(self) -> None:
+    await self._pool.close()
+
+  async def has_legacy_tables(self) -> bool:
+    async with self._pool.acquire() as conn:
+      return bool(await conn.fetchval(
+        "SELECT to_regclass('public.turns') IS NOT NULL AND to_regclass('public.snake_turns') IS NOT NULL"
+      ))
+
+  async def game_ids(self, since:str|None=None) -> list[str]:
+    async with self._pool.acquire() as conn:
+      if since:
+        rows = await conn.fetch("""
+          SELECT game_id FROM games WHERE started_at >= $1::timestamptz
+          ORDER BY started_at, game_id
+          """, since)
+      else:
+        rows = await conn.fetch("SELECT game_id FROM games ORDER BY started_at, game_id")
+    return [row["game_id"] for row in rows]
+
+  async def read_bundle(self, game_id:str) -> GameBundle|None:
+    async with self._pool.acquire() as conn:
+      game = await conn.fetchrow(f"SELECT {', '.join(GAME_COLUMNS)} FROM games WHERE game_id = $1", game_id)
+      if game is None:
+        return None
+      identities = await conn.fetch("""
+        SELECT snake_id, snake_name, is_you, customizations
+        FROM game_snakes WHERE game_id = $1
+        """, game_id)
+      turns = await conn.fetch("""
+        SELECT turn, observed_at, my_move, my_thinking, board_state, food, hazards
+        FROM turns WHERE game_id = $1
+        """, game_id)
+      snakes = await conn.fetch("""
+        SELECT snake_id, turn, snake_name, health, length, head_x, head_y, body, is_you, latency
+        FROM snake_turns WHERE game_id = $1
+        """, game_id)
+    return build_legacy_bundle(
+      dict(game), [dict(row) for row in identities],
+      [dict(row) for row in turns], [dict(row) for row in snakes],
+    )
+
+  async def turn_counts(self) -> dict[str, int]:
+    async with self._pool.acquire() as conn:
+      rows = await conn.fetch("SELECT game_id, COUNT(*) AS turns FROM turns GROUP BY game_id")
+    return {row["game_id"]: int(row["turns"]) for row in rows}
@@ -0,0 +1,100 @@
+"""Read legacy-layout games from a SQLite file (opened read-only)."""
+
+from pathlib import Path
+import sqlite3
+
+from server.database.compact.games import GAME_COLUMNS
+from server.database.compact.records import GameBundle
+
+from .convert import build_legacy_bundle, parse_json
+
+_TURN_COLUMNS = {
+  "turn": "turn", "observed_at": "observed_at", "my_move": "my_move",
+  "my_thinking": "my_thinking_json", "board_state": "board_state_json",
+  "food": "food_json", "hazards": "hazards_json",
+}
+_SNAKE_COLUMNS = {
+  "snake_id": "snake_id", "turn": "turn", "snake_name": "snake_name",
+  "health": "health", "length": "length", "head_x": "head_x", "head_y": "head_y",
+  "body": "body_json", "is_you": "is_you", "latency": "latency",
+}
+
+class LegacySqliteReader:
+  def __init__(self, path:str|Path):
+    self.path = Path(path)
+    self._connection = sqlite3.connect(f"file:{self.path}?mode=ro", uri=True, timeout=60)
+    self._connection.row_factory = sqlite3.Row
+    self._connection.execute("PRAGMA query_only = ON")
+
+  def __enter__(self) -> "LegacySqliteReader":
+    return self
+
+  def __exit__(self, *exc) -> None:
+    self.close()
+
+  def close(self) -> None:
+    self._connection.close()
+
+  def _columns(self, table:str) -> set[str]:
+    return {row[1] for row in self._connection.execute(f"PRAGMA table_info({table})")}
+
+  def _select(self, table:str, mapping:dict[str, str]) -> str:
+    available = self._columns(table)
+    return ", ".join(
+      f"{source} AS {name}" if source in available else f"NULL AS {name}"
+      for name, source in mapping.items()
+    )
+
+  def has_legacy_tables(self) -> bool:
+    return bool(self._columns("turns")) and bool(self._columns("snake_turns"))
+
+  def game_ids(self, since:str|None=None) -> list[str]:
+    if since:
+      rows = self._connection.execute(
+        "SELECT game_id FROM games WHERE started_at >= ? ORDER BY started_at, game_id", (since,),
+      )
+    else:
+      rows = self._connection.execute("SELECT game_id FROM games ORDER BY started_at, game_id")
+    return [row[0] for row in rows]
+
+  def read_bundle(self, game_id:str) -> GameBundle|None:
+    available = self._columns("games")
+    selected = ", ".join(
+      column if column in available
+      else "quality_reasons_json AS quality_reasons" if column == "quality_reasons" and "quality_reasons_json" in available
+      else f"NULL AS {column}"
+      for column in GAME_COLUMNS
+    )
+    game = self._connection.execute(f"SELECT {selected} FROM games WHERE game_id = ?", (game_id,)).fetchone()
+    if game is None:
+      return None
+    game = dict(game)
+    game["quality_reasons"] = parse_json(game.get("quality_reasons"))
+
+    identities = []
+    snake_columns = self._columns("game_snakes")
+    if snake_columns:
+      customizations = "customizations_json" if "customizations_json" in snake_columns else "NULL"
+      identities = [
+        dict(row) for row in self._connection.execute(f"""
+          SELECT snake_id, snake_name, is_you, {customizations} AS customizations
+          FROM game_snakes WHERE game_id = ?
+          """, (game_id,))
+      ]
+    turns = [
+      dict(row) for row in self._connection.execute(
+        f"SELECT {self._select('turns', _TURN_COLUMNS)} FROM turns WHERE game_id = ?", (game_id,),
+      )
+    ]
+    snakes = [
+      dict(row) for row in self._connection.execute(
+        f"SELECT {self._select('snake_turns', _SNAKE_COLUMNS)} FROM snake_turns WHERE game_id = ?", (game_id,),
+      )
+    ]
+    return build_legacy_bundle(game, identities, turns, snakes)
+
+  def turn_counts(self) -> dict[str, int]:
+    return {
+      row[0]: int(row[1])
+      for row in self._connection.execute("SELECT game_id, COUNT(*) FROM turns GROUP BY game_id")
+    }
				`@@ -0,0 +1 @@`
				`"""Read the legacy turns/snake_turns layout and migrate it to compact storage."""`