feat(database): store gameplay replays in compact form

- Add compact storage package: codec, records, hydrate, replay blobs, cache.
- Running games write game_turns and snake_states keyed by integer ids.
- Encode moves as 1-byte codes and bodies as head plus step codes.
- Store food and hazards only when they changed since the previous turn.
- Strip reasoning fields that duplicate stored values and restore on read.
- Pack finished games into one zstd blob per game in game_replays.
- Merge late turns into packed blobs during the stale-game sweep.
- Extract SQLite and PostgreSQL schemas into dedicated modules.
- Add read/import game bundle and list_game_ids to the backend template.
- Add legacy readers and converter for the old turns/snake_turns layout.
- Add migrate_gameplay_to_compact.py with in-place or cross-DB migration.
- Verify every migrated game and keep legacy rows on failure.
- Support --since, --minimum-quality and guarded --drop-legacy.
- Add just recipe migrate-gameplay-postgres.
- Rewrite merge script to copy game by game via backend bundles.
- Move benchmark state loading into benchmark_states for both backends.
- Update PostgreSQL cleanup script to the new replay tables.
- Drop sqlite-zstd build stages from the Dockerfile.
- Remove old migration scripts, normalized_turn and their tests.
- Add tests for compact storage, schema, legacy migration and PostgreSQL.
- Document the compact layout and migration workflow in the README.

server/database/benchmark_states.py

@@ -1,108 +1,57 @@
-"""Load sampled gameplay positions from SQLite or PostgreSQL."""
+"""Load sampled gameplay positions from a SQLite or PostgreSQL gameplay DB."""
 import asyncio
-import json
 from urllib.parse import urlparse
 
-def is_postgresql_source(source: str) -> bool:
+def is_postgresql_source(source:str) -> bool:
   return urlparse(source).scheme.lower() in {"postgres", "postgresql"}
 
-def _decode_json(value, default):
-  if value is None:
-    return default
-  if isinstance(value, str):
-    return json.loads(value)
-  return value
-
-def _build_state(row, snake_rows) -> tuple[dict, dict] | None:
-  board = _decode_json(row[1], {})
-  you = _decode_json(row[2], {})
-  if not board.get("snakes"):
-    snakes = []
-    for snake_row in snake_rows:
-      snake_id = snake_row[0]
-      snake_name = snake_row[1] or (row[6] if snake_id == row[5] else snake_id)
-      snakes.append({
-        "id": snake_id,
-        "name": snake_name,
-        "health": snake_row[2],
-        "length": snake_row[3],
-        "head": {"x": snake_row[4], "y": snake_row[5]},
-        "body": _decode_json(snake_row[6], []),
-        "customizations": _decode_json(snake_row[7], {}),
-      })
-    board = {
-      "width": row[7],
-      "height": row[8],
-      "food": _decode_json(row[3], []),
-      "hazards": _decode_json(row[4], []),
-      "snakes": snakes,
-    }
-  if not you:
-    you = next(
-      (snake for snake in board.get("snakes", []) if snake.get("id") == row[5]),
-      {},
-    )
+def replay_turn_to_state(game:dict, turn:dict) -> tuple[dict, dict]|None:
+  board = turn.get("board") or {}
+  you = turn.get("you") or {}
   if not you or not board.get("snakes"):
     return None
   return board, {
     "you": you,
-    "game_id": row[9],
-    "source": row[10] or "custom",
-    "map": row[11] or "standard",
+    "game_id": game["game_id"],
+    "source": game.get("source") or "custom",
+    "map": game.get("map") or "standard",
     "ruleset": {
-      "name": row[12] or "standard",
-      "version": row[13] or "v1.0.0",
+      "name": game.get("ruleset_name") or "standard",
+      "version": game.get("ruleset_version") or "v1.0.0",
       "settings": {},
     },
-    "turn": int(row[14]),
+    "turn": int(turn["turn"]),
   }
 
-async def _load_postgresql_states(dsn: str, samples: int, stride: int) -> list[tuple[dict, dict]]:
-  try:
-    import asyncpg
-  except ImportError as exc:
-    raise RuntimeError("asyncpg is required for PostgreSQL benchmark sources") from exc
+async def sample_states(backend, samples:int, stride:int) -> list[tuple[dict, dict]]:
+  """Every ``stride``-th stored turn, newest games first."""
+  states = []
+  seen = 0
+  for game_id in reversed(await backend.list_game_ids()):
+    replay = await backend.get_game_replay(game_id)
+    if replay is None:
+      continue
+    for turn in reversed(replay["turns"]):
+      if seen % stride == 0:
+        state = replay_turn_to_state(replay["game"], turn)
+        if state is not None:
+          states.append(state)
+          if len(states) >= samples:
+            return states
+      seen += 1
+  return states
 
-  connection = await asyncpg.connect(dsn=dsn)
+async def _load_states(source:str, samples:int, stride:int) -> list[tuple[dict, dict]]:
+  if is_postgresql_source(source):
+    from server.database.backend.PostgresqlGameplayBackend import PostgresqlGameplayBackend  # noqa: PLC0415
+    backend = PostgresqlGameplayBackend(source)
+  else:
+    from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend  # noqa: PLC0415
+    backend = SqliteGameplayBackend(source, read_only=True)
   try:
-    max_id = int(await connection.fetchval("SELECT max(id) FROM turns") or 0)
-    if max_id == 0:
-      return []
-    query = """
-      SELECT t.id, t.board_state, t.you, t.food, t.hazards,
-             g.your_snake_id, g.your_snake_name, g.width, g.height,
-             g.game_id, g.source, g.map_name,
-             g.ruleset_name, g.ruleset_version, t.turn
-      FROM turns AS t
-      JOIN games AS g ON g.game_id = t.game_id
-      WHERE t.id >= $1
-      ORDER BY t.id
-      LIMIT 1
-    """
-    snake_query = """
-      SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name),
-             st.health, st.length, st.head_x, st.head_y, st.body,
-             COALESCE(gs.customizations, '{}'::jsonb)
-      FROM snake_turns AS st
-      LEFT JOIN game_snakes AS gs
-        ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
-      WHERE st.game_id = $1 AND st.turn = $2
-      ORDER BY st.id
-    """
-    states = []
-    next_id = max(1, max_id - (samples - 1) * stride)
-    while len(states) < samples and next_id <= max_id:
-      row = await connection.fetchrow(query, next_id)
-      if row is None:
-        break
-      snake_rows = await connection.fetch(snake_query, row[9], row[14])
-      state = _build_state(row, snake_rows)
-      if state is not None:
-        states.append(state)
-      next_id = int(row[0]) + stride
-    return states
+    return await sample_states(backend, samples, stride)
   finally:
-    await connection.close()
+    await backend.close()
 
-def load_postgresql_states(dsn: str, samples: int, stride: int) -> list[tuple[dict, dict]]:
-  return asyncio.run(_load_postgresql_states(dsn, samples, stride))
+def load_states(source:str, samples:int, stride:int) -> list[tuple[dict, dict]]:
+  return asyncio.run(_load_states(source, max(1, samples), max(1, stride)))