feat(database): store gameplay replays in compact form

- Add compact storage package: codec, records, hydrate, replay blobs, cache.
- Running games write game_turns and snake_states keyed by integer ids.
- Encode moves as 1-byte codes and bodies as head plus step codes.
- Store food and hazards only when they changed since the previous turn.
- Strip reasoning fields that duplicate stored values and restore on read.
- Pack finished games into one zstd blob per game in game_replays.
- Merge late turns into packed blobs during the stale-game sweep.
- Extract SQLite and PostgreSQL schemas into dedicated modules.
- Add read/import game bundle and list_game_ids to the backend template.
- Add legacy readers and converter for the old turns/snake_turns layout.
- Add migrate_gameplay_to_compact.py with in-place or cross-DB migration.
- Verify every migrated game and keep legacy rows on failure.
- Support --since, --minimum-quality and guarded --drop-legacy.
- Add just recipe migrate-gameplay-postgres.
- Rewrite merge script to copy game by game via backend bundles.
- Move benchmark state loading into benchmark_states for both backends.
- Update PostgreSQL cleanup script to the new replay tables.
- Drop sqlite-zstd build stages from the Dockerfile.
- Remove old migration scripts, normalized_turn and their tests.
- Add tests for compact storage, schema, legacy migration and PostgreSQL.
- Document the compact layout and migration workflow in the README.

server/database/compact/__init__.py

@@ -0,0 +1 @@
+"""Compact gameplay storage: codecs, records, replay blobs and hydration."""

server/database/compact/cache.py

@@ -0,0 +1,33 @@
+"""Per-process cache of running games to avoid repeated lookups per turn."""
+
+from collections import OrderedDict
+from dataclasses import dataclass, field
+
+from .records import PointsState
+
+@dataclass
+class RunningGame:
+  gid:int
+  snake_ids:dict[str, int] = field(default_factory=dict)
+  points:PointsState|None = None
+
+class RunningGameCache:
+  def __init__(self, max_games:int=512):
+    self._games:OrderedDict[str, RunningGame] = OrderedDict()
+    self._max_games = max(1, max_games)
+
+  def get(self, game_id:str) -> RunningGame|None:
+    game = self._games.get(game_id)
+    if game is not None:
+      self._games.move_to_end(game_id)
+    return game
+
+  def put(self, game_id:str, game:RunningGame) -> RunningGame:
+    self._games[game_id] = game
+    self._games.move_to_end(game_id)
+    while len(self._games) > self._max_games:
+      self._games.popitem(last=False)
+    return game
+
+  def drop(self, game_id:str) -> None:
+    self._games.pop(game_id, None)

server/database/compact/codec.py

@@ -0,0 +1,146 @@
+"""Byte-level encodings for compact gameplay storage.
+
+All encodings are lossless. Values that do not fit the compact form (for
+example coordinates outside 0..255) fall back to a JSON payload, marked by the
+leading format byte, so unusual maps can never corrupt stored data.
+"""
+
+from datetime import datetime, timezone
+import json
+
+MOVES = ("up", "down", "left", "right")
+_MOVE_CODES = {move: index + 1 for index, move in enumerate(MOVES)}
+
+_FORMAT_JSON = 0
+_FORMAT_COMPACT = 1
+
+# Body step codes relative to the previous segment.
+_SAME, _UP, _DOWN, _LEFT, _RIGHT, _ABSOLUTE = range(6)
+_STEPS = {(0, 0): _SAME, (0, 1): _UP, (0, -1): _DOWN, (-1, 0): _LEFT, (1, 0): _RIGHT}
+_STEP_DELTAS = {code: delta for delta, code in _STEPS.items()}
+
+def encode_move(move:str|None) -> int|None:
+  return _MOVE_CODES.get(move) if move is not None else None
+
+def decode_move(code:int|None) -> str|None:
+  if not code:
+    return None
+  return MOVES[int(code) - 1]
+
+def encode_latency(latency:object) -> int|None:
+  if latency is None or latency == "":
+    return None
+  try:
+    value = int(str(latency))
+  except ValueError:
+    return None
+  return value if -32768 <= value <= 32767 else None
+
+def decode_latency(value:int|None) -> str|None:
+  return None if value is None else str(value)
+
+def _point(point:object) -> tuple[int, int]|None:
+  if not isinstance(point, dict):
+    return None
+  x, y = point.get("x"), point.get("y")
+  if type(x) is not int or type(y) is not int:
+    return None
+  return x, y
+
+def _fits_byte(*values:int) -> bool:
+  return all(0 <= value <= 255 for value in values)
+
+def _json_fallback(value:object) -> bytes:
+  return bytes([_FORMAT_JSON]) + json.dumps(value, separators=(",", ":")).encode()
+
+def encode_body(body:list) -> bytes:
+  """Head as absolute byte pair, then one step code per following segment."""
+  points = [_point(segment) for segment in body]
+  if any(point is None for point in points) or not all(_fits_byte(*point) for point in points):
+    return _json_fallback(body)
+
+  output = bytearray([_FORMAT_COMPACT])
+  previous = None
+  for x, y in points:
+    if previous is None:
+      output += bytes((x, y))
+    else:
+      step = _STEPS.get((x - previous[0], y - previous[1]))
+      output += bytes((step,)) if step is not None else bytes((_ABSOLUTE, x, y))
+    previous = (x, y)
+  return bytes(output)
+
+def decode_body(data:bytes|memoryview|None) -> list[dict]:
+  if not data:
+    return []
+  data = bytes(data)
+  if data[0] == _FORMAT_JSON:
+    return json.loads(data[1:])
+
+  body = []
+  index = 1
+  if len(data) >= 3:
+    x, y = data[1], data[2]
+    body.append({"x": x, "y": y})
+    index = 3
+  while index < len(data):
+    code = data[index]
+    if code == _ABSOLUTE:
+      x, y = data[index + 1], data[index + 2]
+      index += 3
+    else:
+      dx, dy = _STEP_DELTAS[code]
+      x, y = x + dx, y + dy
+      index += 1
+    body.append({"x": x, "y": y})
+  return body
+
+def encode_points(points:list|None) -> bytes|None:
+  """Food/hazard lists as byte pairs. ``None`` means "unchanged"."""
+  if points is None:
+    return None
+  pairs = [_point(point) for point in points]
+  if any(pair is None for pair in pairs) or not all(_fits_byte(*pair) for pair in pairs):
+    return _json_fallback(points)
+  output = bytearray([_FORMAT_COMPACT])
+  for x, y in pairs:
+    output += bytes((x, y))
+  return bytes(output)
+
+def decode_points(data:bytes|memoryview|None) -> list[dict]|None:
+  if data is None:
+    return None
+  data = bytes(data)
+  if not data:
+    return []
+  if data[0] == _FORMAT_JSON:
+    return json.loads(data[1:])
+  return [{"x": data[i], "y": data[i + 1]} for i in range(1, len(data) - 1, 2)]
+
+def flatten_points(points:list[dict]) -> list:
+  """Compact JSON form used inside replay blobs: [x0, y0, x1, y1, ...]."""
+  pairs = [_point(point) for point in points]
+  if any(pair is None for pair in pairs):
+    return [points]
+  return [value for pair in pairs for value in pair]
+
+def unflatten_points(values:list) -> list[dict]:
+  if len(values) == 1 and isinstance(values[0], list):
+    return values[0]
+  return [{"x": values[i], "y": values[i + 1]} for i in range(0, len(values), 2)]
+
+def to_epoch_us(value:datetime|str|None) -> int|None:
+  if value is None:
+    return None
+  if isinstance(value, str):
+    value = datetime.fromisoformat(value.replace("Z", "+00:00"))
+  if value.tzinfo is None:
+    value = value.replace(tzinfo=timezone.utc)
+  delta = value - datetime(1970, 1, 1, tzinfo=timezone.utc)
+  return (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds
+
+def from_epoch_us(value:int|None) -> datetime|None:
+  if value is None:
+    return None
+  seconds, micros = divmod(int(value), 1_000_000)
+  return datetime.fromtimestamp(seconds, tz=timezone.utc).replace(microsecond=micros)

server/database/compact/games.py

@@ -0,0 +1,28 @@
+"""Column list of the ``games`` table shared by backends and migrations."""
+
+GAME_COLUMNS = (
+  "game_id", "started_at", "ended_at", "width", "height", "source", "map_name",
+  "ruleset_name", "ruleset_version", "your_snake_id", "your_snake_name",
+  "your_snake_type", "your_snake_version", "game_type", "winner_name",
+  "winner_you", "final_turn", "status", "has_replay", "quality_status",
+  "quality_score", "quality_tier", "quality_reasons",
+)
+
+GAME_DEFAULTS = {
+  "winner_you": False,
+  "final_turn": 0,
+  "status": "running",
+  "has_replay": True,
+  "quality_status": "retained",
+}
+
+def normalize_game(row:dict) -> dict:
+  """Return a dict with every games column, filling legacy gaps with defaults."""
+  game = {column: row.get(column, GAME_DEFAULTS.get(column)) for column in GAME_COLUMNS}
+  for column, default in GAME_DEFAULTS.items():
+    if game[column] is None:
+      game[column] = default
+  game["winner_you"] = bool(game["winner_you"])
+  game["has_replay"] = bool(game["has_replay"])
+  game["final_turn"] = int(game["final_turn"] or 0)
+  return game

server/database/compact/hydrate.py

@@ -0,0 +1,100 @@
+"""Rebuild the dashboard replay payload from compact records."""
+
+from .codec import decode_latency
+from .records import SnakeIdentity, SnakeState, TurnRecord, you_view
+from .thinking import derived_values, restore_thinking
+
+_DIRECTIONS = {(1, 0): "right", (-1, 0): "left", (0, 1): "up", (0, -1): "down"}
+
+def _head(state:SnakeState) -> dict|None:
+  return state.body[0] if state.body else None
+
+def _inferred_move(previous:SnakeState|None, current:SnakeState) -> str|None:
+  if previous is None:
+    return None
+  old, new = _head(previous), _head(current)
+  if old is None or new is None:
+    return None
+  return _DIRECTIONS.get((new["x"] - old["x"], new["y"] - old["y"]))
+
+def _sort_key(identity:SnakeIdentity|None):
+  name = identity.name if identity else None
+  is_you = bool(identity and identity.is_you)
+  return (not is_you, name is None, name or "")
+
+def hydrate_replay_turns(
+  game:dict,
+  identities:dict[str, SnakeIdentity],
+  turns:list[TurnRecord],
+  states:list[SnakeState],
+) -> list[dict]:
+  states_by_turn:dict[int, dict[str, SnakeState]] = {}
+  for state in states:
+    states_by_turn.setdefault(state.turn, {})[state.snake_id] = state
+
+  width, height = game.get("width"), game.get("height")
+  you_id = game.get("your_snake_id")
+  food:list = []
+  hazards:list = []
+  output = []
+  for record in sorted(turns, key=lambda item: item.turn):
+    food = food if record.food is None else record.food
+    hazards = hazards if record.hazards is None else record.hazards
+    current = states_by_turn.get(record.turn, {})
+    previous = states_by_turn.get(record.turn - 1, {})
+    ordered = sorted(current.values(), key=lambda s: _sort_key(identities.get(s.snake_id)))
+
+    snakes = []
+    api_snakes = []
+    for state in ordered:
+      identity = identities.get(state.snake_id)
+      name = identity.name if identity else None
+      customizations = identity.customizations if identity else {}
+      latency = decode_latency(state.latency)
+      api_snake = {
+        "id": state.snake_id,
+        "name": name,
+        "health": state.health,
+        "length": len(state.body),
+        "head": _head(state) or {"x": None, "y": None},
+        "body": state.body,
+        "customizations": customizations,
+      }
+      if latency is not None:
+        api_snake["latency"] = latency
+      api_snakes.append(api_snake)
+      snakes.append({
+        "snake_id": state.snake_id,
+        "snake_name": name,
+        "health": state.health,
+        "length": len(state.body),
+        "head": api_snake["head"],
+        "body": state.body,
+        "customizations": customizations,
+        "is_you": bool(identity and identity.is_you),
+        "inferred_move": _inferred_move(previous.get(state.snake_id), state),
+        "latency": latency,
+      })
+
+    derived = derived_values(
+      record.turn, record.move, you_view(list(current.values()), you_id),
+      food, width, height, len(current),
+    )
+    output.append({
+      "turn": record.turn,
+      "observed_at": record.observed_at,
+      "my_move": record.move,
+      "my_thinking": restore_thinking(record.thinking, derived),
+      "board": {
+        "width": width,
+        "height": height,
+        "food": food,
+        "hazards": hazards,
+        "snakes": api_snakes,
+      },
+      "food": food,
+      "hazards": hazards,
+      "you": next((snake for snake in api_snakes if snake["id"] == you_id), {}),
+      "snakes": snakes,
+    })
+  return output

server/database/compact/quality.py

@@ -0,0 +1,23 @@
+"""Rate a game's replay quality from its records."""
+
+from server.database.game_quality import GameQuality, GameQualityInput, rate_game_quality
+
+from .codec import MOVES
+from .records import GameBundle
+
+def rate_bundle(bundle:GameBundle) -> GameQuality:
+  turns = bundle.turns
+  numbers = [record.turn for record in turns]
+  moves = [record.move for record in turns]
+  return rate_game_quality(GameQualityInput(
+    status=bundle.game.get("status") or "running",
+    final_turn=int(bundle.game.get("final_turn") or 0),
+    turn_rows=len(turns),
+    min_turn=min(numbers) if numbers else None,
+    max_turn=max(numbers) if numbers else None,
+    valid_moves=sum(1 for move in moves if move in MOVES),
+    thinking_rows=sum(1 for record in turns if record.thinking not in (None, {})),
+    distinct_moves=len({move for move in moves if move is not None}),
+    snake_turn_rows=len(bundle.states),
+    winner_name=bundle.game.get("winner_name"),
+  ))

server/database/compact/records.py

@@ -0,0 +1,136 @@
+"""In-memory records shared by both backends, the replay blob and migrations.
+
+A ``TurnRecord`` is already in compact form: ``thinking`` has derivable fields
+stripped, and ``food``/``hazards`` are ``None`` when unchanged since the
+previous stored turn.
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+
+from .codec import encode_latency
+from .thinking import derived_values, slim_thinking
+
+@dataclass(frozen=True)
+class SnakeIdentity:
+  snake_id:str
+  name:str|None
+  is_you:bool
+  customizations:dict = field(default_factory=dict)
+
+@dataclass(frozen=True)
+class SnakeState:
+  snake_id:str
+  turn:int
+  health:int|None
+  latency:int|None
+  body:list
+
+@dataclass(frozen=True)
+class TurnRecord:
+  turn:int
+  observed_at:datetime|None
+  move:str|None
+  thinking:object
+  food:list|None
+  hazards:list|None
+
+@dataclass(frozen=True)
+class PointsState:
+  """Last stored food/hazards, used to store only changes."""
+  turn:int
+  food:list
+  hazards:list
+
+@dataclass
+class GameBundle:
+  game:dict
+  snakes:list[SnakeIdentity]
+  turns:list[TurnRecord]
+  states:list[SnakeState]
+
+def you_view(states:list[SnakeState], you_id:str|None) -> dict|None:
+  state = next((s for s in states if s.snake_id == you_id), None)
+  if state is None:
+    return None
+  return {
+    "head": state.body[0] if state.body else None,
+    "health": state.health,
+    "length": len(state.body),
+  }
+
+def compact_turn(
+  turn:int,
+  observed_at:datetime|None,
+  move:str|None,
+  thinking:object,
+  food:list,
+  hazards:list,
+  states:list[SnakeState],
+  you_id:str|None,
+  width:int|None,
+  height:int|None,
+  previous:PointsState|None,
+) -> tuple[TurnRecord, PointsState]:
+  derived = derived_values(turn, move, you_view(states, you_id), food, width, height, len(states))
+  unchanged = previous is not None and previous.turn == turn - 1
+  record = TurnRecord(
+    turn=turn,
+    observed_at=observed_at,
+    move=move,
+    thinking=slim_thinking(thinking, derived),
+    food=None if unchanged and previous.food == food else food,
+    hazards=None if unchanged and previous.hazards == hazards else hazards,
+  )
+  return record, PointsState(turn, food, hazards)
+
+def records_from_game_state(
+  game_state:dict,
+  my_move:str|None,
+  thinking:object,
+  previous:PointsState|None,
+) -> tuple[TurnRecord, list[SnakeState], list[SnakeIdentity], PointsState]:
+  board = game_state.get("board", {})
+  you_id = game_state.get("you", {}).get("id")
+  turn = int(game_state.get("turn", 0))
+  identities = []
+  states = []
+  for snake in board.get("snakes", []):
+    snake_id = snake.get("id")
+    if snake_id is None:
+      continue
+    identities.append(SnakeIdentity(
+      snake_id=snake_id,
+      name=snake.get("name"),
+      is_you=snake_id == you_id,
+      customizations=snake.get("customizations") or {},
+    ))
+    states.append(SnakeState(
+      snake_id=snake_id,
+      turn=turn,
+      health=snake.get("health"),
+      latency=encode_latency(snake.get("latency")),
+      body=list(snake.get("body", [])),
+    ))
+  record, points = compact_turn(
+    turn, datetime.now(timezone.utc), my_move, thinking,
+    list(board.get("food", [])), list(board.get("hazards", [])),
+    states, you_id, board.get("width"), board.get("height"), previous,
+  )
+  return record, states, identities, points
+
+def merge_records(
+  base_turns:list[TurnRecord],
+  base_states:list[SnakeState],
+  turns:list[TurnRecord],
+  states:list[SnakeState],
+) -> tuple[list[TurnRecord], list[SnakeState]]:
+  """Combine a packed replay with rows written after it was packed."""
+  merged_turns = {record.turn: record for record in base_turns}
+  merged_turns.update({record.turn: record for record in turns})
+  merged_states = {(state.snake_id, state.turn): state for state in base_states}
+  merged_states.update({(state.snake_id, state.turn): state for state in states})
+  return (
+    sorted(merged_turns.values(), key=lambda record: record.turn),
+    sorted(merged_states.values(), key=lambda state: (state.turn, state.snake_id)),
+  )

server/database/compact/replay_blob.py

@@ -0,0 +1,63 @@
+"""Pack a finished game's turns into one zstd-compressed blob.
+
+Consecutive turns are nearly identical, so compressing a whole game at once is
+far smaller than storing one row per turn and snake.
+"""
+
+from compression import zstd
+import json
+
+from .codec import (
+  decode_move, encode_move, flatten_points, from_epoch_us, to_epoch_us,
+  unflatten_points,
+)
+from .records import SnakeState, TurnRecord
+
+REPLAY_FORMAT = 1
+_LEVEL = 10
+
+def _optional_points(points:list|None) -> list|None:
+  return None if points is None else flatten_points(points)
+
+def _optional_unflatten(values:list|None) -> list|None:
+  return None if values is None else unflatten_points(values)
+
+def pack_replay(turns:list[TurnRecord], states:list[SnakeState]) -> bytes:
+  by_snake:dict[str, list] = {}
+  for state in sorted(states, key=lambda s: (s.snake_id, s.turn)):
+    by_snake.setdefault(state.snake_id, []).append(
+      [state.turn, state.health, state.latency, flatten_points(state.body)]
+    )
+  payload = {
+    "t": [
+      [
+        record.turn, to_epoch_us(record.observed_at), encode_move(record.move),
+        record.thinking, _optional_points(record.food), _optional_points(record.hazards),
+      ]
+      for record in sorted(turns, key=lambda item: item.turn)
+    ],
+    "s": by_snake,
+  }
+  raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode()
+  return zstd.compress(raw, level=_LEVEL)
+
+def unpack_replay(data:bytes|memoryview, format_version:int=REPLAY_FORMAT) -> tuple[list[TurnRecord], list[SnakeState]]:
+  if format_version != REPLAY_FORMAT:
+    raise ValueError(f"Unsupported replay format: {format_version}")
+  payload = json.loads(zstd.decompress(bytes(data)))
+  turns = [
+    TurnRecord(
+      turn=turn, observed_at=from_epoch_us(observed), move=decode_move(move),
+      thinking=thinking, food=_optional_unflatten(food), hazards=_optional_unflatten(hazards),
+    )
+    for turn, observed, move, thinking, food, hazards in payload["t"]
+  ]
+  states = [
+    SnakeState(
+      snake_id=snake_id, turn=turn, health=health, latency=latency,
+      body=unflatten_points(body),
+    )
+    for snake_id, rows in payload["s"].items()
+    for turn, health, latency, body in rows
+  ]
+  return turns, states

server/database/compact/thinking.py

@@ -0,0 +1,57 @@
+"""Strip move-reasoning fields that can be rebuilt from the stored turn.
+
+Snakes put values like ``turn``, ``health`` or ``head`` into their reasoning
+payload although the same values are stored in the turn and snake rows. Such a
+field is only removed when it equals the derived value, and a bitmask records
+which fields were removed, so restoring is exact.
+"""
+
+DERIVED_KEYS = ("turn", "move", "head", "health", "length", "food_count", "board_size", "snakes")
+MASK_KEY = "_derived"
+_RAW_KEY = "_raw"
+
+def derived_values(turn:int, move:str|None, you:dict|None, food:list, width:int|None, height:int|None, snake_count:int) -> dict:
+  you = you or {}
+  return {
+    "turn": turn,
+    "move": move,
+    "head": you.get("head"),
+    "health": you.get("health"),
+    "length": you.get("length"),
+    "food_count": len(food),
+    "board_size": f"{width}x{height}",
+    "snakes": snake_count,
+  }
+
+def _same(stored:object, derived:object) -> bool:
+  # bool is an int subclass; never treat True == 1 as derivable.
+  return type(stored) is type(derived) and stored == derived
+
+def slim_thinking(thinking:object, derived:dict) -> object:
+  if not isinstance(thinking, dict):
+    return thinking
+  if MASK_KEY in thinking:
+    # A payload that already uses the marker key is stored verbatim.
+    return {MASK_KEY: 0, _RAW_KEY: thinking}
+  mask = 0
+  slim = {}
+  for key, value in thinking.items():
+    if key in DERIVED_KEYS and _same(value, derived[key]):
+      mask |= 1 << DERIVED_KEYS.index(key)
+    else:
+      slim[key] = value
+  if mask:
+    slim[MASK_KEY] = mask
+  return slim
+
+def restore_thinking(thinking:object, derived:dict) -> object:
+  if not isinstance(thinking, dict) or MASK_KEY not in thinking:
+    return thinking
+  mask = int(thinking[MASK_KEY])
+  if mask == 0 and _RAW_KEY in thinking:
+    return thinking[_RAW_KEY]
+  restored = {key: value for key, value in thinking.items() if key != MASK_KEY}
+  for index, key in enumerate(DERIVED_KEYS):
+    if mask & (1 << index):
+      restored[key] = derived[key]
+  return restored