feat(database): store gameplay replays in compact form
Build and Push Docker Container / build-and-push (push) Successful in 3m52s

- Add compact storage package: codec, records, hydrate, replay blobs, cache.
- Running games write game_turns and snake_states keyed by integer ids.
- Encode moves as 1-byte codes and bodies as head plus step codes.
- Store food and hazards only when they changed since the previous turn.
- Strip reasoning fields that duplicate stored values and restore on read.
- Pack finished games into one zstd blob per game in game_replays.
- Merge late turns into packed blobs during the stale-game sweep.
- Extract SQLite and PostgreSQL schemas into dedicated modules.
- Add read/import game bundle and list_game_ids to the backend template.
- Add legacy readers and converter for the old turns/snake_turns layout.
- Add migrate_gameplay_to_compact.py with in-place or cross-DB migration.
- Verify every migrated game and keep legacy rows on failure.
- Support --since, --minimum-quality and guarded --drop-legacy.
- Add just recipe migrate-gameplay-postgres.
- Rewrite merge script to copy game by game via backend bundles.
- Move benchmark state loading into benchmark_states for both backends.
- Update PostgreSQL cleanup script to the new replay tables.
- Drop sqlite-zstd build stages from the Dockerfile.
- Remove old migration scripts, normalized_turn and their tests.
- Add tests for compact storage, schema, legacy migration and PostgreSQL.
- Document the compact layout and migration workflow in the README.
This commit is contained in:
2026-11-05 00:02:56 +01:00
parent 4403978537
commit 39a7a53873
41 changed files with 2779 additions and 2493 deletions
@@ -0,0 +1,69 @@
"""Read legacy-layout games (turns/snake_turns) from PostgreSQL."""
import json
from server.database.compact.games import GAME_COLUMNS
from server.database.compact.records import GameBundle
from .convert import build_legacy_bundle
class LegacyPostgresqlReader:
def __init__(self, pool):
self._pool = pool
@classmethod
async def connect(cls, dsn:str) -> "LegacyPostgresqlReader":
import asyncpg # noqa: PLC0415
async def _init(conn) -> None:
await conn.set_type_codec("jsonb", encoder=json.dumps, decoder=json.loads, schema="pg_catalog")
await conn.set_type_codec("json", encoder=json.dumps, decoder=json.loads, schema="pg_catalog")
return cls(await asyncpg.create_pool(dsn=dsn, min_size=1, max_size=4, init=_init))
async def close(self) -> None:
await self._pool.close()
async def has_legacy_tables(self) -> bool:
async with self._pool.acquire() as conn:
return bool(await conn.fetchval(
"SELECT to_regclass('public.turns') IS NOT NULL AND to_regclass('public.snake_turns') IS NOT NULL"
))
async def game_ids(self, since:str|None=None) -> list[str]:
async with self._pool.acquire() as conn:
if since:
rows = await conn.fetch("""
SELECT game_id FROM games WHERE started_at >= $1::timestamptz
ORDER BY started_at, game_id
""", since)
else:
rows = await conn.fetch("SELECT game_id FROM games ORDER BY started_at, game_id")
return [row["game_id"] for row in rows]
async def read_bundle(self, game_id:str) -> GameBundle|None:
async with self._pool.acquire() as conn:
game = await conn.fetchrow(f"SELECT {', '.join(GAME_COLUMNS)} FROM games WHERE game_id = $1", game_id)
if game is None:
return None
identities = await conn.fetch("""
SELECT snake_id, snake_name, is_you, customizations
FROM game_snakes WHERE game_id = $1
""", game_id)
turns = await conn.fetch("""
SELECT turn, observed_at, my_move, my_thinking, board_state, food, hazards
FROM turns WHERE game_id = $1
""", game_id)
snakes = await conn.fetch("""
SELECT snake_id, turn, snake_name, health, length, head_x, head_y, body, is_you, latency
FROM snake_turns WHERE game_id = $1
""", game_id)
return build_legacy_bundle(
dict(game), [dict(row) for row in identities],
[dict(row) for row in turns], [dict(row) for row in snakes],
)
async def turn_counts(self) -> dict[str, int]:
async with self._pool.acquire() as conn:
rows = await conn.fetch("SELECT game_id, COUNT(*) AS turns FROM turns GROUP BY game_id")
return {row["game_id"]: int(row["turns"]) for row in rows}