feat(database): store gameplay replays in compact form
Build and Push Docker Container / build-and-push (push) Successful in 3m52s
Build and Push Docker Container / build-and-push (push) Successful in 3m52s
- Add compact storage package: codec, records, hydrate, replay blobs, cache. - Running games write game_turns and snake_states keyed by integer ids. - Encode moves as 1-byte codes and bodies as head plus step codes. - Store food and hazards only when they changed since the previous turn. - Strip reasoning fields that duplicate stored values and restore on read. - Pack finished games into one zstd blob per game in game_replays. - Merge late turns into packed blobs during the stale-game sweep. - Extract SQLite and PostgreSQL schemas into dedicated modules. - Add read/import game bundle and list_game_ids to the backend template. - Add legacy readers and converter for the old turns/snake_turns layout. - Add migrate_gameplay_to_compact.py with in-place or cross-DB migration. - Verify every migrated game and keep legacy rows on failure. - Support --since, --minimum-quality and guarded --drop-legacy. - Add just recipe migrate-gameplay-postgres. - Rewrite merge script to copy game by game via backend bundles. - Move benchmark state loading into benchmark_states for both backends. - Update PostgreSQL cleanup script to the new replay tables. - Drop sqlite-zstd build stages from the Dockerfile. - Remove old migration scripts, normalized_turn and their tests. - Add tests for compact storage, schema, legacy migration and PostgreSQL. - Document the compact layout and migration workflow in the README.
This commit is contained in:
@@ -0,0 +1,69 @@
|
||||
"""Read legacy-layout games (turns/snake_turns) from PostgreSQL."""
|
||||
|
||||
import json
|
||||
|
||||
from server.database.compact.games import GAME_COLUMNS
|
||||
from server.database.compact.records import GameBundle
|
||||
|
||||
from .convert import build_legacy_bundle
|
||||
|
||||
class LegacyPostgresqlReader:
|
||||
def __init__(self, pool):
|
||||
self._pool = pool
|
||||
|
||||
@classmethod
|
||||
async def connect(cls, dsn:str) -> "LegacyPostgresqlReader":
|
||||
import asyncpg # noqa: PLC0415
|
||||
|
||||
async def _init(conn) -> None:
|
||||
await conn.set_type_codec("jsonb", encoder=json.dumps, decoder=json.loads, schema="pg_catalog")
|
||||
await conn.set_type_codec("json", encoder=json.dumps, decoder=json.loads, schema="pg_catalog")
|
||||
|
||||
return cls(await asyncpg.create_pool(dsn=dsn, min_size=1, max_size=4, init=_init))
|
||||
|
||||
async def close(self) -> None:
|
||||
await self._pool.close()
|
||||
|
||||
async def has_legacy_tables(self) -> bool:
|
||||
async with self._pool.acquire() as conn:
|
||||
return bool(await conn.fetchval(
|
||||
"SELECT to_regclass('public.turns') IS NOT NULL AND to_regclass('public.snake_turns') IS NOT NULL"
|
||||
))
|
||||
|
||||
async def game_ids(self, since:str|None=None) -> list[str]:
|
||||
async with self._pool.acquire() as conn:
|
||||
if since:
|
||||
rows = await conn.fetch("""
|
||||
SELECT game_id FROM games WHERE started_at >= $1::timestamptz
|
||||
ORDER BY started_at, game_id
|
||||
""", since)
|
||||
else:
|
||||
rows = await conn.fetch("SELECT game_id FROM games ORDER BY started_at, game_id")
|
||||
return [row["game_id"] for row in rows]
|
||||
|
||||
async def read_bundle(self, game_id:str) -> GameBundle|None:
|
||||
async with self._pool.acquire() as conn:
|
||||
game = await conn.fetchrow(f"SELECT {', '.join(GAME_COLUMNS)} FROM games WHERE game_id = $1", game_id)
|
||||
if game is None:
|
||||
return None
|
||||
identities = await conn.fetch("""
|
||||
SELECT snake_id, snake_name, is_you, customizations
|
||||
FROM game_snakes WHERE game_id = $1
|
||||
""", game_id)
|
||||
turns = await conn.fetch("""
|
||||
SELECT turn, observed_at, my_move, my_thinking, board_state, food, hazards
|
||||
FROM turns WHERE game_id = $1
|
||||
""", game_id)
|
||||
snakes = await conn.fetch("""
|
||||
SELECT snake_id, turn, snake_name, health, length, head_x, head_y, body, is_you, latency
|
||||
FROM snake_turns WHERE game_id = $1
|
||||
""", game_id)
|
||||
return build_legacy_bundle(
|
||||
dict(game), [dict(row) for row in identities],
|
||||
[dict(row) for row in turns], [dict(row) for row in snakes],
|
||||
)
|
||||
|
||||
async def turn_counts(self) -> dict[str, int]:
|
||||
async with self._pool.acquire() as conn:
|
||||
rows = await conn.fetch("SELECT game_id, COUNT(*) AS turns FROM turns GROUP BY game_id")
|
||||
return {row["game_id"]: int(row["turns"]) for row in rows}
|
||||
Reference in New Issue
Block a user