feat(database): store gameplay replays in compact form
Build and Push Docker Container / build-and-push (push) Successful in 3m52s
Build and Push Docker Container / build-and-push (push) Successful in 3m52s
- Add compact storage package: codec, records, hydrate, replay blobs, cache. - Running games write game_turns and snake_states keyed by integer ids. - Encode moves as 1-byte codes and bodies as head plus step codes. - Store food and hazards only when they changed since the previous turn. - Strip reasoning fields that duplicate stored values and restore on read. - Pack finished games into one zstd blob per game in game_replays. - Merge late turns into packed blobs during the stale-game sweep. - Extract SQLite and PostgreSQL schemas into dedicated modules. - Add read/import game bundle and list_game_ids to the backend template. - Add legacy readers and converter for the old turns/snake_turns layout. - Add migrate_gameplay_to_compact.py with in-place or cross-DB migration. - Verify every migrated game and keep legacy rows on failure. - Support --since, --minimum-quality and guarded --drop-legacy. - Add just recipe migrate-gameplay-postgres. - Rewrite merge script to copy game by game via backend bundles. - Move benchmark state loading into benchmark_states for both backends. - Update PostgreSQL cleanup script to the new replay tables. - Drop sqlite-zstd build stages from the Dockerfile. - Remove old migration scripts, normalized_turn and their tests. - Add tests for compact storage, schema, legacy migration and PostgreSQL. - Document the compact layout and migration workflow in the README.
This commit is contained in:
@@ -1,10 +1,8 @@
|
||||
"""PostgreSQL gameplay backend using asyncpg.
|
||||
|
||||
JSON columns use the JSONB type so PostgreSQL stores them in a binary,
|
||||
decomposed format and automatically compresses large values via TOAST
|
||||
(Oversized-Attribute Storage Technique). No application-level
|
||||
serialisation/deserialisation round-trip is needed for reads — asyncpg
|
||||
decodes JSONB rows directly into Python dicts/lists.
|
||||
Running games are stored as compact rows (``game_turns``/``snake_states``).
|
||||
When a game ends it is packed into one zstd blob in ``game_replays`` and its
|
||||
rows are removed. See ``server.database.compact`` for the encodings.
|
||||
|
||||
Connection: pass a DSN via the `dsn` constructor argument, e.g.
|
||||
postgresql://user:password@host:5432/dbname
|
||||
@@ -12,13 +10,22 @@ Connection: pass a DSN via the `dsn` constructor argument, e.g.
|
||||
or set GAMEPLAY_DB_PG_DSN in the environment.
|
||||
"""
|
||||
|
||||
import asyncio, logging, sqlite3, json, sys
|
||||
import json, logging, sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse, urlunparse
|
||||
|
||||
from .Template import GameplayBackendTemplate
|
||||
from server.database.normalized_turn import compact_turn_json
|
||||
from .postgresql_schema import DROP_REDUNDANT_INDEXES, TABLES_DDL
|
||||
from server.database.compact.cache import RunningGame, RunningGameCache
|
||||
from server.database.compact.codec import (
|
||||
decode_body, decode_move, decode_points, encode_body, encode_move, encode_points,
|
||||
)
|
||||
from server.database.compact.games import GAME_COLUMNS, normalize_game
|
||||
from server.database.compact.records import (
|
||||
GameBundle, SnakeIdentity, SnakeState, TurnRecord, merge_records, records_from_game_state,
|
||||
)
|
||||
from server.database.compact.replay_blob import REPLAY_FORMAT, pack_replay, unpack_replay
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
if not logger.handlers:
|
||||
@@ -27,106 +34,46 @@ if not logger.handlers:
|
||||
logger.addHandler(_handler)
|
||||
logger.propagate = False
|
||||
|
||||
# DDL --------------------------------------------------------------------- #
|
||||
_SCHEMA_LOCK_ID = 7_315_002
|
||||
|
||||
_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS games (
|
||||
game_id TEXT PRIMARY KEY,
|
||||
started_at TIMESTAMPTZ NOT NULL,
|
||||
ended_at TIMESTAMPTZ,
|
||||
width INTEGER,
|
||||
height INTEGER,
|
||||
source TEXT,
|
||||
map_name TEXT,
|
||||
ruleset_name TEXT,
|
||||
ruleset_version TEXT,
|
||||
your_snake_id TEXT,
|
||||
your_snake_name TEXT,
|
||||
your_snake_type TEXT,
|
||||
your_snake_version TEXT,
|
||||
game_type TEXT,
|
||||
winner_name TEXT,
|
||||
winner_you BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
final_turn INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL DEFAULT 'running',
|
||||
has_replay BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
quality_status TEXT NOT NULL DEFAULT 'retained',
|
||||
quality_score INTEGER,
|
||||
quality_tier TEXT,
|
||||
quality_reasons JSONB
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS turns (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
game_id TEXT NOT NULL REFERENCES games(game_id) ON DELETE CASCADE,
|
||||
turn INTEGER NOT NULL,
|
||||
observed_at TIMESTAMPTZ NOT NULL,
|
||||
my_move TEXT,
|
||||
my_thinking JSONB,
|
||||
board_state JSONB NOT NULL,
|
||||
snakes JSONB NOT NULL,
|
||||
you JSONB NOT NULL,
|
||||
food JSONB NOT NULL,
|
||||
hazards JSONB NOT NULL,
|
||||
UNIQUE (game_id, turn)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS game_snakes (
|
||||
game_id TEXT NOT NULL REFERENCES games(game_id) ON DELETE CASCADE,
|
||||
snake_id TEXT NOT NULL,
|
||||
snake_name TEXT,
|
||||
is_you BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
customizations JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
PRIMARY KEY (game_id, snake_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS snake_turns (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
game_id TEXT NOT NULL REFERENCES games(game_id) ON DELETE CASCADE,
|
||||
turn INTEGER NOT NULL,
|
||||
snake_id TEXT NOT NULL,
|
||||
snake_name TEXT,
|
||||
health INTEGER,
|
||||
length INTEGER,
|
||||
head_x INTEGER,
|
||||
head_y INTEGER,
|
||||
body JSONB NOT NULL,
|
||||
is_you BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
inferred_move TEXT,
|
||||
latency TEXT,
|
||||
UNIQUE (game_id, turn, snake_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_turns_game_turn ON turns(game_id, turn);
|
||||
CREATE INDEX IF NOT EXISTS idx_games_status ON games(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_snake_turns_game_turn ON snake_turns(game_id, turn);
|
||||
_GAME_SELECT = """
|
||||
SELECT gid, game_id, started_at, ended_at, width, height, source, map_name,
|
||||
ruleset_name, ruleset_version, game_type, your_snake_id, your_snake_name,
|
||||
your_snake_type, your_snake_version, winner_name, winner_you, final_turn,
|
||||
status, has_replay, quality_status, quality_score, quality_tier, quality_reasons
|
||||
FROM games
|
||||
"""
|
||||
|
||||
# Schema evolution: add new columns to existing tables (idempotent).
|
||||
_ALTER_DDL = """
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS game_type TEXT;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS your_snake_type TEXT;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS your_snake_version TEXT;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS winner_name TEXT;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS has_replay BOOLEAN NOT NULL DEFAULT TRUE;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_status TEXT NOT NULL DEFAULT 'retained';
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_score INTEGER;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_tier TEXT;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_reasons JSONB;
|
||||
ALTER TABLE turns ADD COLUMN IF NOT EXISTS my_thinking JSONB;
|
||||
ALTER TABLE snake_turns ADD COLUMN IF NOT EXISTS latency TEXT;
|
||||
ALTER TABLE game_snakes ADD COLUMN IF NOT EXISTS customizations JSONB NOT NULL DEFAULT '{}'::jsonb;
|
||||
_UPSERT_SNAKES = """
|
||||
INSERT INTO game_snakes (game_id, snake_id, snake_name, is_you, customizations)
|
||||
SELECT $1, s.snake_id, s.snake_name, s.is_you, s.customizations::jsonb
|
||||
FROM unnest($2::text[], $3::text[], $4::bool[], $5::text[])
|
||||
AS s(snake_id, snake_name, is_you, customizations)
|
||||
ON CONFLICT (game_id, snake_id) DO UPDATE SET
|
||||
snake_name = EXCLUDED.snake_name,
|
||||
is_you = EXCLUDED.is_you,
|
||||
customizations = EXCLUDED.customizations
|
||||
RETURNING snake_id, gsid
|
||||
"""
|
||||
|
||||
# Force TOAST compression on the large JSONB columns so that even
|
||||
# moderately-sized payloads get compressed on-disk.
|
||||
_TOAST_DDL = """
|
||||
ALTER TABLE turns ALTER COLUMN board_state SET STORAGE EXTENDED;
|
||||
ALTER TABLE turns ALTER COLUMN snakes SET STORAGE EXTENDED;
|
||||
ALTER TABLE turns ALTER COLUMN you SET STORAGE EXTENDED;
|
||||
ALTER TABLE turns ALTER COLUMN food SET STORAGE EXTENDED;
|
||||
ALTER TABLE turns ALTER COLUMN hazards SET STORAGE EXTENDED;
|
||||
ALTER TABLE snake_turns ALTER COLUMN body SET STORAGE EXTENDED;
|
||||
_UPSERT_TURN = """
|
||||
INSERT INTO game_turns (gid, turn, observed_at, my_move, thinking, food, hazards)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (gid, turn) DO UPDATE SET
|
||||
observed_at = EXCLUDED.observed_at,
|
||||
my_move = EXCLUDED.my_move,
|
||||
thinking = EXCLUDED.thinking,
|
||||
food = EXCLUDED.food,
|
||||
hazards = EXCLUDED.hazards
|
||||
"""
|
||||
|
||||
_UPSERT_STATES = """
|
||||
INSERT INTO snake_states (gsid, turn, health, latency, body)
|
||||
SELECT * FROM unnest($1::int[], $2::int[], $3::smallint[], $4::smallint[], $5::bytea[])
|
||||
ON CONFLICT (gsid, turn) DO UPDATE SET
|
||||
health = EXCLUDED.health,
|
||||
latency = EXCLUDED.latency,
|
||||
body = EXCLUDED.body
|
||||
"""
|
||||
|
||||
class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
@@ -142,6 +89,7 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
self._max_size = max_size
|
||||
self._sqlite_migration_path = sqlite_migration_path
|
||||
self._pool = None # asyncpg.Pool, typed at runtime
|
||||
self._running = RunningGameCache()
|
||||
|
||||
# ── DSN normalisation ──────────────────────────────────────────────────────
|
||||
|
||||
@@ -149,25 +97,16 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
|
||||
@classmethod
|
||||
def _ensure_db_name(cls, dsn:str) -> str:
|
||||
"""Return *dsn* with a database name appended when none is present.
|
||||
|
||||
A DSN has no database name when its path component is empty or ``/``.
|
||||
In that case ``battlesnake`` is appended so asyncpg gets a complete
|
||||
connection string without the caller having to remember to add one.
|
||||
"""
|
||||
"""Return *dsn* with a database name appended when none is present."""
|
||||
parsed = urlparse(dsn)
|
||||
db = parsed.path.lstrip("/")
|
||||
if db:
|
||||
if parsed.path.lstrip("/"):
|
||||
return dsn
|
||||
|
||||
new_path = f"/{cls._DEFAULT_DB_NAME}"
|
||||
return urlunparse(parsed._replace(path=new_path))
|
||||
return urlunparse(parsed._replace(path=f"/{cls._DEFAULT_DB_NAME}"))
|
||||
|
||||
# ── pool / schema ──────────────────────────────────────────────────────────
|
||||
|
||||
async def initialize(self) -> None:
|
||||
"""Eagerly create the connection pool on startup so schema init and
|
||||
SQLite migration run immediately rather than on the first game request."""
|
||||
"""Eagerly create the pool so schema init and legacy SQLite import run on startup."""
|
||||
await self._get_pool()
|
||||
|
||||
async def _get_pool(self):
|
||||
@@ -194,27 +133,22 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
init=_init_conn,
|
||||
)
|
||||
await self._initialize_schema()
|
||||
await self._maybe_migrate_from_sqlite()
|
||||
await self._maybe_import_legacy_sqlite()
|
||||
return self._pool
|
||||
|
||||
async def _ensure_database_exists(self, asyncpg, target_dsn:str) -> None:
|
||||
"""Connect to the postgres maintenance DB and CREATE the target database
|
||||
if it does not already exist. Uses a plain connection (not a pool) so
|
||||
the CREATE DATABASE statement can run outside any transaction."""
|
||||
"""CREATE the target database through the maintenance DB when missing."""
|
||||
parsed = urlparse(target_dsn)
|
||||
db_name = parsed.path.lstrip("/")
|
||||
maintenance_dsn = urlunparse(parsed._replace(path="/postgres"))
|
||||
try:
|
||||
conn = await asyncpg.connect(dsn=maintenance_dsn)
|
||||
except Exception:
|
||||
# Fall back to connecting without specifying a database — some setups
|
||||
# (e.g. Cloud SQL, managed PG) disallow direct access to 'postgres'.
|
||||
# Some managed setups disallow direct access to 'postgres'.
|
||||
maintenance_dsn = urlunparse(parsed._replace(path=""))
|
||||
conn = await asyncpg.connect(dsn=maintenance_dsn)
|
||||
try:
|
||||
exists = await conn.fetchval(
|
||||
"SELECT 1 FROM pg_database WHERE datname = $1", db_name
|
||||
)
|
||||
exists = await conn.fetchval("SELECT 1 FROM pg_database WHERE datname = $1", db_name)
|
||||
if not exists:
|
||||
await conn.execute(f'CREATE DATABASE "{db_name}"')
|
||||
logger.info(f"PostgreSQL: created database '{db_name}'")
|
||||
@@ -224,199 +158,143 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
async def _initialize_schema(self) -> None:
|
||||
assert self._pool is not None
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.execute(_DDL)
|
||||
await conn.execute(_ALTER_DDL)
|
||||
# TOAST storage hints are idempotent; ignore errors on repeated runs.
|
||||
# Several workers may start at once; serialize DDL.
|
||||
await conn.execute("SELECT pg_advisory_lock($1)", _SCHEMA_LOCK_ID)
|
||||
try:
|
||||
await conn.execute(_TOAST_DDL)
|
||||
except Exception as exc:
|
||||
logger.debug(f"TOAST DDL skipped (likely already set): {exc}")
|
||||
await conn.execute(TABLES_DDL)
|
||||
finally:
|
||||
await conn.execute("SELECT pg_advisory_unlock($1)", _SCHEMA_LOCK_ID)
|
||||
for statement in DROP_REDUNDANT_INDEXES:
|
||||
await conn.execute(statement)
|
||||
|
||||
# ── sqlite migration ───────────────────────────────────────────────────────
|
||||
|
||||
async def _maybe_migrate_from_sqlite(self) -> None:
|
||||
async def _maybe_import_legacy_sqlite(self) -> None:
|
||||
if not self._sqlite_migration_path:
|
||||
return
|
||||
|
||||
src = Path(self._sqlite_migration_path)
|
||||
if not src.exists():
|
||||
source = Path(self._sqlite_migration_path)
|
||||
if not source.exists():
|
||||
return
|
||||
|
||||
logger.info(f"SQLite migration: found {src}, starting migration to PostgreSQL …")
|
||||
from server.database.legacy.migrate import migrate_legacy # noqa: PLC0415
|
||||
from server.database.legacy.reader_sqlite import LegacySqliteReader # noqa: PLC0415
|
||||
|
||||
logger.info(f"Legacy SQLite import: found {source}, importing into PostgreSQL …")
|
||||
try:
|
||||
games, turns, snake_turns = await asyncio.to_thread(self._read_sqlite_data_sync, str(src))
|
||||
await self._insert_migrated_data(games, turns, snake_turns)
|
||||
|
||||
done_path = src.with_suffix(".migrated")
|
||||
src.rename(done_path)
|
||||
|
||||
logger.info(
|
||||
f"SQLite migration complete: {len(games)} games, {len(turns)} turns, "
|
||||
f"{len(snake_turns)} snake_turns migrated. "
|
||||
f"Source file renamed to {done_path.name}"
|
||||
)
|
||||
with LegacySqliteReader(source) as reader:
|
||||
stats = await migrate_legacy(reader, self)
|
||||
if stats.failed:
|
||||
logger.warning(f"Legacy SQLite import incomplete ({stats.failed} games failed); source kept")
|
||||
return
|
||||
done_path = source.with_suffix(".migrated")
|
||||
source.rename(done_path)
|
||||
logger.info(f"Legacy SQLite import complete: {stats.games} games. Source renamed to {done_path.name}")
|
||||
except Exception:
|
||||
logger.exception("SQLite migration failed — PostgreSQL data is untouched, original SQLite file kept")
|
||||
|
||||
def _read_sqlite_data_sync(self, db_path:str) -> tuple[list[sqlite3.Row], list[sqlite3.Row], list[sqlite3.Row]]:
|
||||
conn = sqlite3.connect(db_path, timeout=30, isolation_level=None)
|
||||
conn.row_factory = sqlite3.Row
|
||||
|
||||
try:
|
||||
game_columns = {
|
||||
row["name"] for row in conn.execute("PRAGMA table_info(games)").fetchall()
|
||||
}
|
||||
if "winner_name" in game_columns:
|
||||
winner_name_expression = "winner_name"
|
||||
elif "winner_names_json" in game_columns:
|
||||
winner_name_expression = "winner_names_json"
|
||||
else:
|
||||
winner_name_expression = "NULL"
|
||||
games = conn.execute(f"""
|
||||
SELECT game_id, started_at, ended_at, width, height, source, map_name,
|
||||
ruleset_name, ruleset_version, your_snake_id, your_snake_name,
|
||||
your_snake_type, your_snake_version, game_type,
|
||||
{winner_name_expression} AS winner_name, winner_you, final_turn, status
|
||||
FROM games
|
||||
ORDER BY started_at ASC
|
||||
""").fetchall()
|
||||
turns = conn.execute("""
|
||||
SELECT game_id, turn, observed_at, my_move, my_thinking_json,
|
||||
board_state_json, snakes_json, you_json, food_json, hazards_json
|
||||
FROM turns
|
||||
ORDER BY game_id ASC, turn ASC
|
||||
""").fetchall()
|
||||
snake_turns = conn.execute("""
|
||||
SELECT game_id, turn, snake_id, snake_name, health, length,
|
||||
head_x, head_y, body_json, is_you, inferred_move, latency
|
||||
FROM snake_turns
|
||||
ORDER BY game_id ASC, turn ASC, snake_id ASC
|
||||
""").fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
return games, turns, snake_turns
|
||||
|
||||
def _parse_ts(self, value:str|None) -> datetime|None:
|
||||
"""Parse an ISO-8601 TEXT timestamp from SQLite into a timezone-aware datetime."""
|
||||
ts = self._parse_utc_timestamp(value)
|
||||
return ts # already UTC-aware from base class helper
|
||||
|
||||
def _parse_json(self, value: str|None) -> object:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return json.loads(value)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return None
|
||||
|
||||
def _migrated_winner_name(self, value:str|None) -> str|None:
|
||||
"""Accept both the current winner_name and legacy winner_names_json value."""
|
||||
if not value:
|
||||
return None
|
||||
parsed = self._parse_json(value)
|
||||
if isinstance(parsed, list):
|
||||
return next((str(name) for name in parsed if name), None)
|
||||
return value
|
||||
|
||||
async def _insert_migrated_data(self, games:list, turns:list, snake_turns:list) -> None:
|
||||
assert self._pool is not None
|
||||
async with self._pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
# games ─────────────────────────────────────────────────────────────
|
||||
# winner_name is TEXT — no cast needed.
|
||||
await conn.executemany("""
|
||||
INSERT INTO games (
|
||||
game_id, started_at, ended_at, width, height, source, map_name,
|
||||
ruleset_name, ruleset_version, your_snake_id, your_snake_name,
|
||||
your_snake_type, your_snake_version, game_type,
|
||||
winner_name, winner_you, final_turn, status
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18)
|
||||
ON CONFLICT (game_id) DO NOTHING
|
||||
""",
|
||||
[
|
||||
(
|
||||
row["game_id"],
|
||||
self._parse_ts(row["started_at"]),
|
||||
self._parse_ts(row["ended_at"]),
|
||||
row["width"],
|
||||
row["height"],
|
||||
row["source"],
|
||||
row["map_name"],
|
||||
row["ruleset_name"],
|
||||
row["ruleset_version"],
|
||||
row["your_snake_id"],
|
||||
row["your_snake_name"],
|
||||
row["your_snake_type"],
|
||||
row["your_snake_version"],
|
||||
row["game_type"],
|
||||
self._migrated_winner_name(row["winner_name"]),
|
||||
bool(row["winner_you"]),
|
||||
row["final_turn"],
|
||||
row["status"],
|
||||
)
|
||||
for row in games
|
||||
],
|
||||
)
|
||||
|
||||
await conn.executemany("""
|
||||
INSERT INTO turns (
|
||||
game_id, turn, observed_at, my_move, my_thinking,
|
||||
board_state, snakes, you, food, hazards
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
|
||||
ON CONFLICT (game_id, turn) DO NOTHING
|
||||
""",
|
||||
[
|
||||
(
|
||||
row["game_id"],
|
||||
row["turn"],
|
||||
self._parse_ts(row["observed_at"]),
|
||||
row["my_move"],
|
||||
self._parse_json(row["my_thinking_json"]),
|
||||
self._parse_json(row["board_state_json"]),
|
||||
self._parse_json(row["snakes_json"]),
|
||||
self._parse_json(row["you_json"]),
|
||||
self._parse_json(row["food_json"]),
|
||||
self._parse_json(row["hazards_json"]),
|
||||
)
|
||||
for row in turns
|
||||
],
|
||||
)
|
||||
|
||||
# snake_turns
|
||||
await conn.executemany("""
|
||||
INSERT INTO snake_turns (
|
||||
game_id, turn, snake_id, snake_name, health, length,
|
||||
head_x, head_y, body, is_you, inferred_move, latency
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)
|
||||
ON CONFLICT (game_id, turn, snake_id) DO NOTHING
|
||||
""",
|
||||
[
|
||||
(
|
||||
row["game_id"],
|
||||
row["turn"],
|
||||
row["snake_id"],
|
||||
row["snake_name"],
|
||||
row["health"],
|
||||
row["length"],
|
||||
row["head_x"],
|
||||
row["head_y"],
|
||||
self._parse_json(row["body_json"]),
|
||||
bool(row["is_you"]),
|
||||
row["inferred_move"],
|
||||
row["latency"],
|
||||
)
|
||||
for row in snake_turns
|
||||
],
|
||||
)
|
||||
logger.exception("Legacy SQLite import failed — original SQLite file kept")
|
||||
|
||||
# ── helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
def _utc_now_ts(self) -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
# The pool init callback registers JSON/JSONB codecs so asyncpg automatically
|
||||
# encodes Python dicts/lists on write and decodes them on read.
|
||||
async def _running_game(self, conn, game_id:str) -> RunningGame:
|
||||
game = self._running.get(game_id)
|
||||
if game is None:
|
||||
gid = await conn.fetchval("SELECT gid FROM games WHERE game_id = $1", game_id)
|
||||
if gid is None:
|
||||
raise LookupError(f"Turn for unknown game {game_id!r}")
|
||||
game = RunningGame(gid=gid)
|
||||
return game
|
||||
|
||||
async def _upsert_identities(self, conn, game_id:str, identities:list[SnakeIdentity]) -> dict[str, int]:
|
||||
if not identities:
|
||||
return {}
|
||||
rows = await conn.fetch(
|
||||
_UPSERT_SNAKES, game_id,
|
||||
[identity.snake_id for identity in identities],
|
||||
[identity.name for identity in identities],
|
||||
[identity.is_you for identity in identities],
|
||||
[json.dumps(identity.customizations or {}) for identity in identities],
|
||||
)
|
||||
return {row["snake_id"]: row["gsid"] for row in rows}
|
||||
|
||||
async def _write_rows(self, conn, gid:int, gsids:dict[str, int], turns:list[TurnRecord], states:list[SnakeState]) -> None:
|
||||
if turns:
|
||||
await conn.executemany(_UPSERT_TURN, [
|
||||
(
|
||||
gid, record.turn, record.observed_at or self._utc_now_ts(), encode_move(record.move),
|
||||
record.thinking, encode_points(record.food), encode_points(record.hazards),
|
||||
)
|
||||
for record in turns
|
||||
])
|
||||
states = [state for state in states if state.snake_id in gsids]
|
||||
if states:
|
||||
await conn.execute(
|
||||
_UPSERT_STATES,
|
||||
[gsids[state.snake_id] for state in states],
|
||||
[state.turn for state in states],
|
||||
[state.health for state in states],
|
||||
[state.latency for state in states],
|
||||
[encode_body(state.body) for state in states],
|
||||
)
|
||||
|
||||
async def _load_records(self, conn, game_id:str, gid:int) -> tuple[list[SnakeIdentity], list[TurnRecord], list[SnakeState]]:
|
||||
identity_rows = await conn.fetch("""
|
||||
SELECT snake_id, snake_name, is_you, customizations
|
||||
FROM game_snakes WHERE game_id = $1
|
||||
""", game_id)
|
||||
identities = [
|
||||
SnakeIdentity(row["snake_id"], row["snake_name"], bool(row["is_you"]), row["customizations"] or {})
|
||||
for row in identity_rows
|
||||
]
|
||||
blob = await conn.fetchrow("SELECT format, data FROM game_replays WHERE gid = $1", gid)
|
||||
turn_rows = await conn.fetch("""
|
||||
SELECT turn, observed_at, my_move, thinking, food, hazards
|
||||
FROM game_turns WHERE gid = $1 ORDER BY turn
|
||||
""", gid)
|
||||
state_rows = await conn.fetch("""
|
||||
SELECT gs.snake_id, s.turn, s.health, s.latency, s.body
|
||||
FROM game_snakes AS gs
|
||||
JOIN snake_states AS s ON s.gsid = gs.gsid
|
||||
WHERE gs.game_id = $1
|
||||
""", game_id)
|
||||
turns = [
|
||||
TurnRecord(
|
||||
turn=row["turn"], observed_at=row["observed_at"], move=decode_move(row["my_move"]),
|
||||
thinking=row["thinking"], food=decode_points(row["food"]), hazards=decode_points(row["hazards"]),
|
||||
)
|
||||
for row in turn_rows
|
||||
]
|
||||
states = [
|
||||
SnakeState(row["snake_id"], row["turn"], row["health"], row["latency"], decode_body(row["body"]))
|
||||
for row in state_rows
|
||||
]
|
||||
if blob is not None:
|
||||
packed_turns, packed_states = unpack_replay(blob["data"], blob["format"])
|
||||
turns, states = merge_records(packed_turns, packed_states, turns, states)
|
||||
return identities, turns, states
|
||||
|
||||
async def _store_replay(self, conn, game_id:str, gid:int, turns:list[TurnRecord], states:list[SnakeState]) -> None:
|
||||
await conn.execute("""
|
||||
INSERT INTO game_replays (gid, format, turn_count, data) VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (gid) DO UPDATE SET
|
||||
format = EXCLUDED.format, turn_count = EXCLUDED.turn_count, data = EXCLUDED.data
|
||||
""", gid, REPLAY_FORMAT, len(turns), pack_replay(turns, states))
|
||||
await conn.execute("DELETE FROM game_turns WHERE gid = $1", gid)
|
||||
await conn.execute("""
|
||||
DELETE FROM snake_states
|
||||
WHERE gsid IN (SELECT gsid FROM game_snakes WHERE game_id = $1)
|
||||
""", game_id)
|
||||
|
||||
async def _pack_game(self, conn, game_id:str) -> None:
|
||||
"""Move a game's rows into its replay blob (idempotent)."""
|
||||
async with conn.transaction():
|
||||
gid = await conn.fetchval("SELECT gid FROM games WHERE game_id = $1 FOR UPDATE", game_id)
|
||||
if gid is None:
|
||||
return
|
||||
has_rows = await conn.fetchval("SELECT EXISTS (SELECT 1 FROM game_turns WHERE gid = $1)", gid)
|
||||
if not has_rows:
|
||||
return
|
||||
_, turns, states = await self._load_records(conn, game_id, gid)
|
||||
await self._store_replay(conn, game_id, gid, turns, states)
|
||||
|
||||
# ── write methods ──────────────────────────────────────────────────────────
|
||||
|
||||
@@ -465,111 +343,24 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
)
|
||||
|
||||
async def record_turn(self, game_state:dict, my_move:str|None, my_thinking:dict|None=None) -> None:
|
||||
game = game_state.get("game", {})
|
||||
board = game_state.get("board", {})
|
||||
snakes = self._extract_snakes(game_state)
|
||||
you = self._extract_you(game_state)
|
||||
game_id = game.get("id")
|
||||
turn = int(game_state.get("turn", 0))
|
||||
|
||||
board_json, snakes_json, you_json, food_json, hazards_json = compact_turn_json(board)
|
||||
|
||||
game_id = game_state.get("game", {}).get("id")
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
await conn.executemany("""
|
||||
INSERT INTO game_snakes (
|
||||
game_id, snake_id, snake_name, is_you, customizations
|
||||
) VALUES ($1,$2,$3,$4,$5)
|
||||
ON CONFLICT (game_id, snake_id) DO UPDATE SET
|
||||
snake_name = EXCLUDED.snake_name,
|
||||
is_you = EXCLUDED.is_you,
|
||||
customizations = EXCLUDED.customizations
|
||||
""",
|
||||
[
|
||||
(
|
||||
game_id,
|
||||
snake.get("id"),
|
||||
snake.get("name"),
|
||||
snake.get("id") == you.get("id"),
|
||||
snake.get("customizations") or {},
|
||||
)
|
||||
for snake in snakes if snake.get("id") is not None
|
||||
],
|
||||
running = await self._running_game(conn, game_id)
|
||||
record, states, identities, points = records_from_game_state(
|
||||
game_state, my_move, my_thinking, running.points,
|
||||
)
|
||||
await conn.execute("""
|
||||
INSERT INTO turns (
|
||||
game_id, turn, observed_at, my_move, my_thinking,
|
||||
board_state, snakes, you, food, hazards
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
|
||||
ON CONFLICT (game_id, turn) DO UPDATE SET
|
||||
observed_at = EXCLUDED.observed_at,
|
||||
my_move = EXCLUDED.my_move,
|
||||
my_thinking = EXCLUDED.my_thinking,
|
||||
board_state = EXCLUDED.board_state,
|
||||
snakes = EXCLUDED.snakes,
|
||||
you = EXCLUDED.you,
|
||||
food = EXCLUDED.food,
|
||||
hazards = EXCLUDED.hazards
|
||||
""",
|
||||
game_id,
|
||||
turn,
|
||||
self._utc_now_ts(),
|
||||
my_move,
|
||||
my_thinking,
|
||||
board_json,
|
||||
snakes_json,
|
||||
you_json,
|
||||
food_json,
|
||||
hazards_json,
|
||||
)
|
||||
|
||||
previous_positions:dict[str, tuple[int, int]] = {}
|
||||
if turn > 0:
|
||||
prev_rows = await conn.fetch("""
|
||||
SELECT snake_id, head_x, head_y
|
||||
FROM snake_turns
|
||||
WHERE game_id = $1 AND turn = $2
|
||||
""",
|
||||
game_id, turn - 1,
|
||||
)
|
||||
previous_positions = {
|
||||
row["snake_id"]: (int(row["head_x"]), int(row["head_y"]))
|
||||
for row in prev_rows
|
||||
if row["head_x"] is not None and row["head_y"] is not None
|
||||
}
|
||||
|
||||
you_id = you.get("id")
|
||||
for params in self._build_snake_turn_params(snakes, you_id, game_id, turn, previous_positions):
|
||||
(p_game_id, p_turn, p_snake_id, p_name, p_health, p_length,
|
||||
p_head_x, p_head_y, p_body, p_is_you, p_inferred, p_latency) = params
|
||||
await conn.execute("""
|
||||
INSERT INTO snake_turns (
|
||||
game_id, turn, snake_id, snake_name, health, length,
|
||||
head_x, head_y, body, is_you, inferred_move, latency
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)
|
||||
ON CONFLICT (game_id, turn, snake_id) DO UPDATE SET
|
||||
snake_name = EXCLUDED.snake_name,
|
||||
health = EXCLUDED.health,
|
||||
length = EXCLUDED.length,
|
||||
head_x = EXCLUDED.head_x,
|
||||
head_y = EXCLUDED.head_y,
|
||||
body = EXCLUDED.body,
|
||||
is_you = EXCLUDED.is_you,
|
||||
inferred_move = EXCLUDED.inferred_move,
|
||||
latency = EXCLUDED.latency
|
||||
""",
|
||||
p_game_id, p_turn, p_snake_id, None, p_health, p_length,
|
||||
p_head_x, p_head_y, p_body, False, p_inferred, p_latency,
|
||||
)
|
||||
|
||||
await conn.execute("""
|
||||
UPDATE games
|
||||
SET final_turn = GREATEST(final_turn, $1)
|
||||
WHERE game_id = $2
|
||||
""",
|
||||
turn, game_id,
|
||||
missing = [identity for identity in identities if identity.snake_id not in running.snake_ids]
|
||||
if missing:
|
||||
running.snake_ids.update(await self._upsert_identities(conn, game_id, identities))
|
||||
await self._write_rows(conn, running.gid, running.snake_ids, [record], states)
|
||||
await conn.execute(
|
||||
"UPDATE games SET final_turn = GREATEST(final_turn, $1) WHERE gid = $2",
|
||||
record.turn, running.gid,
|
||||
)
|
||||
running.points = points
|
||||
self._running.put(game_id, running)
|
||||
|
||||
async def record_game_end(self, game_state:dict) -> None:
|
||||
game_id, winner_name, winner_you, turn = self._extract_game_end_params(game_state)
|
||||
@@ -585,12 +376,10 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
status = 'finished'
|
||||
WHERE game_id = $5
|
||||
""",
|
||||
self._utc_now_ts(),
|
||||
winner_name,
|
||||
winner_you,
|
||||
turn,
|
||||
game_id,
|
||||
self._utc_now_ts(), winner_name, winner_you, turn, game_id,
|
||||
)
|
||||
await self._pack_game(conn, game_id)
|
||||
self._running.drop(game_id)
|
||||
|
||||
# ── stale game finalization ────────────────────────────────────────────────
|
||||
|
||||
@@ -602,7 +391,7 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
rows = await conn.fetch("""
|
||||
SELECT game_id, started_at, final_turn, your_snake_id
|
||||
SELECT gid, game_id, started_at, final_turn, your_snake_id
|
||||
FROM games
|
||||
WHERE status = 'running'
|
||||
ORDER BY started_at ASC
|
||||
@@ -618,40 +407,9 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
continue
|
||||
|
||||
game_id = row["game_id"]
|
||||
your_snake_id = row["your_snake_id"]
|
||||
final_turn = int(row["final_turn"] or 0)
|
||||
|
||||
snake_rows = await conn.fetch("""
|
||||
SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name) AS snake_name
|
||||
FROM snake_turns AS st
|
||||
LEFT JOIN game_snakes AS gs
|
||||
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
|
||||
WHERE st.game_id = $1 AND st.turn = $2
|
||||
ORDER BY COALESCE(gs.is_you, st.is_you) DESC, snake_name ASC
|
||||
""",
|
||||
game_id, final_turn,
|
||||
)
|
||||
|
||||
if len(snake_rows) == 0:
|
||||
latest_row = await conn.fetchrow(
|
||||
"SELECT MAX(turn) AS latest_turn FROM snake_turns WHERE game_id = $1",
|
||||
game_id,
|
||||
)
|
||||
|
||||
if latest_row is not None and latest_row["latest_turn"] is not None:
|
||||
final_turn = int(latest_row["latest_turn"])
|
||||
snake_rows = await conn.fetch("""
|
||||
SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name) AS snake_name
|
||||
FROM snake_turns AS st
|
||||
LEFT JOIN game_snakes AS gs
|
||||
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
|
||||
WHERE st.game_id = $1 AND st.turn = $2
|
||||
ORDER BY COALESCE(gs.is_you, st.is_you) DESC, snake_name ASC
|
||||
""",
|
||||
game_id, final_turn,
|
||||
)
|
||||
|
||||
winner_you, survivor_name = self._calculate_survivor(snake_rows, your_snake_id)
|
||||
identities, _, states = await self._load_records(conn, game_id, row["gid"])
|
||||
final_turn, survivors = self._survivors(identities, states, int(row["final_turn"] or 0))
|
||||
winner_you, survivor_name = self._calculate_survivor(survivors, row["your_snake_id"])
|
||||
|
||||
tag = await conn.execute("""
|
||||
UPDATE games
|
||||
@@ -662,14 +420,21 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
status = 'finished'
|
||||
WHERE game_id = $5 AND status = 'running'
|
||||
""",
|
||||
self._utc_now_ts(),
|
||||
survivor_name,
|
||||
winner_you,
|
||||
final_turn,
|
||||
game_id,
|
||||
self._utc_now_ts(), survivor_name, winner_you, final_turn, game_id,
|
||||
)
|
||||
if tag and tag.endswith("1"):
|
||||
finalized += 1
|
||||
await self._pack_game(conn, game_id)
|
||||
self._running.drop(game_id)
|
||||
|
||||
# Turns that arrived after a game was packed are folded into its blob.
|
||||
late = await conn.fetch("""
|
||||
SELECT DISTINCT g.game_id
|
||||
FROM game_turns AS t JOIN games AS g ON g.gid = t.gid
|
||||
WHERE g.status = 'finished'
|
||||
""")
|
||||
for late_row in late:
|
||||
await self._pack_game(conn, late_row["game_id"])
|
||||
|
||||
return finalized
|
||||
|
||||
@@ -734,51 +499,51 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
async def get_game_replay(self, game_id:str) -> dict|None:
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
game_row = await conn.fetchrow("""
|
||||
SELECT game_id, started_at, ended_at, width, height, source, map_name,
|
||||
ruleset_name, ruleset_version, game_type, your_snake_id, your_snake_name,
|
||||
your_snake_type, your_snake_version,
|
||||
winner_name, winner_you, final_turn, status
|
||||
FROM games
|
||||
WHERE game_id = $1 AND has_replay
|
||||
""",
|
||||
game_id,
|
||||
)
|
||||
|
||||
game_row = await conn.fetchrow(_GAME_SELECT + " WHERE game_id = $1 AND has_replay", game_id)
|
||||
if game_row is None:
|
||||
return None
|
||||
identities, turns, states = await self._load_records(conn, game_id, game_row["gid"])
|
||||
|
||||
turn_rows = await conn.fetch("""
|
||||
SELECT turn, observed_at, my_move,
|
||||
my_thinking AS my_thinking_json,
|
||||
board_state AS board_state_json,
|
||||
food AS food_json,
|
||||
hazards AS hazards_json,
|
||||
you AS you_json
|
||||
FROM turns
|
||||
WHERE game_id = $1
|
||||
ORDER BY turn ASC
|
||||
""",
|
||||
game_id,
|
||||
)
|
||||
return self._build_game_replay_output(game_row, identities, turns, states)
|
||||
|
||||
snake_rows = await conn.fetch("""
|
||||
SELECT st.turn, st.snake_id,
|
||||
COALESCE(gs.snake_name, st.snake_name) AS snake_name,
|
||||
st.health, st.length, st.head_x, st.head_y, st.body AS body_json,
|
||||
COALESCE(gs.is_you, st.is_you) AS is_you,
|
||||
COALESCE(gs.customizations, '{}'::jsonb) AS customizations_json,
|
||||
st.inferred_move, st.latency
|
||||
FROM snake_turns AS st
|
||||
LEFT JOIN game_snakes AS gs
|
||||
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
|
||||
WHERE st.game_id = $1
|
||||
ORDER BY st.turn ASC, is_you DESC, snake_name ASC
|
||||
""",
|
||||
game_id,
|
||||
)
|
||||
# ── bundles (migration / merge) ────────────────────────────────────────────
|
||||
|
||||
return self._build_game_replay_output(game_row, turn_rows, snake_rows, decode_json=lambda x: x)
|
||||
async def list_game_ids(self) -> list[str]:
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
rows = await conn.fetch("SELECT game_id FROM games ORDER BY started_at, game_id")
|
||||
return [row["game_id"] for row in rows]
|
||||
|
||||
async def read_game_bundle(self, game_id:str) -> GameBundle|None:
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
game_row = await conn.fetchrow(_GAME_SELECT + " WHERE game_id = $1", game_id)
|
||||
if game_row is None:
|
||||
return None
|
||||
identities, turns, states = await self._load_records(conn, game_id, game_row["gid"])
|
||||
return GameBundle(normalize_game(dict(game_row)), identities, turns, states)
|
||||
|
||||
async def import_game_bundle(self, bundle:GameBundle) -> None:
|
||||
game = normalize_game(bundle.game)
|
||||
columns = ", ".join(GAME_COLUMNS)
|
||||
placeholders = ", ".join(f"${index}" for index in range(1, len(GAME_COLUMNS) + 1))
|
||||
updates = ", ".join(f"{column} = EXCLUDED.{column}" for column in GAME_COLUMNS[1:])
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
gid = await conn.fetchval(
|
||||
f"INSERT INTO games ({columns}) VALUES ({placeholders}) "
|
||||
f"ON CONFLICT (game_id) DO UPDATE SET {updates} RETURNING gid",
|
||||
*[game[column] for column in GAME_COLUMNS],
|
||||
)
|
||||
gsids = await self._upsert_identities(conn, game["game_id"], bundle.snakes)
|
||||
if not bundle.turns or not game["has_replay"]:
|
||||
return
|
||||
if game["status"] == "finished":
|
||||
await self._store_replay(conn, game["game_id"], gid, bundle.turns, bundle.states)
|
||||
else:
|
||||
await self._write_rows(conn, gid, gsids, bundle.turns, bundle.states)
|
||||
self._running.drop(game["game_id"])
|
||||
|
||||
# ── lifecycle ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
Reference in New Issue
Block a user