feat(database): store gameplay replays in compact form
Build and Push Docker Container / build-and-push (push) Successful in 3m52s
Build and Push Docker Container / build-and-push (push) Successful in 3m52s
- Add compact storage package: codec, records, hydrate, replay blobs, cache. - Running games write game_turns and snake_states keyed by integer ids. - Encode moves as 1-byte codes and bodies as head plus step codes. - Store food and hazards only when they changed since the previous turn. - Strip reasoning fields that duplicate stored values and restore on read. - Pack finished games into one zstd blob per game in game_replays. - Merge late turns into packed blobs during the stale-game sweep. - Extract SQLite and PostgreSQL schemas into dedicated modules. - Add read/import game bundle and list_game_ids to the backend template. - Add legacy readers and converter for the old turns/snake_turns layout. - Add migrate_gameplay_to_compact.py with in-place or cross-DB migration. - Verify every migrated game and keep legacy rows on failure. - Support --since, --minimum-quality and guarded --drop-legacy. - Add just recipe migrate-gameplay-postgres. - Rewrite merge script to copy game by game via backend bundles. - Move benchmark state loading into benchmark_states for both backends. - Update PostgreSQL cleanup script to the new replay tables. - Drop sqlite-zstd build stages from the Dockerfile. - Remove old migration scripts, normalized_turn and their tests. - Add tests for compact storage, schema, legacy migration and PostgreSQL. - Document the compact layout and migration workflow in the README.
This commit is contained in:
@@ -1,10 +1,8 @@
|
||||
"""PostgreSQL gameplay backend using asyncpg.
|
||||
|
||||
JSON columns use the JSONB type so PostgreSQL stores them in a binary,
|
||||
decomposed format and automatically compresses large values via TOAST
|
||||
(Oversized-Attribute Storage Technique). No application-level
|
||||
serialisation/deserialisation round-trip is needed for reads — asyncpg
|
||||
decodes JSONB rows directly into Python dicts/lists.
|
||||
Running games are stored as compact rows (``game_turns``/``snake_states``).
|
||||
When a game ends it is packed into one zstd blob in ``game_replays`` and its
|
||||
rows are removed. See ``server.database.compact`` for the encodings.
|
||||
|
||||
Connection: pass a DSN via the `dsn` constructor argument, e.g.
|
||||
postgresql://user:password@host:5432/dbname
|
||||
@@ -12,13 +10,22 @@ Connection: pass a DSN via the `dsn` constructor argument, e.g.
|
||||
or set GAMEPLAY_DB_PG_DSN in the environment.
|
||||
"""
|
||||
|
||||
import asyncio, logging, sqlite3, json, sys
|
||||
import json, logging, sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse, urlunparse
|
||||
|
||||
from .Template import GameplayBackendTemplate
|
||||
from server.database.normalized_turn import compact_turn_json
|
||||
from .postgresql_schema import DROP_REDUNDANT_INDEXES, TABLES_DDL
|
||||
from server.database.compact.cache import RunningGame, RunningGameCache
|
||||
from server.database.compact.codec import (
|
||||
decode_body, decode_move, decode_points, encode_body, encode_move, encode_points,
|
||||
)
|
||||
from server.database.compact.games import GAME_COLUMNS, normalize_game
|
||||
from server.database.compact.records import (
|
||||
GameBundle, SnakeIdentity, SnakeState, TurnRecord, merge_records, records_from_game_state,
|
||||
)
|
||||
from server.database.compact.replay_blob import REPLAY_FORMAT, pack_replay, unpack_replay
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
if not logger.handlers:
|
||||
@@ -27,106 +34,46 @@ if not logger.handlers:
|
||||
logger.addHandler(_handler)
|
||||
logger.propagate = False
|
||||
|
||||
# DDL --------------------------------------------------------------------- #
|
||||
_SCHEMA_LOCK_ID = 7_315_002
|
||||
|
||||
_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS games (
|
||||
game_id TEXT PRIMARY KEY,
|
||||
started_at TIMESTAMPTZ NOT NULL,
|
||||
ended_at TIMESTAMPTZ,
|
||||
width INTEGER,
|
||||
height INTEGER,
|
||||
source TEXT,
|
||||
map_name TEXT,
|
||||
ruleset_name TEXT,
|
||||
ruleset_version TEXT,
|
||||
your_snake_id TEXT,
|
||||
your_snake_name TEXT,
|
||||
your_snake_type TEXT,
|
||||
your_snake_version TEXT,
|
||||
game_type TEXT,
|
||||
winner_name TEXT,
|
||||
winner_you BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
final_turn INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL DEFAULT 'running',
|
||||
has_replay BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
quality_status TEXT NOT NULL DEFAULT 'retained',
|
||||
quality_score INTEGER,
|
||||
quality_tier TEXT,
|
||||
quality_reasons JSONB
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS turns (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
game_id TEXT NOT NULL REFERENCES games(game_id) ON DELETE CASCADE,
|
||||
turn INTEGER NOT NULL,
|
||||
observed_at TIMESTAMPTZ NOT NULL,
|
||||
my_move TEXT,
|
||||
my_thinking JSONB,
|
||||
board_state JSONB NOT NULL,
|
||||
snakes JSONB NOT NULL,
|
||||
you JSONB NOT NULL,
|
||||
food JSONB NOT NULL,
|
||||
hazards JSONB NOT NULL,
|
||||
UNIQUE (game_id, turn)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS game_snakes (
|
||||
game_id TEXT NOT NULL REFERENCES games(game_id) ON DELETE CASCADE,
|
||||
snake_id TEXT NOT NULL,
|
||||
snake_name TEXT,
|
||||
is_you BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
customizations JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
PRIMARY KEY (game_id, snake_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS snake_turns (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
game_id TEXT NOT NULL REFERENCES games(game_id) ON DELETE CASCADE,
|
||||
turn INTEGER NOT NULL,
|
||||
snake_id TEXT NOT NULL,
|
||||
snake_name TEXT,
|
||||
health INTEGER,
|
||||
length INTEGER,
|
||||
head_x INTEGER,
|
||||
head_y INTEGER,
|
||||
body JSONB NOT NULL,
|
||||
is_you BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
inferred_move TEXT,
|
||||
latency TEXT,
|
||||
UNIQUE (game_id, turn, snake_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_turns_game_turn ON turns(game_id, turn);
|
||||
CREATE INDEX IF NOT EXISTS idx_games_status ON games(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_snake_turns_game_turn ON snake_turns(game_id, turn);
|
||||
_GAME_SELECT = """
|
||||
SELECT gid, game_id, started_at, ended_at, width, height, source, map_name,
|
||||
ruleset_name, ruleset_version, game_type, your_snake_id, your_snake_name,
|
||||
your_snake_type, your_snake_version, winner_name, winner_you, final_turn,
|
||||
status, has_replay, quality_status, quality_score, quality_tier, quality_reasons
|
||||
FROM games
|
||||
"""
|
||||
|
||||
# Schema evolution: add new columns to existing tables (idempotent).
|
||||
_ALTER_DDL = """
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS game_type TEXT;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS your_snake_type TEXT;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS your_snake_version TEXT;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS winner_name TEXT;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS has_replay BOOLEAN NOT NULL DEFAULT TRUE;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_status TEXT NOT NULL DEFAULT 'retained';
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_score INTEGER;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_tier TEXT;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_reasons JSONB;
|
||||
ALTER TABLE turns ADD COLUMN IF NOT EXISTS my_thinking JSONB;
|
||||
ALTER TABLE snake_turns ADD COLUMN IF NOT EXISTS latency TEXT;
|
||||
ALTER TABLE game_snakes ADD COLUMN IF NOT EXISTS customizations JSONB NOT NULL DEFAULT '{}'::jsonb;
|
||||
_UPSERT_SNAKES = """
|
||||
INSERT INTO game_snakes (game_id, snake_id, snake_name, is_you, customizations)
|
||||
SELECT $1, s.snake_id, s.snake_name, s.is_you, s.customizations::jsonb
|
||||
FROM unnest($2::text[], $3::text[], $4::bool[], $5::text[])
|
||||
AS s(snake_id, snake_name, is_you, customizations)
|
||||
ON CONFLICT (game_id, snake_id) DO UPDATE SET
|
||||
snake_name = EXCLUDED.snake_name,
|
||||
is_you = EXCLUDED.is_you,
|
||||
customizations = EXCLUDED.customizations
|
||||
RETURNING snake_id, gsid
|
||||
"""
|
||||
|
||||
# Force TOAST compression on the large JSONB columns so that even
|
||||
# moderately-sized payloads get compressed on-disk.
|
||||
_TOAST_DDL = """
|
||||
ALTER TABLE turns ALTER COLUMN board_state SET STORAGE EXTENDED;
|
||||
ALTER TABLE turns ALTER COLUMN snakes SET STORAGE EXTENDED;
|
||||
ALTER TABLE turns ALTER COLUMN you SET STORAGE EXTENDED;
|
||||
ALTER TABLE turns ALTER COLUMN food SET STORAGE EXTENDED;
|
||||
ALTER TABLE turns ALTER COLUMN hazards SET STORAGE EXTENDED;
|
||||
ALTER TABLE snake_turns ALTER COLUMN body SET STORAGE EXTENDED;
|
||||
_UPSERT_TURN = """
|
||||
INSERT INTO game_turns (gid, turn, observed_at, my_move, thinking, food, hazards)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (gid, turn) DO UPDATE SET
|
||||
observed_at = EXCLUDED.observed_at,
|
||||
my_move = EXCLUDED.my_move,
|
||||
thinking = EXCLUDED.thinking,
|
||||
food = EXCLUDED.food,
|
||||
hazards = EXCLUDED.hazards
|
||||
"""
|
||||
|
||||
_UPSERT_STATES = """
|
||||
INSERT INTO snake_states (gsid, turn, health, latency, body)
|
||||
SELECT * FROM unnest($1::int[], $2::int[], $3::smallint[], $4::smallint[], $5::bytea[])
|
||||
ON CONFLICT (gsid, turn) DO UPDATE SET
|
||||
health = EXCLUDED.health,
|
||||
latency = EXCLUDED.latency,
|
||||
body = EXCLUDED.body
|
||||
"""
|
||||
|
||||
class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
@@ -142,6 +89,7 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
self._max_size = max_size
|
||||
self._sqlite_migration_path = sqlite_migration_path
|
||||
self._pool = None # asyncpg.Pool, typed at runtime
|
||||
self._running = RunningGameCache()
|
||||
|
||||
# ── DSN normalisation ──────────────────────────────────────────────────────
|
||||
|
||||
@@ -149,25 +97,16 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
|
||||
@classmethod
|
||||
def _ensure_db_name(cls, dsn:str) -> str:
|
||||
"""Return *dsn* with a database name appended when none is present.
|
||||
|
||||
A DSN has no database name when its path component is empty or ``/``.
|
||||
In that case ``battlesnake`` is appended so asyncpg gets a complete
|
||||
connection string without the caller having to remember to add one.
|
||||
"""
|
||||
"""Return *dsn* with a database name appended when none is present."""
|
||||
parsed = urlparse(dsn)
|
||||
db = parsed.path.lstrip("/")
|
||||
if db:
|
||||
if parsed.path.lstrip("/"):
|
||||
return dsn
|
||||
|
||||
new_path = f"/{cls._DEFAULT_DB_NAME}"
|
||||
return urlunparse(parsed._replace(path=new_path))
|
||||
return urlunparse(parsed._replace(path=f"/{cls._DEFAULT_DB_NAME}"))
|
||||
|
||||
# ── pool / schema ──────────────────────────────────────────────────────────
|
||||
|
||||
async def initialize(self) -> None:
|
||||
"""Eagerly create the connection pool on startup so schema init and
|
||||
SQLite migration run immediately rather than on the first game request."""
|
||||
"""Eagerly create the pool so schema init and legacy SQLite import run on startup."""
|
||||
await self._get_pool()
|
||||
|
||||
async def _get_pool(self):
|
||||
@@ -194,27 +133,22 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
init=_init_conn,
|
||||
)
|
||||
await self._initialize_schema()
|
||||
await self._maybe_migrate_from_sqlite()
|
||||
await self._maybe_import_legacy_sqlite()
|
||||
return self._pool
|
||||
|
||||
async def _ensure_database_exists(self, asyncpg, target_dsn:str) -> None:
|
||||
"""Connect to the postgres maintenance DB and CREATE the target database
|
||||
if it does not already exist. Uses a plain connection (not a pool) so
|
||||
the CREATE DATABASE statement can run outside any transaction."""
|
||||
"""CREATE the target database through the maintenance DB when missing."""
|
||||
parsed = urlparse(target_dsn)
|
||||
db_name = parsed.path.lstrip("/")
|
||||
maintenance_dsn = urlunparse(parsed._replace(path="/postgres"))
|
||||
try:
|
||||
conn = await asyncpg.connect(dsn=maintenance_dsn)
|
||||
except Exception:
|
||||
# Fall back to connecting without specifying a database — some setups
|
||||
# (e.g. Cloud SQL, managed PG) disallow direct access to 'postgres'.
|
||||
# Some managed setups disallow direct access to 'postgres'.
|
||||
maintenance_dsn = urlunparse(parsed._replace(path=""))
|
||||
conn = await asyncpg.connect(dsn=maintenance_dsn)
|
||||
try:
|
||||
exists = await conn.fetchval(
|
||||
"SELECT 1 FROM pg_database WHERE datname = $1", db_name
|
||||
)
|
||||
exists = await conn.fetchval("SELECT 1 FROM pg_database WHERE datname = $1", db_name)
|
||||
if not exists:
|
||||
await conn.execute(f'CREATE DATABASE "{db_name}"')
|
||||
logger.info(f"PostgreSQL: created database '{db_name}'")
|
||||
@@ -224,199 +158,143 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
async def _initialize_schema(self) -> None:
|
||||
assert self._pool is not None
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.execute(_DDL)
|
||||
await conn.execute(_ALTER_DDL)
|
||||
# TOAST storage hints are idempotent; ignore errors on repeated runs.
|
||||
# Several workers may start at once; serialize DDL.
|
||||
await conn.execute("SELECT pg_advisory_lock($1)", _SCHEMA_LOCK_ID)
|
||||
try:
|
||||
await conn.execute(_TOAST_DDL)
|
||||
except Exception as exc:
|
||||
logger.debug(f"TOAST DDL skipped (likely already set): {exc}")
|
||||
await conn.execute(TABLES_DDL)
|
||||
finally:
|
||||
await conn.execute("SELECT pg_advisory_unlock($1)", _SCHEMA_LOCK_ID)
|
||||
for statement in DROP_REDUNDANT_INDEXES:
|
||||
await conn.execute(statement)
|
||||
|
||||
# ── sqlite migration ───────────────────────────────────────────────────────
|
||||
|
||||
async def _maybe_migrate_from_sqlite(self) -> None:
|
||||
async def _maybe_import_legacy_sqlite(self) -> None:
|
||||
if not self._sqlite_migration_path:
|
||||
return
|
||||
|
||||
src = Path(self._sqlite_migration_path)
|
||||
if not src.exists():
|
||||
source = Path(self._sqlite_migration_path)
|
||||
if not source.exists():
|
||||
return
|
||||
|
||||
logger.info(f"SQLite migration: found {src}, starting migration to PostgreSQL …")
|
||||
from server.database.legacy.migrate import migrate_legacy # noqa: PLC0415
|
||||
from server.database.legacy.reader_sqlite import LegacySqliteReader # noqa: PLC0415
|
||||
|
||||
logger.info(f"Legacy SQLite import: found {source}, importing into PostgreSQL …")
|
||||
try:
|
||||
games, turns, snake_turns = await asyncio.to_thread(self._read_sqlite_data_sync, str(src))
|
||||
await self._insert_migrated_data(games, turns, snake_turns)
|
||||
|
||||
done_path = src.with_suffix(".migrated")
|
||||
src.rename(done_path)
|
||||
|
||||
logger.info(
|
||||
f"SQLite migration complete: {len(games)} games, {len(turns)} turns, "
|
||||
f"{len(snake_turns)} snake_turns migrated. "
|
||||
f"Source file renamed to {done_path.name}"
|
||||
)
|
||||
with LegacySqliteReader(source) as reader:
|
||||
stats = await migrate_legacy(reader, self)
|
||||
if stats.failed:
|
||||
logger.warning(f"Legacy SQLite import incomplete ({stats.failed} games failed); source kept")
|
||||
return
|
||||
done_path = source.with_suffix(".migrated")
|
||||
source.rename(done_path)
|
||||
logger.info(f"Legacy SQLite import complete: {stats.games} games. Source renamed to {done_path.name}")
|
||||
except Exception:
|
||||
logger.exception("SQLite migration failed — PostgreSQL data is untouched, original SQLite file kept")
|
||||
|
||||
def _read_sqlite_data_sync(self, db_path:str) -> tuple[list[sqlite3.Row], list[sqlite3.Row], list[sqlite3.Row]]:
|
||||
conn = sqlite3.connect(db_path, timeout=30, isolation_level=None)
|
||||
conn.row_factory = sqlite3.Row
|
||||
|
||||
try:
|
||||
game_columns = {
|
||||
row["name"] for row in conn.execute("PRAGMA table_info(games)").fetchall()
|
||||
}
|
||||
if "winner_name" in game_columns:
|
||||
winner_name_expression = "winner_name"
|
||||
elif "winner_names_json" in game_columns:
|
||||
winner_name_expression = "winner_names_json"
|
||||
else:
|
||||
winner_name_expression = "NULL"
|
||||
games = conn.execute(f"""
|
||||
SELECT game_id, started_at, ended_at, width, height, source, map_name,
|
||||
ruleset_name, ruleset_version, your_snake_id, your_snake_name,
|
||||
your_snake_type, your_snake_version, game_type,
|
||||
{winner_name_expression} AS winner_name, winner_you, final_turn, status
|
||||
FROM games
|
||||
ORDER BY started_at ASC
|
||||
""").fetchall()
|
||||
turns = conn.execute("""
|
||||
SELECT game_id, turn, observed_at, my_move, my_thinking_json,
|
||||
board_state_json, snakes_json, you_json, food_json, hazards_json
|
||||
FROM turns
|
||||
ORDER BY game_id ASC, turn ASC
|
||||
""").fetchall()
|
||||
snake_turns = conn.execute("""
|
||||
SELECT game_id, turn, snake_id, snake_name, health, length,
|
||||
head_x, head_y, body_json, is_you, inferred_move, latency
|
||||
FROM snake_turns
|
||||
ORDER BY game_id ASC, turn ASC, snake_id ASC
|
||||
""").fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
return games, turns, snake_turns
|
||||
|
||||
def _parse_ts(self, value:str|None) -> datetime|None:
|
||||
"""Parse an ISO-8601 TEXT timestamp from SQLite into a timezone-aware datetime."""
|
||||
ts = self._parse_utc_timestamp(value)
|
||||
return ts # already UTC-aware from base class helper
|
||||
|
||||
def _parse_json(self, value: str|None) -> object:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return json.loads(value)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return None
|
||||
|
||||
def _migrated_winner_name(self, value:str|None) -> str|None:
|
||||
"""Accept both the current winner_name and legacy winner_names_json value."""
|
||||
if not value:
|
||||
return None
|
||||
parsed = self._parse_json(value)
|
||||
if isinstance(parsed, list):
|
||||
return next((str(name) for name in parsed if name), None)
|
||||
return value
|
||||
|
||||
async def _insert_migrated_data(self, games:list, turns:list, snake_turns:list) -> None:
|
||||
assert self._pool is not None
|
||||
async with self._pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
# games ─────────────────────────────────────────────────────────────
|
||||
# winner_name is TEXT — no cast needed.
|
||||
await conn.executemany("""
|
||||
INSERT INTO games (
|
||||
game_id, started_at, ended_at, width, height, source, map_name,
|
||||
ruleset_name, ruleset_version, your_snake_id, your_snake_name,
|
||||
your_snake_type, your_snake_version, game_type,
|
||||
winner_name, winner_you, final_turn, status
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18)
|
||||
ON CONFLICT (game_id) DO NOTHING
|
||||
""",
|
||||
[
|
||||
(
|
||||
row["game_id"],
|
||||
self._parse_ts(row["started_at"]),
|
||||
self._parse_ts(row["ended_at"]),
|
||||
row["width"],
|
||||
row["height"],
|
||||
row["source"],
|
||||
row["map_name"],
|
||||
row["ruleset_name"],
|
||||
row["ruleset_version"],
|
||||
row["your_snake_id"],
|
||||
row["your_snake_name"],
|
||||
row["your_snake_type"],
|
||||
row["your_snake_version"],
|
||||
row["game_type"],
|
||||
self._migrated_winner_name(row["winner_name"]),
|
||||
bool(row["winner_you"]),
|
||||
row["final_turn"],
|
||||
row["status"],
|
||||
)
|
||||
for row in games
|
||||
],
|
||||
)
|
||||
|
||||
await conn.executemany("""
|
||||
INSERT INTO turns (
|
||||
game_id, turn, observed_at, my_move, my_thinking,
|
||||
board_state, snakes, you, food, hazards
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
|
||||
ON CONFLICT (game_id, turn) DO NOTHING
|
||||
""",
|
||||
[
|
||||
(
|
||||
row["game_id"],
|
||||
row["turn"],
|
||||
self._parse_ts(row["observed_at"]),
|
||||
row["my_move"],
|
||||
self._parse_json(row["my_thinking_json"]),
|
||||
self._parse_json(row["board_state_json"]),
|
||||
self._parse_json(row["snakes_json"]),
|
||||
self._parse_json(row["you_json"]),
|
||||
self._parse_json(row["food_json"]),
|
||||
self._parse_json(row["hazards_json"]),
|
||||
)
|
||||
for row in turns
|
||||
],
|
||||
)
|
||||
|
||||
# snake_turns
|
||||
await conn.executemany("""
|
||||
INSERT INTO snake_turns (
|
||||
game_id, turn, snake_id, snake_name, health, length,
|
||||
head_x, head_y, body, is_you, inferred_move, latency
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)
|
||||
ON CONFLICT (game_id, turn, snake_id) DO NOTHING
|
||||
""",
|
||||
[
|
||||
(
|
||||
row["game_id"],
|
||||
row["turn"],
|
||||
row["snake_id"],
|
||||
row["snake_name"],
|
||||
row["health"],
|
||||
row["length"],
|
||||
row["head_x"],
|
||||
row["head_y"],
|
||||
self._parse_json(row["body_json"]),
|
||||
bool(row["is_you"]),
|
||||
row["inferred_move"],
|
||||
row["latency"],
|
||||
)
|
||||
for row in snake_turns
|
||||
],
|
||||
)
|
||||
logger.exception("Legacy SQLite import failed — original SQLite file kept")
|
||||
|
||||
# ── helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
def _utc_now_ts(self) -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
# The pool init callback registers JSON/JSONB codecs so asyncpg automatically
|
||||
# encodes Python dicts/lists on write and decodes them on read.
|
||||
async def _running_game(self, conn, game_id:str) -> RunningGame:
|
||||
game = self._running.get(game_id)
|
||||
if game is None:
|
||||
gid = await conn.fetchval("SELECT gid FROM games WHERE game_id = $1", game_id)
|
||||
if gid is None:
|
||||
raise LookupError(f"Turn for unknown game {game_id!r}")
|
||||
game = RunningGame(gid=gid)
|
||||
return game
|
||||
|
||||
async def _upsert_identities(self, conn, game_id:str, identities:list[SnakeIdentity]) -> dict[str, int]:
|
||||
if not identities:
|
||||
return {}
|
||||
rows = await conn.fetch(
|
||||
_UPSERT_SNAKES, game_id,
|
||||
[identity.snake_id for identity in identities],
|
||||
[identity.name for identity in identities],
|
||||
[identity.is_you for identity in identities],
|
||||
[json.dumps(identity.customizations or {}) for identity in identities],
|
||||
)
|
||||
return {row["snake_id"]: row["gsid"] for row in rows}
|
||||
|
||||
async def _write_rows(self, conn, gid:int, gsids:dict[str, int], turns:list[TurnRecord], states:list[SnakeState]) -> None:
|
||||
if turns:
|
||||
await conn.executemany(_UPSERT_TURN, [
|
||||
(
|
||||
gid, record.turn, record.observed_at or self._utc_now_ts(), encode_move(record.move),
|
||||
record.thinking, encode_points(record.food), encode_points(record.hazards),
|
||||
)
|
||||
for record in turns
|
||||
])
|
||||
states = [state for state in states if state.snake_id in gsids]
|
||||
if states:
|
||||
await conn.execute(
|
||||
_UPSERT_STATES,
|
||||
[gsids[state.snake_id] for state in states],
|
||||
[state.turn for state in states],
|
||||
[state.health for state in states],
|
||||
[state.latency for state in states],
|
||||
[encode_body(state.body) for state in states],
|
||||
)
|
||||
|
||||
async def _load_records(self, conn, game_id:str, gid:int) -> tuple[list[SnakeIdentity], list[TurnRecord], list[SnakeState]]:
|
||||
identity_rows = await conn.fetch("""
|
||||
SELECT snake_id, snake_name, is_you, customizations
|
||||
FROM game_snakes WHERE game_id = $1
|
||||
""", game_id)
|
||||
identities = [
|
||||
SnakeIdentity(row["snake_id"], row["snake_name"], bool(row["is_you"]), row["customizations"] or {})
|
||||
for row in identity_rows
|
||||
]
|
||||
blob = await conn.fetchrow("SELECT format, data FROM game_replays WHERE gid = $1", gid)
|
||||
turn_rows = await conn.fetch("""
|
||||
SELECT turn, observed_at, my_move, thinking, food, hazards
|
||||
FROM game_turns WHERE gid = $1 ORDER BY turn
|
||||
""", gid)
|
||||
state_rows = await conn.fetch("""
|
||||
SELECT gs.snake_id, s.turn, s.health, s.latency, s.body
|
||||
FROM game_snakes AS gs
|
||||
JOIN snake_states AS s ON s.gsid = gs.gsid
|
||||
WHERE gs.game_id = $1
|
||||
""", game_id)
|
||||
turns = [
|
||||
TurnRecord(
|
||||
turn=row["turn"], observed_at=row["observed_at"], move=decode_move(row["my_move"]),
|
||||
thinking=row["thinking"], food=decode_points(row["food"]), hazards=decode_points(row["hazards"]),
|
||||
)
|
||||
for row in turn_rows
|
||||
]
|
||||
states = [
|
||||
SnakeState(row["snake_id"], row["turn"], row["health"], row["latency"], decode_body(row["body"]))
|
||||
for row in state_rows
|
||||
]
|
||||
if blob is not None:
|
||||
packed_turns, packed_states = unpack_replay(blob["data"], blob["format"])
|
||||
turns, states = merge_records(packed_turns, packed_states, turns, states)
|
||||
return identities, turns, states
|
||||
|
||||
async def _store_replay(self, conn, game_id:str, gid:int, turns:list[TurnRecord], states:list[SnakeState]) -> None:
|
||||
await conn.execute("""
|
||||
INSERT INTO game_replays (gid, format, turn_count, data) VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (gid) DO UPDATE SET
|
||||
format = EXCLUDED.format, turn_count = EXCLUDED.turn_count, data = EXCLUDED.data
|
||||
""", gid, REPLAY_FORMAT, len(turns), pack_replay(turns, states))
|
||||
await conn.execute("DELETE FROM game_turns WHERE gid = $1", gid)
|
||||
await conn.execute("""
|
||||
DELETE FROM snake_states
|
||||
WHERE gsid IN (SELECT gsid FROM game_snakes WHERE game_id = $1)
|
||||
""", game_id)
|
||||
|
||||
async def _pack_game(self, conn, game_id:str) -> None:
|
||||
"""Move a game's rows into its replay blob (idempotent)."""
|
||||
async with conn.transaction():
|
||||
gid = await conn.fetchval("SELECT gid FROM games WHERE game_id = $1 FOR UPDATE", game_id)
|
||||
if gid is None:
|
||||
return
|
||||
has_rows = await conn.fetchval("SELECT EXISTS (SELECT 1 FROM game_turns WHERE gid = $1)", gid)
|
||||
if not has_rows:
|
||||
return
|
||||
_, turns, states = await self._load_records(conn, game_id, gid)
|
||||
await self._store_replay(conn, game_id, gid, turns, states)
|
||||
|
||||
# ── write methods ──────────────────────────────────────────────────────────
|
||||
|
||||
@@ -465,111 +343,24 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
)
|
||||
|
||||
async def record_turn(self, game_state:dict, my_move:str|None, my_thinking:dict|None=None) -> None:
|
||||
game = game_state.get("game", {})
|
||||
board = game_state.get("board", {})
|
||||
snakes = self._extract_snakes(game_state)
|
||||
you = self._extract_you(game_state)
|
||||
game_id = game.get("id")
|
||||
turn = int(game_state.get("turn", 0))
|
||||
|
||||
board_json, snakes_json, you_json, food_json, hazards_json = compact_turn_json(board)
|
||||
|
||||
game_id = game_state.get("game", {}).get("id")
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
await conn.executemany("""
|
||||
INSERT INTO game_snakes (
|
||||
game_id, snake_id, snake_name, is_you, customizations
|
||||
) VALUES ($1,$2,$3,$4,$5)
|
||||
ON CONFLICT (game_id, snake_id) DO UPDATE SET
|
||||
snake_name = EXCLUDED.snake_name,
|
||||
is_you = EXCLUDED.is_you,
|
||||
customizations = EXCLUDED.customizations
|
||||
""",
|
||||
[
|
||||
(
|
||||
game_id,
|
||||
snake.get("id"),
|
||||
snake.get("name"),
|
||||
snake.get("id") == you.get("id"),
|
||||
snake.get("customizations") or {},
|
||||
)
|
||||
for snake in snakes if snake.get("id") is not None
|
||||
],
|
||||
running = await self._running_game(conn, game_id)
|
||||
record, states, identities, points = records_from_game_state(
|
||||
game_state, my_move, my_thinking, running.points,
|
||||
)
|
||||
await conn.execute("""
|
||||
INSERT INTO turns (
|
||||
game_id, turn, observed_at, my_move, my_thinking,
|
||||
board_state, snakes, you, food, hazards
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
|
||||
ON CONFLICT (game_id, turn) DO UPDATE SET
|
||||
observed_at = EXCLUDED.observed_at,
|
||||
my_move = EXCLUDED.my_move,
|
||||
my_thinking = EXCLUDED.my_thinking,
|
||||
board_state = EXCLUDED.board_state,
|
||||
snakes = EXCLUDED.snakes,
|
||||
you = EXCLUDED.you,
|
||||
food = EXCLUDED.food,
|
||||
hazards = EXCLUDED.hazards
|
||||
""",
|
||||
game_id,
|
||||
turn,
|
||||
self._utc_now_ts(),
|
||||
my_move,
|
||||
my_thinking,
|
||||
board_json,
|
||||
snakes_json,
|
||||
you_json,
|
||||
food_json,
|
||||
hazards_json,
|
||||
)
|
||||
|
||||
previous_positions:dict[str, tuple[int, int]] = {}
|
||||
if turn > 0:
|
||||
prev_rows = await conn.fetch("""
|
||||
SELECT snake_id, head_x, head_y
|
||||
FROM snake_turns
|
||||
WHERE game_id = $1 AND turn = $2
|
||||
""",
|
||||
game_id, turn - 1,
|
||||
)
|
||||
previous_positions = {
|
||||
row["snake_id"]: (int(row["head_x"]), int(row["head_y"]))
|
||||
for row in prev_rows
|
||||
if row["head_x"] is not None and row["head_y"] is not None
|
||||
}
|
||||
|
||||
you_id = you.get("id")
|
||||
for params in self._build_snake_turn_params(snakes, you_id, game_id, turn, previous_positions):
|
||||
(p_game_id, p_turn, p_snake_id, p_name, p_health, p_length,
|
||||
p_head_x, p_head_y, p_body, p_is_you, p_inferred, p_latency) = params
|
||||
await conn.execute("""
|
||||
INSERT INTO snake_turns (
|
||||
game_id, turn, snake_id, snake_name, health, length,
|
||||
head_x, head_y, body, is_you, inferred_move, latency
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)
|
||||
ON CONFLICT (game_id, turn, snake_id) DO UPDATE SET
|
||||
snake_name = EXCLUDED.snake_name,
|
||||
health = EXCLUDED.health,
|
||||
length = EXCLUDED.length,
|
||||
head_x = EXCLUDED.head_x,
|
||||
head_y = EXCLUDED.head_y,
|
||||
body = EXCLUDED.body,
|
||||
is_you = EXCLUDED.is_you,
|
||||
inferred_move = EXCLUDED.inferred_move,
|
||||
latency = EXCLUDED.latency
|
||||
""",
|
||||
p_game_id, p_turn, p_snake_id, None, p_health, p_length,
|
||||
p_head_x, p_head_y, p_body, False, p_inferred, p_latency,
|
||||
)
|
||||
|
||||
await conn.execute("""
|
||||
UPDATE games
|
||||
SET final_turn = GREATEST(final_turn, $1)
|
||||
WHERE game_id = $2
|
||||
""",
|
||||
turn, game_id,
|
||||
missing = [identity for identity in identities if identity.snake_id not in running.snake_ids]
|
||||
if missing:
|
||||
running.snake_ids.update(await self._upsert_identities(conn, game_id, identities))
|
||||
await self._write_rows(conn, running.gid, running.snake_ids, [record], states)
|
||||
await conn.execute(
|
||||
"UPDATE games SET final_turn = GREATEST(final_turn, $1) WHERE gid = $2",
|
||||
record.turn, running.gid,
|
||||
)
|
||||
running.points = points
|
||||
self._running.put(game_id, running)
|
||||
|
||||
async def record_game_end(self, game_state:dict) -> None:
|
||||
game_id, winner_name, winner_you, turn = self._extract_game_end_params(game_state)
|
||||
@@ -585,12 +376,10 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
status = 'finished'
|
||||
WHERE game_id = $5
|
||||
""",
|
||||
self._utc_now_ts(),
|
||||
winner_name,
|
||||
winner_you,
|
||||
turn,
|
||||
game_id,
|
||||
self._utc_now_ts(), winner_name, winner_you, turn, game_id,
|
||||
)
|
||||
await self._pack_game(conn, game_id)
|
||||
self._running.drop(game_id)
|
||||
|
||||
# ── stale game finalization ────────────────────────────────────────────────
|
||||
|
||||
@@ -602,7 +391,7 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
rows = await conn.fetch("""
|
||||
SELECT game_id, started_at, final_turn, your_snake_id
|
||||
SELECT gid, game_id, started_at, final_turn, your_snake_id
|
||||
FROM games
|
||||
WHERE status = 'running'
|
||||
ORDER BY started_at ASC
|
||||
@@ -618,40 +407,9 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
continue
|
||||
|
||||
game_id = row["game_id"]
|
||||
your_snake_id = row["your_snake_id"]
|
||||
final_turn = int(row["final_turn"] or 0)
|
||||
|
||||
snake_rows = await conn.fetch("""
|
||||
SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name) AS snake_name
|
||||
FROM snake_turns AS st
|
||||
LEFT JOIN game_snakes AS gs
|
||||
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
|
||||
WHERE st.game_id = $1 AND st.turn = $2
|
||||
ORDER BY COALESCE(gs.is_you, st.is_you) DESC, snake_name ASC
|
||||
""",
|
||||
game_id, final_turn,
|
||||
)
|
||||
|
||||
if len(snake_rows) == 0:
|
||||
latest_row = await conn.fetchrow(
|
||||
"SELECT MAX(turn) AS latest_turn FROM snake_turns WHERE game_id = $1",
|
||||
game_id,
|
||||
)
|
||||
|
||||
if latest_row is not None and latest_row["latest_turn"] is not None:
|
||||
final_turn = int(latest_row["latest_turn"])
|
||||
snake_rows = await conn.fetch("""
|
||||
SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name) AS snake_name
|
||||
FROM snake_turns AS st
|
||||
LEFT JOIN game_snakes AS gs
|
||||
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
|
||||
WHERE st.game_id = $1 AND st.turn = $2
|
||||
ORDER BY COALESCE(gs.is_you, st.is_you) DESC, snake_name ASC
|
||||
""",
|
||||
game_id, final_turn,
|
||||
)
|
||||
|
||||
winner_you, survivor_name = self._calculate_survivor(snake_rows, your_snake_id)
|
||||
identities, _, states = await self._load_records(conn, game_id, row["gid"])
|
||||
final_turn, survivors = self._survivors(identities, states, int(row["final_turn"] or 0))
|
||||
winner_you, survivor_name = self._calculate_survivor(survivors, row["your_snake_id"])
|
||||
|
||||
tag = await conn.execute("""
|
||||
UPDATE games
|
||||
@@ -662,14 +420,21 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
status = 'finished'
|
||||
WHERE game_id = $5 AND status = 'running'
|
||||
""",
|
||||
self._utc_now_ts(),
|
||||
survivor_name,
|
||||
winner_you,
|
||||
final_turn,
|
||||
game_id,
|
||||
self._utc_now_ts(), survivor_name, winner_you, final_turn, game_id,
|
||||
)
|
||||
if tag and tag.endswith("1"):
|
||||
finalized += 1
|
||||
await self._pack_game(conn, game_id)
|
||||
self._running.drop(game_id)
|
||||
|
||||
# Turns that arrived after a game was packed are folded into its blob.
|
||||
late = await conn.fetch("""
|
||||
SELECT DISTINCT g.game_id
|
||||
FROM game_turns AS t JOIN games AS g ON g.gid = t.gid
|
||||
WHERE g.status = 'finished'
|
||||
""")
|
||||
for late_row in late:
|
||||
await self._pack_game(conn, late_row["game_id"])
|
||||
|
||||
return finalized
|
||||
|
||||
@@ -734,51 +499,51 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
|
||||
async def get_game_replay(self, game_id:str) -> dict|None:
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
game_row = await conn.fetchrow("""
|
||||
SELECT game_id, started_at, ended_at, width, height, source, map_name,
|
||||
ruleset_name, ruleset_version, game_type, your_snake_id, your_snake_name,
|
||||
your_snake_type, your_snake_version,
|
||||
winner_name, winner_you, final_turn, status
|
||||
FROM games
|
||||
WHERE game_id = $1 AND has_replay
|
||||
""",
|
||||
game_id,
|
||||
)
|
||||
|
||||
game_row = await conn.fetchrow(_GAME_SELECT + " WHERE game_id = $1 AND has_replay", game_id)
|
||||
if game_row is None:
|
||||
return None
|
||||
identities, turns, states = await self._load_records(conn, game_id, game_row["gid"])
|
||||
|
||||
turn_rows = await conn.fetch("""
|
||||
SELECT turn, observed_at, my_move,
|
||||
my_thinking AS my_thinking_json,
|
||||
board_state AS board_state_json,
|
||||
food AS food_json,
|
||||
hazards AS hazards_json,
|
||||
you AS you_json
|
||||
FROM turns
|
||||
WHERE game_id = $1
|
||||
ORDER BY turn ASC
|
||||
""",
|
||||
game_id,
|
||||
)
|
||||
return self._build_game_replay_output(game_row, identities, turns, states)
|
||||
|
||||
snake_rows = await conn.fetch("""
|
||||
SELECT st.turn, st.snake_id,
|
||||
COALESCE(gs.snake_name, st.snake_name) AS snake_name,
|
||||
st.health, st.length, st.head_x, st.head_y, st.body AS body_json,
|
||||
COALESCE(gs.is_you, st.is_you) AS is_you,
|
||||
COALESCE(gs.customizations, '{}'::jsonb) AS customizations_json,
|
||||
st.inferred_move, st.latency
|
||||
FROM snake_turns AS st
|
||||
LEFT JOIN game_snakes AS gs
|
||||
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
|
||||
WHERE st.game_id = $1
|
||||
ORDER BY st.turn ASC, is_you DESC, snake_name ASC
|
||||
""",
|
||||
game_id,
|
||||
)
|
||||
# ── bundles (migration / merge) ────────────────────────────────────────────
|
||||
|
||||
return self._build_game_replay_output(game_row, turn_rows, snake_rows, decode_json=lambda x: x)
|
||||
async def list_game_ids(self) -> list[str]:
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
rows = await conn.fetch("SELECT game_id FROM games ORDER BY started_at, game_id")
|
||||
return [row["game_id"] for row in rows]
|
||||
|
||||
async def read_game_bundle(self, game_id:str) -> GameBundle|None:
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
game_row = await conn.fetchrow(_GAME_SELECT + " WHERE game_id = $1", game_id)
|
||||
if game_row is None:
|
||||
return None
|
||||
identities, turns, states = await self._load_records(conn, game_id, game_row["gid"])
|
||||
return GameBundle(normalize_game(dict(game_row)), identities, turns, states)
|
||||
|
||||
async def import_game_bundle(self, bundle:GameBundle) -> None:
|
||||
game = normalize_game(bundle.game)
|
||||
columns = ", ".join(GAME_COLUMNS)
|
||||
placeholders = ", ".join(f"${index}" for index in range(1, len(GAME_COLUMNS) + 1))
|
||||
updates = ", ".join(f"{column} = EXCLUDED.{column}" for column in GAME_COLUMNS[1:])
|
||||
pool = await self._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
gid = await conn.fetchval(
|
||||
f"INSERT INTO games ({columns}) VALUES ({placeholders}) "
|
||||
f"ON CONFLICT (game_id) DO UPDATE SET {updates} RETURNING gid",
|
||||
*[game[column] for column in GAME_COLUMNS],
|
||||
)
|
||||
gsids = await self._upsert_identities(conn, game["game_id"], bundle.snakes)
|
||||
if not bundle.turns or not game["has_replay"]:
|
||||
return
|
||||
if game["status"] == "finished":
|
||||
await self._store_replay(conn, game["game_id"], gid, bundle.turns, bundle.states)
|
||||
else:
|
||||
await self._write_rows(conn, gid, gsids, bundle.turns, bundle.states)
|
||||
self._running.drop(game["game_id"])
|
||||
|
||||
# ── lifecycle ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -1,11 +1,24 @@
|
||||
from quart_common.web.env import env_bool
|
||||
"""SQLite gameplay backend with the same compact layout as PostgreSQL.
|
||||
|
||||
import asyncio, sqlite3, json, os, logging, sys
|
||||
Running games use ``game_turns``/``snake_states``; finished games are packed
|
||||
into one zstd blob per game in ``game_replays``.
|
||||
"""
|
||||
|
||||
import asyncio, json, sqlite3, logging, sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from server.database.backend.Template import GameplayBackendTemplate
|
||||
from server.database.normalized_turn import compact_turn_json
|
||||
from server.database.backend.sqlite_schema import create_schema
|
||||
from server.database.compact.cache import RunningGame, RunningGameCache
|
||||
from server.database.compact.codec import (
|
||||
decode_body, decode_move, decode_points, encode_body, encode_move, encode_points,
|
||||
)
|
||||
from server.database.compact.games import GAME_COLUMNS, normalize_game
|
||||
from server.database.compact.records import (
|
||||
GameBundle, SnakeIdentity, SnakeState, TurnRecord, merge_records, records_from_game_state,
|
||||
)
|
||||
from server.database.compact.replay_blob import REPLAY_FORMAT, pack_replay, unpack_replay
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
if not logger.handlers:
|
||||
@@ -14,37 +27,45 @@ if not logger.handlers:
|
||||
logger.addHandler(_handler)
|
||||
logger.propagate = False
|
||||
|
||||
_ZSTD_EXT = Path(os.environ.get("SQLITE_ZSTD_EXT", "/usr/local/lib/libsqlite_zstd.so")).expanduser().resolve()
|
||||
_GAME_SELECT = """
|
||||
SELECT gid, game_id, started_at, ended_at, width, height, source, map_name,
|
||||
ruleset_name, ruleset_version, game_type, your_snake_id, your_snake_name,
|
||||
your_snake_type, your_snake_version, winner_name, winner_you, final_turn,
|
||||
status, has_replay, quality_status, quality_score, quality_tier, quality_reasons_json
|
||||
FROM games
|
||||
"""
|
||||
|
||||
def _game_from_row(row:sqlite3.Row) -> dict:
|
||||
data = dict(row)
|
||||
reasons = data.pop("quality_reasons_json", None)
|
||||
data["quality_reasons"] = json.loads(reasons) if reasons else None
|
||||
return normalize_game(data)
|
||||
|
||||
class SqliteGameplayBackend(GameplayBackendTemplate):
|
||||
def __init__(self, db_path:str, busy_timeout_ms:int=5000, initialize_indexes:bool=True, journal_mode:str="WAL"):
|
||||
def __init__(self, db_path:str, busy_timeout_ms:int=5000, journal_mode:str="WAL", read_only:bool=False):
|
||||
self.db_path = db_path
|
||||
self.busy_timeout_ms = max(1000, int(busy_timeout_ms))
|
||||
self._zstd_available = False
|
||||
self._initialize_indexes = initialize_indexes
|
||||
self._journal_mode = journal_mode
|
||||
self._initialize_database()
|
||||
self._read_only = read_only
|
||||
self._running = RunningGameCache()
|
||||
if not read_only:
|
||||
self._initialize_database()
|
||||
|
||||
# ── connection ─────────────────────────────────────────────────────────────
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
if self._read_only:
|
||||
connection = sqlite3.connect(f"file:{self.db_path}?mode=ro", uri=True, isolation_level=None)
|
||||
connection.row_factory = sqlite3.Row
|
||||
connection.execute("PRAGMA query_only = ON")
|
||||
return connection
|
||||
|
||||
connection = sqlite3.connect(
|
||||
self.db_path,
|
||||
timeout=max(1, self.busy_timeout_ms // 1000),
|
||||
isolation_level=None,
|
||||
)
|
||||
connection.row_factory = sqlite3.Row
|
||||
|
||||
if _ZSTD_EXT.exists() and not env_bool('DISABLE_GAMEPLAY_DB_COMPRESSION', True):
|
||||
try:
|
||||
connection.enable_load_extension(True)
|
||||
connection.load_extension(str(_ZSTD_EXT))
|
||||
self._zstd_available = True
|
||||
except sqlite3.OperationalError as e:
|
||||
logger.warning(f"sqlite-zstd extension skipped: {e}")
|
||||
finally:
|
||||
connection.enable_load_extension(False)
|
||||
|
||||
connection.execute("PRAGMA foreign_keys = ON")
|
||||
connection.execute(f"PRAGMA journal_mode = {self._journal_mode}")
|
||||
connection.execute("PRAGMA synchronous = NORMAL")
|
||||
@@ -53,159 +74,142 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
|
||||
connection.execute(f"PRAGMA busy_timeout = {self.busy_timeout_ms}")
|
||||
return connection
|
||||
|
||||
def _ensure_auto_vacuum_full(self, connection:sqlite3.Connection) -> None:
|
||||
current = connection.execute("PRAGMA auto_vacuum").fetchone()[0]
|
||||
if current != 1:
|
||||
connection.execute("PRAGMA auto_vacuum = FULL")
|
||||
connection.execute("VACUUM")
|
||||
|
||||
# ── schema setup ───────────────────────────────────────────────────────────
|
||||
|
||||
def _initialize_database(self) -> None:
|
||||
Path(self.db_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
with self._connect() as connection:
|
||||
self._ensure_auto_vacuum_full(connection)
|
||||
connection.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS games (
|
||||
game_id TEXT PRIMARY KEY,
|
||||
started_at TEXT NOT NULL,
|
||||
ended_at TEXT,
|
||||
width INTEGER,
|
||||
height INTEGER,
|
||||
source TEXT,
|
||||
map_name TEXT,
|
||||
ruleset_name TEXT,
|
||||
ruleset_version TEXT,
|
||||
your_snake_id TEXT,
|
||||
your_snake_name TEXT,
|
||||
your_snake_type TEXT,
|
||||
your_snake_version TEXT,
|
||||
winner_name TEXT,
|
||||
winner_you INTEGER NOT NULL DEFAULT 0,
|
||||
final_turn INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL DEFAULT 'running',
|
||||
has_replay INTEGER NOT NULL DEFAULT 1,
|
||||
quality_status TEXT NOT NULL DEFAULT 'retained',
|
||||
quality_score INTEGER,
|
||||
quality_tier TEXT,
|
||||
quality_reasons_json TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS turns (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
game_id TEXT NOT NULL,
|
||||
turn INTEGER NOT NULL,
|
||||
observed_at TEXT NOT NULL,
|
||||
my_move TEXT,
|
||||
my_thinking_json TEXT,
|
||||
board_state_json TEXT NOT NULL,
|
||||
snakes_json TEXT NOT NULL,
|
||||
you_json TEXT NOT NULL,
|
||||
food_json TEXT NOT NULL,
|
||||
hazards_json TEXT NOT NULL,
|
||||
UNIQUE (game_id, turn),
|
||||
FOREIGN KEY (game_id) REFERENCES games(game_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS game_snakes (
|
||||
game_id TEXT NOT NULL,
|
||||
snake_id TEXT NOT NULL,
|
||||
snake_name TEXT,
|
||||
is_you INTEGER NOT NULL DEFAULT 0,
|
||||
customizations_json TEXT NOT NULL DEFAULT '{}',
|
||||
PRIMARY KEY (game_id, snake_id),
|
||||
FOREIGN KEY (game_id) REFERENCES games(game_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS snake_turns (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
game_id TEXT NOT NULL,
|
||||
turn INTEGER NOT NULL,
|
||||
snake_id TEXT NOT NULL,
|
||||
snake_name TEXT,
|
||||
health INTEGER,
|
||||
length INTEGER,
|
||||
head_x INTEGER,
|
||||
head_y INTEGER,
|
||||
body_json TEXT NOT NULL,
|
||||
is_you INTEGER NOT NULL DEFAULT 0,
|
||||
inferred_move TEXT,
|
||||
UNIQUE (game_id, turn, snake_id),
|
||||
FOREIGN KEY (game_id) REFERENCES games(game_id) ON DELETE CASCADE
|
||||
);
|
||||
""")
|
||||
if self._initialize_indexes:
|
||||
self._create_indexes_if_tables(connection)
|
||||
self._ensure_column_exists(connection, "turns", "my_thinking_json", "TEXT")
|
||||
self._ensure_column_exists(connection, "games", "your_snake_type", "TEXT")
|
||||
self._ensure_column_exists(connection, "games", "your_snake_version", "TEXT")
|
||||
self._ensure_column_exists(connection, "games", "game_type", "TEXT")
|
||||
self._ensure_column_exists(connection, "snake_turns", "latency", "TEXT")
|
||||
self._ensure_column_exists(connection, "game_snakes", "customizations_json", "TEXT NOT NULL DEFAULT '{}'")
|
||||
self._ensure_column_exists(connection, "games", "winner_name", "TEXT")
|
||||
self._ensure_column_exists(connection, "games", "has_replay", "INTEGER NOT NULL DEFAULT 1")
|
||||
self._ensure_column_exists(connection, "games", "quality_status", "TEXT NOT NULL DEFAULT 'retained'")
|
||||
self._ensure_column_exists(connection, "games", "quality_score", "INTEGER")
|
||||
self._ensure_column_exists(connection, "games", "quality_tier", "TEXT")
|
||||
self._ensure_column_exists(connection, "games", "quality_reasons_json", "TEXT")
|
||||
if self._zstd_available:
|
||||
self._enable_zstd_compression(connection)
|
||||
connection = self._connect()
|
||||
try:
|
||||
if connection.execute("PRAGMA auto_vacuum").fetchone()[0] != 1:
|
||||
connection.execute("PRAGMA auto_vacuum = FULL")
|
||||
connection.execute("VACUUM")
|
||||
create_schema(connection)
|
||||
connection.execute("PRAGMA optimize")
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def _create_indexes_if_tables(self, connection:sqlite3.Connection) -> None:
|
||||
real_tables = {
|
||||
row[0] for row in connection.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table'"
|
||||
).fetchall()
|
||||
}
|
||||
indexes = [
|
||||
("idx_turns_game_turn", "turns", "game_id, turn"),
|
||||
("idx_games_status", "games", "status"),
|
||||
("idx_snake_turns_game_turn", "snake_turns", "game_id, turn"),
|
||||
]
|
||||
for idx_name, table, cols in indexes:
|
||||
if table in real_tables:
|
||||
connection.execute(f"CREATE INDEX IF NOT EXISTS {idx_name} ON {table}({cols})")
|
||||
# ── record helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
def _ensure_column_exists(self, connection:sqlite3.Connection, table_name:str, column_name:str, column_type:str) -> None:
|
||||
obj = connection.execute(
|
||||
"SELECT type FROM sqlite_master WHERE name = ?", (table_name,)
|
||||
).fetchone()
|
||||
def _running_game(self, connection:sqlite3.Connection, game_id:str) -> RunningGame:
|
||||
game = self._running.get(game_id)
|
||||
if game is None:
|
||||
row = connection.execute("SELECT gid FROM games WHERE game_id = ?", (game_id,)).fetchone()
|
||||
if row is None:
|
||||
raise LookupError(f"Turn for unknown game {game_id!r}")
|
||||
game = RunningGame(gid=row["gid"])
|
||||
return game
|
||||
|
||||
if obj and obj["type"] == "view":
|
||||
underlying = f"_{table_name}_zstd"
|
||||
exists = connection.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE name = ? AND type = 'table'", (underlying,)
|
||||
def _upsert_identities(self, connection:sqlite3.Connection, game_id:str, identities:list[SnakeIdentity]) -> dict[str, int]:
|
||||
gsids = {}
|
||||
for identity in identities:
|
||||
row = connection.execute("""
|
||||
INSERT INTO game_snakes (game_id, snake_id, snake_name, is_you, customizations_json, gsid)
|
||||
VALUES (?, ?, ?, ?, ?, (SELECT COALESCE(MAX(gsid), 0) + 1 FROM game_snakes))
|
||||
ON CONFLICT(game_id, snake_id) DO UPDATE SET
|
||||
snake_name = excluded.snake_name,
|
||||
is_you = excluded.is_you,
|
||||
customizations_json = excluded.customizations_json
|
||||
RETURNING gsid
|
||||
""",
|
||||
(
|
||||
game_id, identity.snake_id, identity.name, 1 if identity.is_you else 0,
|
||||
self._to_json(identity.customizations or {}),
|
||||
),
|
||||
).fetchone()
|
||||
if not exists:
|
||||
return
|
||||
actual_table = underlying
|
||||
else:
|
||||
actual_table = table_name
|
||||
gsids[identity.snake_id] = row["gsid"]
|
||||
return gsids
|
||||
|
||||
existing = connection.execute(f"PRAGMA table_info({actual_table})").fetchall()
|
||||
if any(row["name"] == column_name for row in existing):
|
||||
return
|
||||
connection.execute(f"ALTER TABLE {actual_table} ADD COLUMN {column_name} {column_type}")
|
||||
|
||||
def _enable_zstd_compression(self, connection:sqlite3.Connection) -> None:
|
||||
compressed_columns = [
|
||||
("turns", "board_state_json"),
|
||||
("turns", "snakes_json"),
|
||||
("turns", "you_json"),
|
||||
("turns", "food_json"),
|
||||
("turns", "hazards_json"),
|
||||
("snake_turns", "body_json"),
|
||||
]
|
||||
for table, column in compressed_columns:
|
||||
try:
|
||||
connection.execute(
|
||||
"SELECT zstd_enable_transparent(?)",
|
||||
[json.dumps({"table": table, "column": column, "compression_level": 6, "dict_chooser": "'a'"})],
|
||||
def _write_rows(self, connection:sqlite3.Connection, gid:int, gsids:dict[str, int], turns:list[TurnRecord], states:list[SnakeState]) -> None:
|
||||
connection.executemany("""
|
||||
INSERT INTO game_turns (gid, turn, observed_at, my_move, thinking_json, food, hazards)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(gid, turn) DO UPDATE SET
|
||||
observed_at = excluded.observed_at,
|
||||
my_move = excluded.my_move,
|
||||
thinking_json = excluded.thinking_json,
|
||||
food = excluded.food,
|
||||
hazards = excluded.hazards
|
||||
""",
|
||||
[
|
||||
(
|
||||
gid, record.turn, self._ts_to_str(record.observed_at) or self._utc_now(),
|
||||
encode_move(record.move),
|
||||
self._to_json(record.thinking) if record.thinking is not None else None,
|
||||
encode_points(record.food), encode_points(record.hazards),
|
||||
)
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
connection.execute("SELECT zstd_incremental_maintenance(null, 1)")
|
||||
for record in turns
|
||||
],
|
||||
)
|
||||
connection.executemany("""
|
||||
INSERT INTO snake_states (gsid, turn, health, latency, body) VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(gsid, turn) DO UPDATE SET
|
||||
health = excluded.health,
|
||||
latency = excluded.latency,
|
||||
body = excluded.body
|
||||
""",
|
||||
[
|
||||
(gsids[state.snake_id], state.turn, state.health, state.latency, encode_body(state.body))
|
||||
for state in states if state.snake_id in gsids
|
||||
],
|
||||
)
|
||||
|
||||
def _load_records(self, connection:sqlite3.Connection, game_id:str, gid:int) -> tuple[list[SnakeIdentity], list[TurnRecord], list[SnakeState]]:
|
||||
identities = [
|
||||
SnakeIdentity(row["snake_id"], row["snake_name"], bool(row["is_you"]), self._from_json(row["customizations_json"]) or {})
|
||||
for row in connection.execute("""
|
||||
SELECT snake_id, snake_name, is_you, customizations_json
|
||||
FROM game_snakes WHERE game_id = ?
|
||||
""", (game_id,))
|
||||
]
|
||||
turns = [
|
||||
TurnRecord(
|
||||
turn=row["turn"], observed_at=row["observed_at"], move=decode_move(row["my_move"]),
|
||||
thinking=self._from_json(row["thinking_json"]),
|
||||
food=decode_points(row["food"]), hazards=decode_points(row["hazards"]),
|
||||
)
|
||||
for row in connection.execute("""
|
||||
SELECT turn, observed_at, my_move, thinking_json, food, hazards
|
||||
FROM game_turns WHERE gid = ? ORDER BY turn
|
||||
""", (gid,))
|
||||
]
|
||||
states = [
|
||||
SnakeState(row["snake_id"], row["turn"], row["health"], row["latency"], decode_body(row["body"]))
|
||||
for row in connection.execute("""
|
||||
SELECT gs.snake_id, s.turn, s.health, s.latency, s.body
|
||||
FROM game_snakes AS gs
|
||||
JOIN snake_states AS s ON s.gsid = gs.gsid
|
||||
WHERE gs.game_id = ?
|
||||
""", (game_id,))
|
||||
]
|
||||
blob = connection.execute("SELECT format, data FROM game_replays WHERE gid = ?", (gid,)).fetchone()
|
||||
if blob is not None:
|
||||
packed_turns, packed_states = unpack_replay(blob["data"], blob["format"])
|
||||
turns, states = merge_records(packed_turns, packed_states, turns, states)
|
||||
return identities, turns, states
|
||||
|
||||
def _store_replay(self, connection:sqlite3.Connection, game_id:str, gid:int, turns:list[TurnRecord], states:list[SnakeState]) -> None:
|
||||
connection.execute("""
|
||||
INSERT INTO game_replays (gid, format, turn_count, data) VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(gid) DO UPDATE SET
|
||||
format = excluded.format, turn_count = excluded.turn_count, data = excluded.data
|
||||
""", (gid, REPLAY_FORMAT, len(turns), pack_replay(turns, states)))
|
||||
connection.execute("DELETE FROM game_turns WHERE gid = ?", (gid,))
|
||||
connection.execute(
|
||||
"DELETE FROM snake_states WHERE gsid IN (SELECT gsid FROM game_snakes WHERE game_id = ?)",
|
||||
(game_id,),
|
||||
)
|
||||
|
||||
def _pack_game(self, connection:sqlite3.Connection, game_id:str) -> None:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
row = connection.execute("SELECT gid FROM games WHERE game_id = ?", (game_id,)).fetchone()
|
||||
if row is not None and connection.execute(
|
||||
"SELECT 1 FROM game_turns WHERE gid = ? LIMIT 1", (row["gid"],),
|
||||
).fetchone():
|
||||
_, turns, states = self._load_records(connection, game_id, row["gid"])
|
||||
self._store_replay(connection, game_id, row["gid"], turns, states)
|
||||
connection.execute("COMMIT")
|
||||
except Exception:
|
||||
connection.execute("ROLLBACK")
|
||||
raise
|
||||
|
||||
# ── sync write methods ─────────────────────────────────────────────────────
|
||||
|
||||
@@ -216,13 +220,15 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
|
||||
ruleset = game.get("ruleset", {})
|
||||
game_type = self._derive_game_type(board, ruleset)
|
||||
|
||||
with self._connect() as connection:
|
||||
connection = self._connect()
|
||||
try:
|
||||
connection.execute("""
|
||||
INSERT INTO games (
|
||||
game_id, started_at, width, height, source, map_name,
|
||||
ruleset_name, ruleset_version, your_snake_id, your_snake_name,
|
||||
your_snake_type, your_snake_version, game_type, status
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'running')
|
||||
your_snake_type, your_snake_version, game_type, status, gid
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'running',
|
||||
(SELECT COALESCE(MAX(gid), 0) + 1 FROM games))
|
||||
ON CONFLICT(game_id) DO UPDATE SET
|
||||
width = excluded.width,
|
||||
height = excluded.height,
|
||||
@@ -238,166 +244,71 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
|
||||
status = 'running'
|
||||
""",
|
||||
(
|
||||
game.get("id"),
|
||||
self._utc_now(),
|
||||
board.get("width"),
|
||||
board.get("height"),
|
||||
game.get("source"),
|
||||
game.get("map"),
|
||||
ruleset.get("name"),
|
||||
ruleset.get("version"),
|
||||
you.get("id"),
|
||||
you.get("name"),
|
||||
snake_type,
|
||||
snake_version,
|
||||
game_type,
|
||||
game.get("id"), self._utc_now(), board.get("width"), board.get("height"),
|
||||
game.get("source"), game.get("map"), ruleset.get("name"), ruleset.get("version"),
|
||||
you.get("id"), you.get("name"), snake_type, snake_version, game_type,
|
||||
),
|
||||
)
|
||||
connection.execute("PRAGMA wal_checkpoint(PASSIVE)")
|
||||
connection.execute("PRAGMA optimize")
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def _record_turn_sync(self, game_state:dict, my_move:str|None, my_thinking:dict|None) -> None:
|
||||
game = game_state.get("game", {})
|
||||
board = game_state.get("board", {})
|
||||
snakes = self._extract_snakes(game_state)
|
||||
you = self._extract_you(game_state)
|
||||
game_id = game.get("id")
|
||||
turn = int(game_state.get("turn", 0))
|
||||
|
||||
board_json, snakes_json, you_json, food_json, hazards_json = compact_turn_json(board)
|
||||
|
||||
with self._connect() as connection:
|
||||
connection.executemany("""
|
||||
INSERT INTO game_snakes (
|
||||
game_id, snake_id, snake_name, is_you, customizations_json
|
||||
) VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(game_id, snake_id) DO UPDATE SET
|
||||
snake_name = excluded.snake_name,
|
||||
is_you = excluded.is_you,
|
||||
customizations_json = excluded.customizations_json
|
||||
""",
|
||||
[
|
||||
(
|
||||
game_id,
|
||||
snake.get("id"),
|
||||
snake.get("name"),
|
||||
1 if snake.get("id") == you.get("id") else 0,
|
||||
self._to_json(snake.get("customizations") or {}),
|
||||
)
|
||||
for snake in snakes if snake.get("id") is not None
|
||||
],
|
||||
)
|
||||
connection.execute("""
|
||||
INSERT INTO turns (
|
||||
game_id, turn, observed_at, my_move, my_thinking_json,
|
||||
board_state_json, snakes_json, you_json, food_json, hazards_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(game_id, turn) DO UPDATE SET
|
||||
observed_at = excluded.observed_at,
|
||||
my_move = excluded.my_move,
|
||||
my_thinking_json = excluded.my_thinking_json,
|
||||
board_state_json = excluded.board_state_json,
|
||||
snakes_json = excluded.snakes_json,
|
||||
you_json = excluded.you_json,
|
||||
food_json = excluded.food_json,
|
||||
hazards_json = excluded.hazards_json
|
||||
""",
|
||||
(
|
||||
game_id,
|
||||
turn,
|
||||
self._utc_now(),
|
||||
my_move,
|
||||
self._to_json(my_thinking) if my_thinking is not None else None,
|
||||
self._to_json(board_json),
|
||||
self._to_json(snakes_json),
|
||||
self._to_json(you_json),
|
||||
self._to_json(food_json),
|
||||
self._to_json(hazards_json),
|
||||
),
|
||||
)
|
||||
|
||||
previous_positions: dict[str, tuple[int, int]] = {}
|
||||
if turn > 0:
|
||||
previous_rows = connection.execute("""
|
||||
SELECT snake_id, head_x, head_y
|
||||
FROM snake_turns
|
||||
WHERE game_id = ? AND turn = ?
|
||||
""",
|
||||
(game_id, turn - 1),
|
||||
).fetchall()
|
||||
previous_positions = {
|
||||
row["snake_id"]: (int(row["head_x"]), int(row["head_y"]))
|
||||
for row in previous_rows
|
||||
if row["head_x"] is not None and row["head_y"] is not None
|
||||
}
|
||||
|
||||
you_id = you.get("id")
|
||||
for params in self._build_snake_turn_params(snakes, you_id, game_id, turn, previous_positions):
|
||||
(p_game_id, p_turn, p_snake_id, p_name, p_health, p_length,
|
||||
p_head_x, p_head_y, p_body, p_is_you, p_inferred, p_latency) = params
|
||||
connection.execute("""
|
||||
INSERT INTO snake_turns (
|
||||
game_id, turn, snake_id, snake_name, health, length,
|
||||
head_x, head_y, body_json, is_you, inferred_move, latency
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(game_id, turn, snake_id) DO UPDATE SET
|
||||
snake_name = excluded.snake_name,
|
||||
health = excluded.health,
|
||||
length = excluded.length,
|
||||
head_x = excluded.head_x,
|
||||
head_y = excluded.head_y,
|
||||
body_json = excluded.body_json,
|
||||
is_you = excluded.is_you,
|
||||
inferred_move = excluded.inferred_move,
|
||||
latency = excluded.latency
|
||||
""",
|
||||
(
|
||||
p_game_id, p_turn, p_snake_id, None, p_health, p_length,
|
||||
p_head_x, p_head_y, self._to_json(p_body),
|
||||
0,
|
||||
p_inferred, p_latency,
|
||||
),
|
||||
game_id = game_state.get("game", {}).get("id")
|
||||
connection = self._connect()
|
||||
try:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
running = self._running_game(connection, game_id)
|
||||
record, states, identities, points = records_from_game_state(
|
||||
game_state, my_move, my_thinking, running.points,
|
||||
)
|
||||
|
||||
connection.execute("""
|
||||
UPDATE games
|
||||
SET final_turn = CASE WHEN ? > final_turn THEN ? ELSE final_turn END
|
||||
WHERE game_id = ?
|
||||
""",
|
||||
(turn, turn, game_id),
|
||||
)
|
||||
if any(identity.snake_id not in running.snake_ids for identity in identities):
|
||||
running.snake_ids.update(self._upsert_identities(connection, game_id, identities))
|
||||
self._write_rows(connection, running.gid, running.snake_ids, [record], states)
|
||||
connection.execute(
|
||||
"UPDATE games SET final_turn = MAX(final_turn, ?) WHERE gid = ?",
|
||||
(record.turn, running.gid),
|
||||
)
|
||||
connection.execute("COMMIT")
|
||||
except Exception:
|
||||
connection.execute("ROLLBACK")
|
||||
raise
|
||||
finally:
|
||||
connection.close()
|
||||
running.points = points
|
||||
self._running.put(game_id, running)
|
||||
|
||||
def _record_game_end_sync(self, game_state:dict) -> None:
|
||||
game_id, winner_name, winner_you, turn = self._extract_game_end_params(game_state)
|
||||
|
||||
with self._connect() as connection:
|
||||
connection = self._connect()
|
||||
try:
|
||||
connection.execute("""
|
||||
UPDATE games
|
||||
SET ended_at = ?,
|
||||
winner_name = ?,
|
||||
winner_you = ?,
|
||||
final_turn = CASE WHEN ? > final_turn THEN ? ELSE final_turn END,
|
||||
final_turn = MAX(final_turn, ?),
|
||||
status = 'finished'
|
||||
WHERE game_id = ?
|
||||
""",
|
||||
(
|
||||
self._utc_now(),
|
||||
winner_name,
|
||||
1 if winner_you else 0,
|
||||
turn,
|
||||
turn,
|
||||
game_id,
|
||||
),
|
||||
(self._utc_now(), winner_name, 1 if winner_you else 0, turn, game_id),
|
||||
)
|
||||
self._pack_game(connection, game_id)
|
||||
finally:
|
||||
connection.close()
|
||||
self._running.drop(game_id)
|
||||
|
||||
def _finalize_stale_running_games_sync(self, stale_after_seconds:int=600) -> int:
|
||||
threshold = max(60, int(stale_after_seconds))
|
||||
now_utc = datetime.now(timezone.utc)
|
||||
finalized = 0
|
||||
|
||||
with self._connect() as connection:
|
||||
connection = self._connect()
|
||||
try:
|
||||
rows = connection.execute("""
|
||||
SELECT game_id, started_at, final_turn, your_snake_id
|
||||
SELECT gid, game_id, started_at, final_turn, your_snake_id
|
||||
FROM games
|
||||
WHERE status = 'running'
|
||||
ORDER BY started_at ASC
|
||||
@@ -405,73 +316,47 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
|
||||
|
||||
for row in rows:
|
||||
started_at = self._parse_utc_timestamp(row["started_at"])
|
||||
if started_at is None:
|
||||
continue
|
||||
if (now_utc - started_at).total_seconds() < threshold:
|
||||
if started_at is None or (now_utc - started_at).total_seconds() < threshold:
|
||||
continue
|
||||
|
||||
game_id = row["game_id"]
|
||||
your_snake_id = row["your_snake_id"]
|
||||
final_turn = int(row["final_turn"] or 0)
|
||||
|
||||
snake_rows = connection.execute("""
|
||||
SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name) AS snake_name
|
||||
FROM snake_turns AS st
|
||||
LEFT JOIN game_snakes AS gs
|
||||
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
|
||||
WHERE st.game_id = ? AND st.turn = ?
|
||||
ORDER BY COALESCE(gs.is_you, st.is_you) DESC, snake_name ASC
|
||||
""",
|
||||
(game_id, final_turn),
|
||||
).fetchall()
|
||||
|
||||
if len(snake_rows) == 0:
|
||||
latest_row = connection.execute(
|
||||
"SELECT MAX(turn) AS latest_turn FROM snake_turns WHERE game_id = ?",
|
||||
(game_id,),
|
||||
).fetchone()
|
||||
if latest_row is not None and latest_row["latest_turn"] is not None:
|
||||
final_turn = int(latest_row["latest_turn"])
|
||||
snake_rows = connection.execute("""
|
||||
SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name) AS snake_name
|
||||
FROM snake_turns AS st
|
||||
LEFT JOIN game_snakes AS gs
|
||||
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
|
||||
WHERE st.game_id = ? AND st.turn = ?
|
||||
ORDER BY COALESCE(gs.is_you, st.is_you) DESC, snake_name ASC
|
||||
""",
|
||||
(game_id, final_turn),
|
||||
).fetchall()
|
||||
|
||||
winner_you, survivor_name = self._calculate_survivor(snake_rows, your_snake_id)
|
||||
identities, _, states = self._load_records(connection, game_id, row["gid"])
|
||||
final_turn, survivors = self._survivors(identities, states, int(row["final_turn"] or 0))
|
||||
winner_you, survivor_name = self._calculate_survivor(survivors, row["your_snake_id"])
|
||||
|
||||
result = connection.execute("""
|
||||
UPDATE games
|
||||
SET ended_at = ?,
|
||||
winner_name = ?,
|
||||
winner_you = ?,
|
||||
final_turn = CASE WHEN ? > final_turn THEN ? ELSE final_turn END,
|
||||
final_turn = MAX(final_turn, ?),
|
||||
status = 'finished'
|
||||
WHERE game_id = ? AND status = 'running'
|
||||
""",
|
||||
(
|
||||
self._utc_now(),
|
||||
survivor_name,
|
||||
1 if winner_you else 0,
|
||||
final_turn,
|
||||
final_turn,
|
||||
game_id,
|
||||
),
|
||||
(self._utc_now(), survivor_name, 1 if winner_you else 0, final_turn, game_id),
|
||||
)
|
||||
if result.rowcount > 0:
|
||||
finalized += 1
|
||||
self._pack_game(connection, game_id)
|
||||
self._running.drop(game_id)
|
||||
|
||||
late = connection.execute("""
|
||||
SELECT DISTINCT g.game_id
|
||||
FROM game_turns AS t JOIN games AS g ON g.gid = t.gid
|
||||
WHERE g.status = 'finished'
|
||||
""").fetchall()
|
||||
for late_row in late:
|
||||
self._pack_game(connection, late_row["game_id"])
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
return finalized
|
||||
|
||||
# ── sync read methods ──────────────────────────────────────────────────────
|
||||
|
||||
def _get_summary_sync(self, recent_limit:int=15) -> dict:
|
||||
with self._connect() as connection:
|
||||
connection = self._connect()
|
||||
try:
|
||||
totals = connection.execute("""
|
||||
SELECT
|
||||
COUNT(*) AS total_games,
|
||||
@@ -505,11 +390,14 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
|
||||
""",
|
||||
(max(1, int(recent_limit)),),
|
||||
).fetchall()
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
return self._build_summary_output(totals, by_type, recent)
|
||||
|
||||
def _list_games_sync(self, limit:int=50) -> list[dict]:
|
||||
with self._connect() as connection:
|
||||
connection = self._connect()
|
||||
try:
|
||||
rows = connection.execute("""
|
||||
SELECT game_id, started_at, ended_at, map_name, source, ruleset_name, game_type,
|
||||
your_snake_name, your_snake_type, your_snake_version,
|
||||
@@ -521,52 +409,75 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
|
||||
""",
|
||||
(max(1, int(limit)),),
|
||||
).fetchall()
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
return self._build_game_list_output(rows)
|
||||
|
||||
def _get_game_replay_sync(self, game_id:str) -> dict|None:
|
||||
with self._connect() as connection:
|
||||
game_row = connection.execute("""
|
||||
SELECT game_id, started_at, ended_at, width, height, source, map_name,
|
||||
ruleset_name, ruleset_version, game_type, your_snake_id, your_snake_name,
|
||||
your_snake_type, your_snake_version,
|
||||
winner_name, winner_you, final_turn, status
|
||||
FROM games
|
||||
WHERE game_id = ? AND has_replay = 1
|
||||
""",
|
||||
(game_id,),
|
||||
).fetchone()
|
||||
|
||||
connection = self._connect()
|
||||
try:
|
||||
game_row = connection.execute(_GAME_SELECT + " WHERE game_id = ? AND has_replay = 1", (game_id,)).fetchone()
|
||||
if game_row is None:
|
||||
return None
|
||||
identities, turns, states = self._load_records(connection, game_id, game_row["gid"])
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
turn_rows = connection.execute("""
|
||||
SELECT turn, observed_at, my_move, my_thinking_json,
|
||||
board_state_json, food_json, hazards_json, you_json
|
||||
FROM turns
|
||||
WHERE game_id = ?
|
||||
ORDER BY turn ASC
|
||||
""",
|
||||
(game_id,),
|
||||
).fetchall()
|
||||
return self._build_game_replay_output(game_row, identities, turns, states)
|
||||
|
||||
snake_rows = connection.execute("""
|
||||
SELECT st.turn, st.snake_id,
|
||||
COALESCE(gs.snake_name, st.snake_name) AS snake_name,
|
||||
st.health, st.length, st.head_x, st.head_y, st.body_json,
|
||||
COALESCE(gs.is_you, st.is_you) AS is_you,
|
||||
COALESCE(gs.customizations_json, '{}') AS customizations_json,
|
||||
st.inferred_move, st.latency
|
||||
FROM snake_turns AS st
|
||||
LEFT JOIN game_snakes AS gs
|
||||
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
|
||||
WHERE st.game_id = ?
|
||||
ORDER BY st.turn ASC, is_you DESC, snake_name ASC
|
||||
""",
|
||||
(game_id,),
|
||||
).fetchall()
|
||||
# ── bundles (migration / merge) ────────────────────────────────────────────
|
||||
|
||||
return self._build_game_replay_output(game_row, turn_rows, snake_rows, decode_json=self._from_json)
|
||||
def list_game_ids_sync(self) -> list[str]:
|
||||
connection = self._connect()
|
||||
try:
|
||||
return [row[0] for row in connection.execute("SELECT game_id FROM games ORDER BY started_at, game_id")]
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def read_game_bundle_sync(self, game_id:str) -> GameBundle|None:
|
||||
connection = self._connect()
|
||||
try:
|
||||
row = connection.execute(_GAME_SELECT + " WHERE game_id = ?", (game_id,)).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
identities, turns, states = self._load_records(connection, game_id, row["gid"])
|
||||
finally:
|
||||
connection.close()
|
||||
return GameBundle(_game_from_row(row), identities, turns, states)
|
||||
|
||||
def import_game_bundle_sync(self, bundle:GameBundle) -> None:
|
||||
game = normalize_game(bundle.game)
|
||||
columns = [column for column in GAME_COLUMNS if column != "quality_reasons"] + ["quality_reasons_json"]
|
||||
values = [game[column] for column in GAME_COLUMNS if column != "quality_reasons"]
|
||||
values.append(self._to_json(game["quality_reasons"]) if game["quality_reasons"] is not None else None)
|
||||
values = [self._ts_to_str(value) if isinstance(value, datetime) else value for value in values]
|
||||
updates = ", ".join(f"{column} = excluded.{column}" for column in columns[1:])
|
||||
|
||||
connection = self._connect()
|
||||
try:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
connection.execute(
|
||||
f"INSERT INTO games ({', '.join(columns)}, gid) "
|
||||
f"VALUES ({', '.join('?' for _ in columns)}, (SELECT COALESCE(MAX(gid), 0) + 1 FROM games)) "
|
||||
f"ON CONFLICT(game_id) DO UPDATE SET {updates}",
|
||||
values,
|
||||
)
|
||||
gid = connection.execute("SELECT gid FROM games WHERE game_id = ?", (game["game_id"],)).fetchone()["gid"]
|
||||
gsids = self._upsert_identities(connection, game["game_id"], bundle.snakes)
|
||||
if bundle.turns and game["has_replay"]:
|
||||
if game["status"] == "finished":
|
||||
self._store_replay(connection, game["game_id"], gid, bundle.turns, bundle.states)
|
||||
else:
|
||||
self._write_rows(connection, gid, gsids, bundle.turns, bundle.states)
|
||||
connection.execute("COMMIT")
|
||||
except Exception:
|
||||
connection.execute("ROLLBACK")
|
||||
raise
|
||||
finally:
|
||||
connection.close()
|
||||
self._running.drop(game["game_id"])
|
||||
|
||||
# ── public async interface ─────────────────────────────────────────────────
|
||||
|
||||
@@ -591,5 +502,14 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
|
||||
async def get_game_replay(self, game_id:str) -> dict|None:
|
||||
return await asyncio.to_thread(self._get_game_replay_sync, game_id)
|
||||
|
||||
async def list_game_ids(self) -> list[str]:
|
||||
return await asyncio.to_thread(self.list_game_ids_sync)
|
||||
|
||||
async def read_game_bundle(self, game_id:str) -> GameBundle|None:
|
||||
return await asyncio.to_thread(self.read_game_bundle_sync, game_id)
|
||||
|
||||
async def import_game_bundle(self, bundle:GameBundle) -> None:
|
||||
await asyncio.to_thread(self.import_game_bundle_sync, bundle)
|
||||
|
||||
async def close(self) -> None:
|
||||
return None
|
||||
|
||||
@@ -2,7 +2,8 @@ import json
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from server.database.normalized_turn import hydrate_replay_turns
|
||||
from server.database.compact.hydrate import hydrate_replay_turns
|
||||
from server.database.compact.records import GameBundle, SnakeIdentity, SnakeState, TurnRecord
|
||||
|
||||
class GameplayBackendTemplate:
|
||||
"""Abstract base for gameplay database backends.
|
||||
@@ -40,6 +41,17 @@ class GameplayBackendTemplate:
|
||||
async def get_game_replay(self, game_id:str) -> dict|None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def read_game_bundle(self, game_id:str) -> GameBundle|None:
|
||||
"""Return all stored data of one game (used by migrations and merges)."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def import_game_bundle(self, bundle:GameBundle) -> None:
|
||||
"""Store a complete game; finished games are packed into a replay blob."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def list_game_ids(self) -> list[str]:
|
||||
raise NotImplementedError
|
||||
|
||||
async def close(self) -> None:
|
||||
return None
|
||||
|
||||
@@ -79,22 +91,6 @@ class GameplayBackendTemplate:
|
||||
def _extract_you(self, game_state:dict) -> dict:
|
||||
return dict(game_state.get("you", {}))
|
||||
|
||||
def _infer_direction(self, old_head:tuple[int, int]|None, new_head:tuple[int, int]|None) -> str|None:
|
||||
if old_head is None or new_head is None:
|
||||
return None
|
||||
|
||||
dx = new_head[0] - old_head[0]
|
||||
dy = new_head[1] - old_head[1]
|
||||
if dx == 1 and dy == 0:
|
||||
return "right"
|
||||
if dx == -1 and dy == 0:
|
||||
return "left"
|
||||
if dx == 0 and dy == 1:
|
||||
return "up"
|
||||
if dx == 0 and dy == -1:
|
||||
return "down"
|
||||
return None
|
||||
|
||||
def _derive_game_type(self, board:dict, ruleset:dict) -> str:
|
||||
if len(board.get("snakes", [])) == 2:
|
||||
return "duel"
|
||||
@@ -159,14 +155,21 @@ class GameplayBackendTemplate:
|
||||
"status": row["status"],
|
||||
} for row in rows]
|
||||
|
||||
def _build_game_replay_output(self, game_row, turn_rows, snake_rows, decode_json) -> dict:
|
||||
"""Build the full replay dict.
|
||||
|
||||
decode_json: callable applied to raw column values that may be JSON strings
|
||||
(SQLite) or already-decoded objects (PostgreSQL). Pass self._from_json for
|
||||
SQLite; pass (lambda x: x) for PostgreSQL.
|
||||
"""
|
||||
hydrated_turns = hydrate_replay_turns(game_row, turn_rows, snake_rows, decode_json)
|
||||
def _build_game_replay_output(
|
||||
self,
|
||||
game_row,
|
||||
identities:list[SnakeIdentity],
|
||||
turns:list[TurnRecord],
|
||||
states:list[SnakeState],
|
||||
) -> dict:
|
||||
game = {
|
||||
"width": game_row["width"],
|
||||
"height": game_row["height"],
|
||||
"your_snake_id": game_row["your_snake_id"],
|
||||
}
|
||||
hydrated_turns = hydrate_replay_turns(
|
||||
game, {identity.snake_id: identity for identity in identities}, turns, states,
|
||||
)
|
||||
|
||||
return {
|
||||
"game": {
|
||||
@@ -197,45 +200,6 @@ class GameplayBackendTemplate:
|
||||
|
||||
# ── shared write-path helpers ──────────────────────────────────────────────
|
||||
|
||||
def _build_snake_turn_params(
|
||||
self,
|
||||
snakes:list[dict],
|
||||
you_id:str|None,
|
||||
game_id:str|None,
|
||||
turn:int,
|
||||
previous_positions:dict,
|
||||
) -> list[tuple]:
|
||||
"""Return one parameter tuple per snake for a snake_turns INSERT."""
|
||||
result = []
|
||||
for snake in snakes:
|
||||
snake_id = snake.get("id")
|
||||
if snake_id is None:
|
||||
continue
|
||||
head = snake.get("head", {})
|
||||
head_x = head.get("x")
|
||||
head_y = head.get("y")
|
||||
new_head = (
|
||||
(int(head_x), int(head_y))
|
||||
if head_x is not None and head_y is not None
|
||||
else None
|
||||
)
|
||||
inferred = self._infer_direction(previous_positions.get(snake_id), new_head)
|
||||
result.append((
|
||||
game_id,
|
||||
turn,
|
||||
snake_id,
|
||||
snake.get("name"),
|
||||
snake.get("health"),
|
||||
snake.get("length"),
|
||||
head_x,
|
||||
head_y,
|
||||
snake.get("body", []),
|
||||
snake_id == you_id,
|
||||
inferred,
|
||||
snake.get("latency"),
|
||||
))
|
||||
return result
|
||||
|
||||
def _extract_game_end_params(self, game_state:dict) -> tuple:
|
||||
"""Return (game_id, winner_name, winner_you, turn) from a game_end payload."""
|
||||
game = game_state.get("game", {})
|
||||
@@ -249,6 +213,25 @@ class GameplayBackendTemplate:
|
||||
turn = int(game_state.get("turn", 0))
|
||||
return game_id, winner_name, winner_you, turn
|
||||
|
||||
def _survivors(self, identities:list[SnakeIdentity], states:list[SnakeState], final_turn:int) -> tuple[int, list[dict]]:
|
||||
"""Snakes alive at the final (or latest stored) turn, you first, then by name."""
|
||||
turns = {state.turn for state in states}
|
||||
if final_turn not in turns and turns:
|
||||
final_turn = max(turns)
|
||||
by_id = {identity.snake_id: identity for identity in identities}
|
||||
alive = []
|
||||
for state in states:
|
||||
if state.turn != final_turn:
|
||||
continue
|
||||
identity = by_id.get(state.snake_id)
|
||||
alive.append({
|
||||
"snake_id": state.snake_id,
|
||||
"snake_name": identity.name if identity else None,
|
||||
"is_you": bool(identity and identity.is_you),
|
||||
})
|
||||
alive.sort(key=lambda row: (not row["is_you"], row["snake_name"] is None, row["snake_name"] or ""))
|
||||
return final_turn, alive
|
||||
|
||||
def _calculate_survivor(self, snake_rows, your_snake_id:str|None) -> tuple[bool, str|None]:
|
||||
"""Return (winner_you, survivor_name) from a list of snake_turns rows."""
|
||||
survivor_ids = [s["snake_id"] for s in snake_rows if s["snake_id"]]
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
"""PostgreSQL schema for compact gameplay storage.
|
||||
|
||||
Finished games live in ``game_replays`` as one zstd blob per game. Running
|
||||
games use ``game_turns`` and ``snake_states`` keyed by small integer ids
|
||||
(``games.gid`` / ``game_snakes.gsid``) instead of repeating text ids per row.
|
||||
"""
|
||||
|
||||
TABLES_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS games (
|
||||
game_id TEXT PRIMARY KEY,
|
||||
started_at TIMESTAMPTZ NOT NULL,
|
||||
ended_at TIMESTAMPTZ,
|
||||
width INTEGER,
|
||||
height INTEGER,
|
||||
source TEXT,
|
||||
map_name TEXT,
|
||||
ruleset_name TEXT,
|
||||
ruleset_version TEXT,
|
||||
your_snake_id TEXT,
|
||||
your_snake_name TEXT,
|
||||
your_snake_type TEXT,
|
||||
your_snake_version TEXT,
|
||||
game_type TEXT,
|
||||
winner_name TEXT,
|
||||
winner_you BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
final_turn INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL DEFAULT 'running',
|
||||
has_replay BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
quality_status TEXT NOT NULL DEFAULT 'retained',
|
||||
quality_score INTEGER,
|
||||
quality_tier TEXT,
|
||||
quality_reasons JSONB
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS game_snakes (
|
||||
game_id TEXT NOT NULL REFERENCES games(game_id) ON DELETE CASCADE,
|
||||
snake_id TEXT NOT NULL,
|
||||
snake_name TEXT,
|
||||
is_you BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
customizations JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
PRIMARY KEY (game_id, snake_id)
|
||||
);
|
||||
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS game_type TEXT;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS your_snake_type TEXT;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS your_snake_version TEXT;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS winner_name TEXT;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS has_replay BOOLEAN NOT NULL DEFAULT TRUE;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_status TEXT NOT NULL DEFAULT 'retained';
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_score INTEGER;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_tier TEXT;
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_reasons JSONB;
|
||||
ALTER TABLE game_snakes ADD COLUMN IF NOT EXISTS customizations JSONB NOT NULL DEFAULT '{}'::jsonb;
|
||||
|
||||
ALTER TABLE games ADD COLUMN IF NOT EXISTS gid INTEGER GENERATED BY DEFAULT AS IDENTITY;
|
||||
ALTER TABLE game_snakes ADD COLUMN IF NOT EXISTS gsid INTEGER GENERATED BY DEFAULT AS IDENTITY;
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'games_gid_key') THEN
|
||||
ALTER TABLE games ADD CONSTRAINT games_gid_key UNIQUE (gid);
|
||||
END IF;
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'game_snakes_gsid_key') THEN
|
||||
ALTER TABLE game_snakes ADD CONSTRAINT game_snakes_gsid_key UNIQUE (gsid);
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS game_turns (
|
||||
gid INTEGER NOT NULL REFERENCES games(gid) ON DELETE CASCADE,
|
||||
turn INTEGER NOT NULL,
|
||||
observed_at TIMESTAMPTZ NOT NULL,
|
||||
my_move SMALLINT,
|
||||
thinking JSONB,
|
||||
food BYTEA,
|
||||
hazards BYTEA,
|
||||
PRIMARY KEY (gid, turn)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS snake_states (
|
||||
gsid INTEGER NOT NULL REFERENCES game_snakes(gsid) ON DELETE CASCADE,
|
||||
turn INTEGER NOT NULL,
|
||||
health SMALLINT,
|
||||
latency SMALLINT,
|
||||
body BYTEA NOT NULL,
|
||||
PRIMARY KEY (gsid, turn)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS game_replays (
|
||||
gid INTEGER PRIMARY KEY REFERENCES games(gid) ON DELETE CASCADE,
|
||||
format SMALLINT NOT NULL,
|
||||
turn_count INTEGER NOT NULL,
|
||||
data BYTEA NOT NULL
|
||||
);
|
||||
|
||||
-- Replay blobs are already zstd-compressed; skip TOAST compression attempts.
|
||||
ALTER TABLE game_replays ALTER COLUMN data SET STORAGE EXTERNAL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_games_status ON games(status);
|
||||
"""
|
||||
|
||||
# Superseded by the UNIQUE constraints of the legacy turns/snake_turns tables.
|
||||
# CONCURRENTLY avoids blocking writes; each must run outside a transaction.
|
||||
DROP_REDUNDANT_INDEXES = (
|
||||
"DROP INDEX CONCURRENTLY IF EXISTS idx_turns_game_turn",
|
||||
"DROP INDEX CONCURRENTLY IF EXISTS idx_snake_turns_game_turn",
|
||||
)
|
||||
@@ -0,0 +1,114 @@
|
||||
"""SQLite schema for compact gameplay storage (mirrors postgresql_schema)."""
|
||||
|
||||
import sqlite3
|
||||
|
||||
BASE_TABLES = """
|
||||
CREATE TABLE IF NOT EXISTS games (
|
||||
game_id TEXT PRIMARY KEY,
|
||||
started_at TEXT NOT NULL,
|
||||
ended_at TEXT,
|
||||
width INTEGER,
|
||||
height INTEGER,
|
||||
source TEXT,
|
||||
map_name TEXT,
|
||||
ruleset_name TEXT,
|
||||
ruleset_version TEXT,
|
||||
your_snake_id TEXT,
|
||||
your_snake_name TEXT,
|
||||
your_snake_type TEXT,
|
||||
your_snake_version TEXT,
|
||||
game_type TEXT,
|
||||
winner_name TEXT,
|
||||
winner_you INTEGER NOT NULL DEFAULT 0,
|
||||
final_turn INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL DEFAULT 'running',
|
||||
has_replay INTEGER NOT NULL DEFAULT 1,
|
||||
quality_status TEXT NOT NULL DEFAULT 'retained',
|
||||
quality_score INTEGER,
|
||||
quality_tier TEXT,
|
||||
quality_reasons_json TEXT,
|
||||
gid INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS game_snakes (
|
||||
game_id TEXT NOT NULL,
|
||||
snake_id TEXT NOT NULL,
|
||||
snake_name TEXT,
|
||||
is_you INTEGER NOT NULL DEFAULT 0,
|
||||
customizations_json TEXT NOT NULL DEFAULT '{}',
|
||||
gsid INTEGER,
|
||||
PRIMARY KEY (game_id, snake_id),
|
||||
FOREIGN KEY (game_id) REFERENCES games(game_id) ON DELETE CASCADE
|
||||
);
|
||||
"""
|
||||
|
||||
# Columns added over time; older databases get them via ALTER TABLE.
|
||||
LEGACY_COLUMNS = (
|
||||
("games", "your_snake_type", "TEXT"),
|
||||
("games", "your_snake_version", "TEXT"),
|
||||
("games", "game_type", "TEXT"),
|
||||
("games", "winner_name", "TEXT"),
|
||||
("games", "has_replay", "INTEGER NOT NULL DEFAULT 1"),
|
||||
("games", "quality_status", "TEXT NOT NULL DEFAULT 'retained'"),
|
||||
("games", "quality_score", "INTEGER"),
|
||||
("games", "quality_tier", "TEXT"),
|
||||
("games", "quality_reasons_json", "TEXT"),
|
||||
("games", "gid", "INTEGER"),
|
||||
("game_snakes", "customizations_json", "TEXT NOT NULL DEFAULT '{}'"),
|
||||
("game_snakes", "gsid", "INTEGER"),
|
||||
)
|
||||
|
||||
COMPACT_TABLES = """
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS games_gid_key ON games(gid);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS game_snakes_gsid_key ON game_snakes(gsid);
|
||||
CREATE INDEX IF NOT EXISTS idx_games_status ON games(status);
|
||||
|
||||
UPDATE games
|
||||
SET gid = rowid + (SELECT COALESCE(MAX(gid), 0) FROM games)
|
||||
WHERE gid IS NULL;
|
||||
|
||||
UPDATE game_snakes
|
||||
SET gsid = rowid + (SELECT COALESCE(MAX(gsid), 0) FROM game_snakes)
|
||||
WHERE gsid IS NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS game_turns (
|
||||
gid INTEGER NOT NULL REFERENCES games(gid) ON DELETE CASCADE,
|
||||
turn INTEGER NOT NULL,
|
||||
observed_at TEXT NOT NULL,
|
||||
my_move INTEGER,
|
||||
thinking_json TEXT,
|
||||
food BLOB,
|
||||
hazards BLOB,
|
||||
PRIMARY KEY (gid, turn)
|
||||
) WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS snake_states (
|
||||
gsid INTEGER NOT NULL REFERENCES game_snakes(gsid) ON DELETE CASCADE,
|
||||
turn INTEGER NOT NULL,
|
||||
health INTEGER,
|
||||
latency INTEGER,
|
||||
body BLOB NOT NULL,
|
||||
PRIMARY KEY (gsid, turn)
|
||||
) WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS game_replays (
|
||||
gid INTEGER PRIMARY KEY REFERENCES games(gid) ON DELETE CASCADE,
|
||||
format INTEGER NOT NULL,
|
||||
turn_count INTEGER NOT NULL,
|
||||
data BLOB NOT NULL
|
||||
);
|
||||
|
||||
DROP INDEX IF EXISTS idx_turns_game_turn;
|
||||
DROP INDEX IF EXISTS idx_snake_turns_game_turn;
|
||||
"""
|
||||
|
||||
def ensure_column(connection:sqlite3.Connection, table:str, column:str, column_type:str) -> None:
|
||||
existing = {row[1] for row in connection.execute(f"PRAGMA table_info({table})")}
|
||||
if column not in existing:
|
||||
connection.execute(f"ALTER TABLE {table} ADD COLUMN {column} {column_type}")
|
||||
|
||||
def create_schema(connection:sqlite3.Connection) -> None:
|
||||
connection.executescript(BASE_TABLES)
|
||||
for table, column, column_type in LEGACY_COLUMNS:
|
||||
ensure_column(connection, table, column, column_type)
|
||||
connection.executescript(COMPACT_TABLES)
|
||||
@@ -1,108 +1,57 @@
|
||||
"""Load sampled gameplay positions from SQLite or PostgreSQL."""
|
||||
"""Load sampled gameplay positions from a SQLite or PostgreSQL gameplay DB."""
|
||||
import asyncio
|
||||
import json
|
||||
from urllib.parse import urlparse
|
||||
|
||||
def is_postgresql_source(source: str) -> bool:
|
||||
def is_postgresql_source(source:str) -> bool:
|
||||
return urlparse(source).scheme.lower() in {"postgres", "postgresql"}
|
||||
|
||||
def _decode_json(value, default):
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, str):
|
||||
return json.loads(value)
|
||||
return value
|
||||
|
||||
def _build_state(row, snake_rows) -> tuple[dict, dict] | None:
|
||||
board = _decode_json(row[1], {})
|
||||
you = _decode_json(row[2], {})
|
||||
if not board.get("snakes"):
|
||||
snakes = []
|
||||
for snake_row in snake_rows:
|
||||
snake_id = snake_row[0]
|
||||
snake_name = snake_row[1] or (row[6] if snake_id == row[5] else snake_id)
|
||||
snakes.append({
|
||||
"id": snake_id,
|
||||
"name": snake_name,
|
||||
"health": snake_row[2],
|
||||
"length": snake_row[3],
|
||||
"head": {"x": snake_row[4], "y": snake_row[5]},
|
||||
"body": _decode_json(snake_row[6], []),
|
||||
"customizations": _decode_json(snake_row[7], {}),
|
||||
})
|
||||
board = {
|
||||
"width": row[7],
|
||||
"height": row[8],
|
||||
"food": _decode_json(row[3], []),
|
||||
"hazards": _decode_json(row[4], []),
|
||||
"snakes": snakes,
|
||||
}
|
||||
if not you:
|
||||
you = next(
|
||||
(snake for snake in board.get("snakes", []) if snake.get("id") == row[5]),
|
||||
{},
|
||||
)
|
||||
def replay_turn_to_state(game:dict, turn:dict) -> tuple[dict, dict]|None:
|
||||
board = turn.get("board") or {}
|
||||
you = turn.get("you") or {}
|
||||
if not you or not board.get("snakes"):
|
||||
return None
|
||||
return board, {
|
||||
"you": you,
|
||||
"game_id": row[9],
|
||||
"source": row[10] or "custom",
|
||||
"map": row[11] or "standard",
|
||||
"game_id": game["game_id"],
|
||||
"source": game.get("source") or "custom",
|
||||
"map": game.get("map") or "standard",
|
||||
"ruleset": {
|
||||
"name": row[12] or "standard",
|
||||
"version": row[13] or "v1.0.0",
|
||||
"name": game.get("ruleset_name") or "standard",
|
||||
"version": game.get("ruleset_version") or "v1.0.0",
|
||||
"settings": {},
|
||||
},
|
||||
"turn": int(row[14]),
|
||||
"turn": int(turn["turn"]),
|
||||
}
|
||||
|
||||
async def _load_postgresql_states(dsn: str, samples: int, stride: int) -> list[tuple[dict, dict]]:
|
||||
try:
|
||||
import asyncpg
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("asyncpg is required for PostgreSQL benchmark sources") from exc
|
||||
async def sample_states(backend, samples:int, stride:int) -> list[tuple[dict, dict]]:
|
||||
"""Every ``stride``-th stored turn, newest games first."""
|
||||
states = []
|
||||
seen = 0
|
||||
for game_id in reversed(await backend.list_game_ids()):
|
||||
replay = await backend.get_game_replay(game_id)
|
||||
if replay is None:
|
||||
continue
|
||||
for turn in reversed(replay["turns"]):
|
||||
if seen % stride == 0:
|
||||
state = replay_turn_to_state(replay["game"], turn)
|
||||
if state is not None:
|
||||
states.append(state)
|
||||
if len(states) >= samples:
|
||||
return states
|
||||
seen += 1
|
||||
return states
|
||||
|
||||
connection = await asyncpg.connect(dsn=dsn)
|
||||
async def _load_states(source:str, samples:int, stride:int) -> list[tuple[dict, dict]]:
|
||||
if is_postgresql_source(source):
|
||||
from server.database.backend.PostgresqlGameplayBackend import PostgresqlGameplayBackend # noqa: PLC0415
|
||||
backend = PostgresqlGameplayBackend(source)
|
||||
else:
|
||||
from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend # noqa: PLC0415
|
||||
backend = SqliteGameplayBackend(source, read_only=True)
|
||||
try:
|
||||
max_id = int(await connection.fetchval("SELECT max(id) FROM turns") or 0)
|
||||
if max_id == 0:
|
||||
return []
|
||||
query = """
|
||||
SELECT t.id, t.board_state, t.you, t.food, t.hazards,
|
||||
g.your_snake_id, g.your_snake_name, g.width, g.height,
|
||||
g.game_id, g.source, g.map_name,
|
||||
g.ruleset_name, g.ruleset_version, t.turn
|
||||
FROM turns AS t
|
||||
JOIN games AS g ON g.game_id = t.game_id
|
||||
WHERE t.id >= $1
|
||||
ORDER BY t.id
|
||||
LIMIT 1
|
||||
"""
|
||||
snake_query = """
|
||||
SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name),
|
||||
st.health, st.length, st.head_x, st.head_y, st.body,
|
||||
COALESCE(gs.customizations, '{}'::jsonb)
|
||||
FROM snake_turns AS st
|
||||
LEFT JOIN game_snakes AS gs
|
||||
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
|
||||
WHERE st.game_id = $1 AND st.turn = $2
|
||||
ORDER BY st.id
|
||||
"""
|
||||
states = []
|
||||
next_id = max(1, max_id - (samples - 1) * stride)
|
||||
while len(states) < samples and next_id <= max_id:
|
||||
row = await connection.fetchrow(query, next_id)
|
||||
if row is None:
|
||||
break
|
||||
snake_rows = await connection.fetch(snake_query, row[9], row[14])
|
||||
state = _build_state(row, snake_rows)
|
||||
if state is not None:
|
||||
states.append(state)
|
||||
next_id = int(row[0]) + stride
|
||||
return states
|
||||
return await sample_states(backend, samples, stride)
|
||||
finally:
|
||||
await connection.close()
|
||||
await backend.close()
|
||||
|
||||
def load_postgresql_states(dsn: str, samples: int, stride: int) -> list[tuple[dict, dict]]:
|
||||
return asyncio.run(_load_postgresql_states(dsn, samples, stride))
|
||||
def load_states(source:str, samples:int, stride:int) -> list[tuple[dict, dict]]:
|
||||
return asyncio.run(_load_states(source, max(1, samples), max(1, stride)))
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Compact gameplay storage: codecs, records, replay blobs and hydration."""
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Per-process cache of running games to avoid repeated lookups per turn."""
|
||||
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .records import PointsState
|
||||
|
||||
@dataclass
|
||||
class RunningGame:
|
||||
gid:int
|
||||
snake_ids:dict[str, int] = field(default_factory=dict)
|
||||
points:PointsState|None = None
|
||||
|
||||
class RunningGameCache:
|
||||
def __init__(self, max_games:int=512):
|
||||
self._games:OrderedDict[str, RunningGame] = OrderedDict()
|
||||
self._max_games = max(1, max_games)
|
||||
|
||||
def get(self, game_id:str) -> RunningGame|None:
|
||||
game = self._games.get(game_id)
|
||||
if game is not None:
|
||||
self._games.move_to_end(game_id)
|
||||
return game
|
||||
|
||||
def put(self, game_id:str, game:RunningGame) -> RunningGame:
|
||||
self._games[game_id] = game
|
||||
self._games.move_to_end(game_id)
|
||||
while len(self._games) > self._max_games:
|
||||
self._games.popitem(last=False)
|
||||
return game
|
||||
|
||||
def drop(self, game_id:str) -> None:
|
||||
self._games.pop(game_id, None)
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Byte-level encodings for compact gameplay storage.
|
||||
|
||||
All encodings are lossless. Values that do not fit the compact form (for
|
||||
example coordinates outside 0..255) fall back to a JSON payload, marked by the
|
||||
leading format byte, so unusual maps can never corrupt stored data.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
|
||||
MOVES = ("up", "down", "left", "right")
|
||||
_MOVE_CODES = {move: index + 1 for index, move in enumerate(MOVES)}
|
||||
|
||||
_FORMAT_JSON = 0
|
||||
_FORMAT_COMPACT = 1
|
||||
|
||||
# Body step codes relative to the previous segment.
|
||||
_SAME, _UP, _DOWN, _LEFT, _RIGHT, _ABSOLUTE = range(6)
|
||||
_STEPS = {(0, 0): _SAME, (0, 1): _UP, (0, -1): _DOWN, (-1, 0): _LEFT, (1, 0): _RIGHT}
|
||||
_STEP_DELTAS = {code: delta for delta, code in _STEPS.items()}
|
||||
|
||||
def encode_move(move:str|None) -> int|None:
|
||||
return _MOVE_CODES.get(move) if move is not None else None
|
||||
|
||||
def decode_move(code:int|None) -> str|None:
|
||||
if not code:
|
||||
return None
|
||||
return MOVES[int(code) - 1]
|
||||
|
||||
def encode_latency(latency:object) -> int|None:
|
||||
if latency is None or latency == "":
|
||||
return None
|
||||
try:
|
||||
value = int(str(latency))
|
||||
except ValueError:
|
||||
return None
|
||||
return value if -32768 <= value <= 32767 else None
|
||||
|
||||
def decode_latency(value:int|None) -> str|None:
|
||||
return None if value is None else str(value)
|
||||
|
||||
def _point(point:object) -> tuple[int, int]|None:
|
||||
if not isinstance(point, dict):
|
||||
return None
|
||||
x, y = point.get("x"), point.get("y")
|
||||
if type(x) is not int or type(y) is not int:
|
||||
return None
|
||||
return x, y
|
||||
|
||||
def _fits_byte(*values:int) -> bool:
|
||||
return all(0 <= value <= 255 for value in values)
|
||||
|
||||
def _json_fallback(value:object) -> bytes:
|
||||
return bytes([_FORMAT_JSON]) + json.dumps(value, separators=(",", ":")).encode()
|
||||
|
||||
def encode_body(body:list) -> bytes:
|
||||
"""Head as absolute byte pair, then one step code per following segment."""
|
||||
points = [_point(segment) for segment in body]
|
||||
if any(point is None for point in points) or not all(_fits_byte(*point) for point in points):
|
||||
return _json_fallback(body)
|
||||
|
||||
output = bytearray([_FORMAT_COMPACT])
|
||||
previous = None
|
||||
for x, y in points:
|
||||
if previous is None:
|
||||
output += bytes((x, y))
|
||||
else:
|
||||
step = _STEPS.get((x - previous[0], y - previous[1]))
|
||||
output += bytes((step,)) if step is not None else bytes((_ABSOLUTE, x, y))
|
||||
previous = (x, y)
|
||||
return bytes(output)
|
||||
|
||||
def decode_body(data:bytes|memoryview|None) -> list[dict]:
|
||||
if not data:
|
||||
return []
|
||||
data = bytes(data)
|
||||
if data[0] == _FORMAT_JSON:
|
||||
return json.loads(data[1:])
|
||||
|
||||
body = []
|
||||
index = 1
|
||||
if len(data) >= 3:
|
||||
x, y = data[1], data[2]
|
||||
body.append({"x": x, "y": y})
|
||||
index = 3
|
||||
while index < len(data):
|
||||
code = data[index]
|
||||
if code == _ABSOLUTE:
|
||||
x, y = data[index + 1], data[index + 2]
|
||||
index += 3
|
||||
else:
|
||||
dx, dy = _STEP_DELTAS[code]
|
||||
x, y = x + dx, y + dy
|
||||
index += 1
|
||||
body.append({"x": x, "y": y})
|
||||
return body
|
||||
|
||||
def encode_points(points:list|None) -> bytes|None:
|
||||
"""Food/hazard lists as byte pairs. ``None`` means "unchanged"."""
|
||||
if points is None:
|
||||
return None
|
||||
pairs = [_point(point) for point in points]
|
||||
if any(pair is None for pair in pairs) or not all(_fits_byte(*pair) for pair in pairs):
|
||||
return _json_fallback(points)
|
||||
output = bytearray([_FORMAT_COMPACT])
|
||||
for x, y in pairs:
|
||||
output += bytes((x, y))
|
||||
return bytes(output)
|
||||
|
||||
def decode_points(data:bytes|memoryview|None) -> list[dict]|None:
|
||||
if data is None:
|
||||
return None
|
||||
data = bytes(data)
|
||||
if not data:
|
||||
return []
|
||||
if data[0] == _FORMAT_JSON:
|
||||
return json.loads(data[1:])
|
||||
return [{"x": data[i], "y": data[i + 1]} for i in range(1, len(data) - 1, 2)]
|
||||
|
||||
def flatten_points(points:list[dict]) -> list:
|
||||
"""Compact JSON form used inside replay blobs: [x0, y0, x1, y1, ...]."""
|
||||
pairs = [_point(point) for point in points]
|
||||
if any(pair is None for pair in pairs):
|
||||
return [points]
|
||||
return [value for pair in pairs for value in pair]
|
||||
|
||||
def unflatten_points(values:list) -> list[dict]:
|
||||
if len(values) == 1 and isinstance(values[0], list):
|
||||
return values[0]
|
||||
return [{"x": values[i], "y": values[i + 1]} for i in range(0, len(values), 2)]
|
||||
|
||||
def to_epoch_us(value:datetime|str|None) -> int|None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, str):
|
||||
value = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
delta = value - datetime(1970, 1, 1, tzinfo=timezone.utc)
|
||||
return (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds
|
||||
|
||||
def from_epoch_us(value:int|None) -> datetime|None:
|
||||
if value is None:
|
||||
return None
|
||||
seconds, micros = divmod(int(value), 1_000_000)
|
||||
return datetime.fromtimestamp(seconds, tz=timezone.utc).replace(microsecond=micros)
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Column list of the ``games`` table shared by backends and migrations."""
|
||||
|
||||
GAME_COLUMNS = (
|
||||
"game_id", "started_at", "ended_at", "width", "height", "source", "map_name",
|
||||
"ruleset_name", "ruleset_version", "your_snake_id", "your_snake_name",
|
||||
"your_snake_type", "your_snake_version", "game_type", "winner_name",
|
||||
"winner_you", "final_turn", "status", "has_replay", "quality_status",
|
||||
"quality_score", "quality_tier", "quality_reasons",
|
||||
)
|
||||
|
||||
GAME_DEFAULTS = {
|
||||
"winner_you": False,
|
||||
"final_turn": 0,
|
||||
"status": "running",
|
||||
"has_replay": True,
|
||||
"quality_status": "retained",
|
||||
}
|
||||
|
||||
def normalize_game(row:dict) -> dict:
|
||||
"""Return a dict with every games column, filling legacy gaps with defaults."""
|
||||
game = {column: row.get(column, GAME_DEFAULTS.get(column)) for column in GAME_COLUMNS}
|
||||
for column, default in GAME_DEFAULTS.items():
|
||||
if game[column] is None:
|
||||
game[column] = default
|
||||
game["winner_you"] = bool(game["winner_you"])
|
||||
game["has_replay"] = bool(game["has_replay"])
|
||||
game["final_turn"] = int(game["final_turn"] or 0)
|
||||
return game
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Rebuild the dashboard replay payload from compact records."""
|
||||
|
||||
from .codec import decode_latency
|
||||
from .records import SnakeIdentity, SnakeState, TurnRecord, you_view
|
||||
from .thinking import derived_values, restore_thinking
|
||||
|
||||
_DIRECTIONS = {(1, 0): "right", (-1, 0): "left", (0, 1): "up", (0, -1): "down"}
|
||||
|
||||
def _head(state:SnakeState) -> dict|None:
|
||||
return state.body[0] if state.body else None
|
||||
|
||||
def _inferred_move(previous:SnakeState|None, current:SnakeState) -> str|None:
|
||||
if previous is None:
|
||||
return None
|
||||
old, new = _head(previous), _head(current)
|
||||
if old is None or new is None:
|
||||
return None
|
||||
return _DIRECTIONS.get((new["x"] - old["x"], new["y"] - old["y"]))
|
||||
|
||||
def _sort_key(identity:SnakeIdentity|None):
|
||||
name = identity.name if identity else None
|
||||
is_you = bool(identity and identity.is_you)
|
||||
return (not is_you, name is None, name or "")
|
||||
|
||||
def hydrate_replay_turns(
|
||||
game:dict,
|
||||
identities:dict[str, SnakeIdentity],
|
||||
turns:list[TurnRecord],
|
||||
states:list[SnakeState],
|
||||
) -> list[dict]:
|
||||
states_by_turn:dict[int, dict[str, SnakeState]] = {}
|
||||
for state in states:
|
||||
states_by_turn.setdefault(state.turn, {})[state.snake_id] = state
|
||||
|
||||
width, height = game.get("width"), game.get("height")
|
||||
you_id = game.get("your_snake_id")
|
||||
food:list = []
|
||||
hazards:list = []
|
||||
output = []
|
||||
for record in sorted(turns, key=lambda item: item.turn):
|
||||
food = food if record.food is None else record.food
|
||||
hazards = hazards if record.hazards is None else record.hazards
|
||||
current = states_by_turn.get(record.turn, {})
|
||||
previous = states_by_turn.get(record.turn - 1, {})
|
||||
ordered = sorted(current.values(), key=lambda s: _sort_key(identities.get(s.snake_id)))
|
||||
|
||||
snakes = []
|
||||
api_snakes = []
|
||||
for state in ordered:
|
||||
identity = identities.get(state.snake_id)
|
||||
name = identity.name if identity else None
|
||||
customizations = identity.customizations if identity else {}
|
||||
latency = decode_latency(state.latency)
|
||||
api_snake = {
|
||||
"id": state.snake_id,
|
||||
"name": name,
|
||||
"health": state.health,
|
||||
"length": len(state.body),
|
||||
"head": _head(state) or {"x": None, "y": None},
|
||||
"body": state.body,
|
||||
"customizations": customizations,
|
||||
}
|
||||
if latency is not None:
|
||||
api_snake["latency"] = latency
|
||||
api_snakes.append(api_snake)
|
||||
snakes.append({
|
||||
"snake_id": state.snake_id,
|
||||
"snake_name": name,
|
||||
"health": state.health,
|
||||
"length": len(state.body),
|
||||
"head": api_snake["head"],
|
||||
"body": state.body,
|
||||
"customizations": customizations,
|
||||
"is_you": bool(identity and identity.is_you),
|
||||
"inferred_move": _inferred_move(previous.get(state.snake_id), state),
|
||||
"latency": latency,
|
||||
})
|
||||
|
||||
derived = derived_values(
|
||||
record.turn, record.move, you_view(list(current.values()), you_id),
|
||||
food, width, height, len(current),
|
||||
)
|
||||
output.append({
|
||||
"turn": record.turn,
|
||||
"observed_at": record.observed_at,
|
||||
"my_move": record.move,
|
||||
"my_thinking": restore_thinking(record.thinking, derived),
|
||||
"board": {
|
||||
"width": width,
|
||||
"height": height,
|
||||
"food": food,
|
||||
"hazards": hazards,
|
||||
"snakes": api_snakes,
|
||||
},
|
||||
"food": food,
|
||||
"hazards": hazards,
|
||||
"you": next((snake for snake in api_snakes if snake["id"] == you_id), {}),
|
||||
"snakes": snakes,
|
||||
})
|
||||
return output
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Rate a game's replay quality from its records."""
|
||||
|
||||
from server.database.game_quality import GameQuality, GameQualityInput, rate_game_quality
|
||||
|
||||
from .codec import MOVES
|
||||
from .records import GameBundle
|
||||
|
||||
def rate_bundle(bundle:GameBundle) -> GameQuality:
|
||||
turns = bundle.turns
|
||||
numbers = [record.turn for record in turns]
|
||||
moves = [record.move for record in turns]
|
||||
return rate_game_quality(GameQualityInput(
|
||||
status=bundle.game.get("status") or "running",
|
||||
final_turn=int(bundle.game.get("final_turn") or 0),
|
||||
turn_rows=len(turns),
|
||||
min_turn=min(numbers) if numbers else None,
|
||||
max_turn=max(numbers) if numbers else None,
|
||||
valid_moves=sum(1 for move in moves if move in MOVES),
|
||||
thinking_rows=sum(1 for record in turns if record.thinking not in (None, {})),
|
||||
distinct_moves=len({move for move in moves if move is not None}),
|
||||
snake_turn_rows=len(bundle.states),
|
||||
winner_name=bundle.game.get("winner_name"),
|
||||
))
|
||||
@@ -0,0 +1,136 @@
|
||||
"""In-memory records shared by both backends, the replay blob and migrations.
|
||||
|
||||
A ``TurnRecord`` is already in compact form: ``thinking`` has derivable fields
|
||||
stripped, and ``food``/``hazards`` are ``None`` when unchanged since the
|
||||
previous stored turn.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from .codec import encode_latency
|
||||
from .thinking import derived_values, slim_thinking
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SnakeIdentity:
|
||||
snake_id:str
|
||||
name:str|None
|
||||
is_you:bool
|
||||
customizations:dict = field(default_factory=dict)
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SnakeState:
|
||||
snake_id:str
|
||||
turn:int
|
||||
health:int|None
|
||||
latency:int|None
|
||||
body:list
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TurnRecord:
|
||||
turn:int
|
||||
observed_at:datetime|None
|
||||
move:str|None
|
||||
thinking:object
|
||||
food:list|None
|
||||
hazards:list|None
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PointsState:
|
||||
"""Last stored food/hazards, used to store only changes."""
|
||||
turn:int
|
||||
food:list
|
||||
hazards:list
|
||||
|
||||
@dataclass
|
||||
class GameBundle:
|
||||
game:dict
|
||||
snakes:list[SnakeIdentity]
|
||||
turns:list[TurnRecord]
|
||||
states:list[SnakeState]
|
||||
|
||||
def you_view(states:list[SnakeState], you_id:str|None) -> dict|None:
|
||||
state = next((s for s in states if s.snake_id == you_id), None)
|
||||
if state is None:
|
||||
return None
|
||||
return {
|
||||
"head": state.body[0] if state.body else None,
|
||||
"health": state.health,
|
||||
"length": len(state.body),
|
||||
}
|
||||
|
||||
def compact_turn(
|
||||
turn:int,
|
||||
observed_at:datetime|None,
|
||||
move:str|None,
|
||||
thinking:object,
|
||||
food:list,
|
||||
hazards:list,
|
||||
states:list[SnakeState],
|
||||
you_id:str|None,
|
||||
width:int|None,
|
||||
height:int|None,
|
||||
previous:PointsState|None,
|
||||
) -> tuple[TurnRecord, PointsState]:
|
||||
derived = derived_values(turn, move, you_view(states, you_id), food, width, height, len(states))
|
||||
unchanged = previous is not None and previous.turn == turn - 1
|
||||
record = TurnRecord(
|
||||
turn=turn,
|
||||
observed_at=observed_at,
|
||||
move=move,
|
||||
thinking=slim_thinking(thinking, derived),
|
||||
food=None if unchanged and previous.food == food else food,
|
||||
hazards=None if unchanged and previous.hazards == hazards else hazards,
|
||||
)
|
||||
return record, PointsState(turn, food, hazards)
|
||||
|
||||
def records_from_game_state(
|
||||
game_state:dict,
|
||||
my_move:str|None,
|
||||
thinking:object,
|
||||
previous:PointsState|None,
|
||||
) -> tuple[TurnRecord, list[SnakeState], list[SnakeIdentity], PointsState]:
|
||||
board = game_state.get("board", {})
|
||||
you_id = game_state.get("you", {}).get("id")
|
||||
turn = int(game_state.get("turn", 0))
|
||||
identities = []
|
||||
states = []
|
||||
for snake in board.get("snakes", []):
|
||||
snake_id = snake.get("id")
|
||||
if snake_id is None:
|
||||
continue
|
||||
identities.append(SnakeIdentity(
|
||||
snake_id=snake_id,
|
||||
name=snake.get("name"),
|
||||
is_you=snake_id == you_id,
|
||||
customizations=snake.get("customizations") or {},
|
||||
))
|
||||
states.append(SnakeState(
|
||||
snake_id=snake_id,
|
||||
turn=turn,
|
||||
health=snake.get("health"),
|
||||
latency=encode_latency(snake.get("latency")),
|
||||
body=list(snake.get("body", [])),
|
||||
))
|
||||
record, points = compact_turn(
|
||||
turn, datetime.now(timezone.utc), my_move, thinking,
|
||||
list(board.get("food", [])), list(board.get("hazards", [])),
|
||||
states, you_id, board.get("width"), board.get("height"), previous,
|
||||
)
|
||||
return record, states, identities, points
|
||||
|
||||
def merge_records(
|
||||
base_turns:list[TurnRecord],
|
||||
base_states:list[SnakeState],
|
||||
turns:list[TurnRecord],
|
||||
states:list[SnakeState],
|
||||
) -> tuple[list[TurnRecord], list[SnakeState]]:
|
||||
"""Combine a packed replay with rows written after it was packed."""
|
||||
merged_turns = {record.turn: record for record in base_turns}
|
||||
merged_turns.update({record.turn: record for record in turns})
|
||||
merged_states = {(state.snake_id, state.turn): state for state in base_states}
|
||||
merged_states.update({(state.snake_id, state.turn): state for state in states})
|
||||
return (
|
||||
sorted(merged_turns.values(), key=lambda record: record.turn),
|
||||
sorted(merged_states.values(), key=lambda state: (state.turn, state.snake_id)),
|
||||
)
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Pack a finished game's turns into one zstd-compressed blob.
|
||||
|
||||
Consecutive turns are nearly identical, so compressing a whole game at once is
|
||||
far smaller than storing one row per turn and snake.
|
||||
"""
|
||||
|
||||
from compression import zstd
|
||||
import json
|
||||
|
||||
from .codec import (
|
||||
decode_move, encode_move, flatten_points, from_epoch_us, to_epoch_us,
|
||||
unflatten_points,
|
||||
)
|
||||
from .records import SnakeState, TurnRecord
|
||||
|
||||
REPLAY_FORMAT = 1
|
||||
_LEVEL = 10
|
||||
|
||||
def _optional_points(points:list|None) -> list|None:
|
||||
return None if points is None else flatten_points(points)
|
||||
|
||||
def _optional_unflatten(values:list|None) -> list|None:
|
||||
return None if values is None else unflatten_points(values)
|
||||
|
||||
def pack_replay(turns:list[TurnRecord], states:list[SnakeState]) -> bytes:
|
||||
by_snake:dict[str, list] = {}
|
||||
for state in sorted(states, key=lambda s: (s.snake_id, s.turn)):
|
||||
by_snake.setdefault(state.snake_id, []).append(
|
||||
[state.turn, state.health, state.latency, flatten_points(state.body)]
|
||||
)
|
||||
payload = {
|
||||
"t": [
|
||||
[
|
||||
record.turn, to_epoch_us(record.observed_at), encode_move(record.move),
|
||||
record.thinking, _optional_points(record.food), _optional_points(record.hazards),
|
||||
]
|
||||
for record in sorted(turns, key=lambda item: item.turn)
|
||||
],
|
||||
"s": by_snake,
|
||||
}
|
||||
raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode()
|
||||
return zstd.compress(raw, level=_LEVEL)
|
||||
|
||||
def unpack_replay(data:bytes|memoryview, format_version:int=REPLAY_FORMAT) -> tuple[list[TurnRecord], list[SnakeState]]:
|
||||
if format_version != REPLAY_FORMAT:
|
||||
raise ValueError(f"Unsupported replay format: {format_version}")
|
||||
payload = json.loads(zstd.decompress(bytes(data)))
|
||||
turns = [
|
||||
TurnRecord(
|
||||
turn=turn, observed_at=from_epoch_us(observed), move=decode_move(move),
|
||||
thinking=thinking, food=_optional_unflatten(food), hazards=_optional_unflatten(hazards),
|
||||
)
|
||||
for turn, observed, move, thinking, food, hazards in payload["t"]
|
||||
]
|
||||
states = [
|
||||
SnakeState(
|
||||
snake_id=snake_id, turn=turn, health=health, latency=latency,
|
||||
body=unflatten_points(body),
|
||||
)
|
||||
for snake_id, rows in payload["s"].items()
|
||||
for turn, health, latency, body in rows
|
||||
]
|
||||
return turns, states
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Strip move-reasoning fields that can be rebuilt from the stored turn.
|
||||
|
||||
Snakes put values like ``turn``, ``health`` or ``head`` into their reasoning
|
||||
payload although the same values are stored in the turn and snake rows. Such a
|
||||
field is only removed when it equals the derived value, and a bitmask records
|
||||
which fields were removed, so restoring is exact.
|
||||
"""
|
||||
|
||||
DERIVED_KEYS = ("turn", "move", "head", "health", "length", "food_count", "board_size", "snakes")
|
||||
MASK_KEY = "_derived"
|
||||
_RAW_KEY = "_raw"
|
||||
|
||||
def derived_values(turn:int, move:str|None, you:dict|None, food:list, width:int|None, height:int|None, snake_count:int) -> dict:
|
||||
you = you or {}
|
||||
return {
|
||||
"turn": turn,
|
||||
"move": move,
|
||||
"head": you.get("head"),
|
||||
"health": you.get("health"),
|
||||
"length": you.get("length"),
|
||||
"food_count": len(food),
|
||||
"board_size": f"{width}x{height}",
|
||||
"snakes": snake_count,
|
||||
}
|
||||
|
||||
def _same(stored:object, derived:object) -> bool:
|
||||
# bool is an int subclass; never treat True == 1 as derivable.
|
||||
return type(stored) is type(derived) and stored == derived
|
||||
|
||||
def slim_thinking(thinking:object, derived:dict) -> object:
|
||||
if not isinstance(thinking, dict):
|
||||
return thinking
|
||||
if MASK_KEY in thinking:
|
||||
# A payload that already uses the marker key is stored verbatim.
|
||||
return {MASK_KEY: 0, _RAW_KEY: thinking}
|
||||
mask = 0
|
||||
slim = {}
|
||||
for key, value in thinking.items():
|
||||
if key in DERIVED_KEYS and _same(value, derived[key]):
|
||||
mask |= 1 << DERIVED_KEYS.index(key)
|
||||
else:
|
||||
slim[key] = value
|
||||
if mask:
|
||||
slim[MASK_KEY] = mask
|
||||
return slim
|
||||
|
||||
def restore_thinking(thinking:object, derived:dict) -> object:
|
||||
if not isinstance(thinking, dict) or MASK_KEY not in thinking:
|
||||
return thinking
|
||||
mask = int(thinking[MASK_KEY])
|
||||
if mask == 0 and _RAW_KEY in thinking:
|
||||
return thinking[_RAW_KEY]
|
||||
restored = {key: value for key, value in thinking.items() if key != MASK_KEY}
|
||||
for index, key in enumerate(DERIVED_KEYS):
|
||||
if mask & (1 << index):
|
||||
restored[key] = derived[key]
|
||||
return restored
|
||||
@@ -0,0 +1 @@
|
||||
"""Read the legacy turns/snake_turns layout and migrate it to compact storage."""
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Convert rows of the legacy layout (turns/snake_turns) into a compact bundle."""
|
||||
|
||||
import json
|
||||
|
||||
from server.database.compact.codec import encode_latency
|
||||
from server.database.compact.games import normalize_game
|
||||
from server.database.compact.records import (
|
||||
GameBundle, PointsState, SnakeIdentity, SnakeState, compact_turn,
|
||||
)
|
||||
|
||||
class LegacyDataError(ValueError):
|
||||
"""Legacy rows that cannot be converted without losing data."""
|
||||
|
||||
def parse_json(value:object, default:object=None) -> object:
|
||||
if value is None or value == "":
|
||||
return default
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
try:
|
||||
return json.loads(value)
|
||||
except (json.JSONDecodeError, TypeError) as exc:
|
||||
raise LegacyDataError(f"Invalid JSON value: {value[:120]}") from exc
|
||||
|
||||
def _state_from_row(row:dict) -> SnakeState:
|
||||
body = parse_json(row.get("body"), []) or []
|
||||
head_x, head_y, length = row.get("head_x"), row.get("head_y"), row.get("length")
|
||||
if length is not None and int(length) != len(body):
|
||||
raise LegacyDataError(f"snake {row['snake_id']} turn {row['turn']}: length {length} != body {len(body)}")
|
||||
if body and head_x is not None and (head_x, head_y) != (body[0].get("x"), body[0].get("y")):
|
||||
raise LegacyDataError(f"snake {row['snake_id']} turn {row['turn']}: head differs from body[0]")
|
||||
return SnakeState(
|
||||
snake_id=row["snake_id"], turn=int(row["turn"]), health=row.get("health"),
|
||||
latency=encode_latency(row.get("latency")), body=body,
|
||||
)
|
||||
|
||||
def _states_from_board(board:dict, turn:int) -> list[SnakeState]:
|
||||
return [
|
||||
SnakeState(
|
||||
snake_id=snake["id"], turn=turn, health=snake.get("health"),
|
||||
latency=encode_latency(snake.get("latency")), body=list(snake.get("body", [])),
|
||||
)
|
||||
for snake in board.get("snakes", []) if snake.get("id") is not None
|
||||
]
|
||||
|
||||
def _identities(game:dict, identity_rows:list[dict], snake_rows:list[dict], board_snakes:list[dict]) -> list[SnakeIdentity]:
|
||||
identities:dict[str, SnakeIdentity] = {}
|
||||
for row in identity_rows:
|
||||
identities[row["snake_id"]] = SnakeIdentity(
|
||||
row["snake_id"], row.get("snake_name"), bool(row.get("is_you")),
|
||||
parse_json(row.get("customizations"), {}) or {},
|
||||
)
|
||||
# Older databases have an empty or partial game_snakes table.
|
||||
for row in snake_rows + board_snakes:
|
||||
snake_id = row.get("snake_id") or row.get("id")
|
||||
if snake_id is None or snake_id in identities:
|
||||
continue
|
||||
name = row.get("snake_name") or row.get("name")
|
||||
is_you = bool(row.get("is_you")) or snake_id == game.get("your_snake_id")
|
||||
identities[snake_id] = SnakeIdentity(snake_id, name, is_you, row.get("customizations") or {})
|
||||
for snake_id, identity in list(identities.items()):
|
||||
if identity.name is None:
|
||||
name = next((row.get("snake_name") for row in snake_rows if row.get("snake_id") == snake_id and row.get("snake_name")), None)
|
||||
if name:
|
||||
identities[snake_id] = SnakeIdentity(snake_id, name, identity.is_you, identity.customizations)
|
||||
return list(identities.values())
|
||||
|
||||
def build_legacy_bundle(game_row:dict, identity_rows:list[dict], turn_rows:list[dict], snake_rows:list[dict]) -> GameBundle:
|
||||
game = normalize_game(game_row)
|
||||
states_by_turn:dict[int, list[SnakeState]] = {}
|
||||
for row in snake_rows:
|
||||
state = _state_from_row(row)
|
||||
states_by_turn.setdefault(state.turn, []).append(state)
|
||||
|
||||
turns = []
|
||||
states = []
|
||||
board_snakes = []
|
||||
previous:PointsState|None = None
|
||||
for row in sorted(turn_rows, key=lambda item: int(item["turn"])):
|
||||
turn = int(row["turn"])
|
||||
board = parse_json(row.get("board_state"), {}) or {}
|
||||
food = board["food"] if "food" in board else parse_json(row.get("food"), []) or []
|
||||
hazards = board["hazards"] if "hazards" in board else parse_json(row.get("hazards"), []) or []
|
||||
turn_states = states_by_turn.pop(turn, None)
|
||||
if turn_states is None:
|
||||
turn_states = _states_from_board(board, turn)
|
||||
board_snakes.extend(board.get("snakes", []))
|
||||
states.extend(turn_states)
|
||||
record, previous = compact_turn(
|
||||
turn, row.get("observed_at"), row.get("my_move"), parse_json(row.get("my_thinking")),
|
||||
food, hazards, turn_states, game.get("your_snake_id"), game.get("width"), game.get("height"),
|
||||
previous,
|
||||
)
|
||||
turns.append(record)
|
||||
# Snake rows without a turn row are kept as well.
|
||||
for orphan in states_by_turn.values():
|
||||
states.extend(orphan)
|
||||
|
||||
return GameBundle(game, _identities(game, identity_rows, snake_rows, board_snakes), turns, states)
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Copy legacy-layout games into the compact layout, verifying each game."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
import inspect
|
||||
import json
|
||||
|
||||
from server.database.compact.quality import rate_bundle
|
||||
from server.database.compact.records import GameBundle
|
||||
from server.database.game_quality import quality_meets_minimum
|
||||
|
||||
from .convert import LegacyDataError
|
||||
|
||||
@dataclass
|
||||
class MigrationStats:
|
||||
games:int = 0
|
||||
replays:int = 0
|
||||
turns:int = 0
|
||||
states:int = 0
|
||||
dropped_replays:int = 0
|
||||
failed:int = 0
|
||||
failures:list[str] = field(default_factory=list)
|
||||
|
||||
async def _resolve(value):
|
||||
return await value if inspect.isawaitable(value) else value
|
||||
|
||||
def _signature(bundle:GameBundle) -> tuple:
|
||||
turns = sorted(
|
||||
(record.turn, record.move, json.dumps(record.thinking, sort_keys=True), json.dumps(record.food), json.dumps(record.hazards))
|
||||
for record in bundle.turns
|
||||
)
|
||||
states = sorted(
|
||||
(state.snake_id, state.turn, state.health, state.latency, json.dumps(state.body))
|
||||
for state in bundle.states
|
||||
)
|
||||
return turns, states
|
||||
|
||||
def apply_quality(bundle:GameBundle, minimum_quality:str) -> bool:
|
||||
"""Rate the replay; drop its turns below the minimum. Returns whether it is kept."""
|
||||
quality = rate_bundle(bundle)
|
||||
keep = quality_meets_minimum(quality.tier, minimum_quality)
|
||||
bundle.game.update({
|
||||
"has_replay": keep,
|
||||
"quality_status": "retained" if keep else "low_quality",
|
||||
"quality_score": quality.score,
|
||||
"quality_tier": quality.tier,
|
||||
"quality_reasons": list(quality.reasons),
|
||||
})
|
||||
if not keep:
|
||||
bundle.turns = []
|
||||
bundle.states = []
|
||||
return keep
|
||||
|
||||
async def migrate_legacy(
|
||||
reader,
|
||||
backend,
|
||||
since:str|None=None,
|
||||
minimum_quality:str|None=None,
|
||||
verify:bool=True,
|
||||
progress=None,
|
||||
) -> MigrationStats:
|
||||
stats = MigrationStats()
|
||||
game_ids = await _resolve(reader.game_ids(since))
|
||||
for index, game_id in enumerate(game_ids, start=1):
|
||||
try:
|
||||
bundle = await _resolve(reader.read_bundle(game_id))
|
||||
if bundle is None:
|
||||
continue
|
||||
if minimum_quality and not apply_quality(bundle, minimum_quality):
|
||||
stats.dropped_replays += 1
|
||||
await backend.import_game_bundle(bundle)
|
||||
if verify and bundle.turns and bundle.game["has_replay"]:
|
||||
stored = await backend.read_game_bundle(game_id)
|
||||
if stored is None or _signature(stored) != _signature(bundle):
|
||||
raise LegacyDataError("stored replay differs from source")
|
||||
stats.games += 1
|
||||
stats.replays += int(bool(bundle.turns))
|
||||
stats.turns += len(bundle.turns)
|
||||
stats.states += len(bundle.states)
|
||||
except LegacyDataError as error:
|
||||
stats.failed += 1
|
||||
stats.failures.append(f"{game_id}: {error}")
|
||||
if progress is not None and (index % 500 == 0 or index == len(game_ids)):
|
||||
progress(index, len(game_ids), stats)
|
||||
return stats
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Read legacy-layout games (turns/snake_turns) from PostgreSQL."""
|
||||
|
||||
import json
|
||||
|
||||
from server.database.compact.games import GAME_COLUMNS
|
||||
from server.database.compact.records import GameBundle
|
||||
|
||||
from .convert import build_legacy_bundle
|
||||
|
||||
class LegacyPostgresqlReader:
|
||||
def __init__(self, pool):
|
||||
self._pool = pool
|
||||
|
||||
@classmethod
|
||||
async def connect(cls, dsn:str) -> "LegacyPostgresqlReader":
|
||||
import asyncpg # noqa: PLC0415
|
||||
|
||||
async def _init(conn) -> None:
|
||||
await conn.set_type_codec("jsonb", encoder=json.dumps, decoder=json.loads, schema="pg_catalog")
|
||||
await conn.set_type_codec("json", encoder=json.dumps, decoder=json.loads, schema="pg_catalog")
|
||||
|
||||
return cls(await asyncpg.create_pool(dsn=dsn, min_size=1, max_size=4, init=_init))
|
||||
|
||||
async def close(self) -> None:
|
||||
await self._pool.close()
|
||||
|
||||
async def has_legacy_tables(self) -> bool:
|
||||
async with self._pool.acquire() as conn:
|
||||
return bool(await conn.fetchval(
|
||||
"SELECT to_regclass('public.turns') IS NOT NULL AND to_regclass('public.snake_turns') IS NOT NULL"
|
||||
))
|
||||
|
||||
async def game_ids(self, since:str|None=None) -> list[str]:
|
||||
async with self._pool.acquire() as conn:
|
||||
if since:
|
||||
rows = await conn.fetch("""
|
||||
SELECT game_id FROM games WHERE started_at >= $1::timestamptz
|
||||
ORDER BY started_at, game_id
|
||||
""", since)
|
||||
else:
|
||||
rows = await conn.fetch("SELECT game_id FROM games ORDER BY started_at, game_id")
|
||||
return [row["game_id"] for row in rows]
|
||||
|
||||
async def read_bundle(self, game_id:str) -> GameBundle|None:
|
||||
async with self._pool.acquire() as conn:
|
||||
game = await conn.fetchrow(f"SELECT {', '.join(GAME_COLUMNS)} FROM games WHERE game_id = $1", game_id)
|
||||
if game is None:
|
||||
return None
|
||||
identities = await conn.fetch("""
|
||||
SELECT snake_id, snake_name, is_you, customizations
|
||||
FROM game_snakes WHERE game_id = $1
|
||||
""", game_id)
|
||||
turns = await conn.fetch("""
|
||||
SELECT turn, observed_at, my_move, my_thinking, board_state, food, hazards
|
||||
FROM turns WHERE game_id = $1
|
||||
""", game_id)
|
||||
snakes = await conn.fetch("""
|
||||
SELECT snake_id, turn, snake_name, health, length, head_x, head_y, body, is_you, latency
|
||||
FROM snake_turns WHERE game_id = $1
|
||||
""", game_id)
|
||||
return build_legacy_bundle(
|
||||
dict(game), [dict(row) for row in identities],
|
||||
[dict(row) for row in turns], [dict(row) for row in snakes],
|
||||
)
|
||||
|
||||
async def turn_counts(self) -> dict[str, int]:
|
||||
async with self._pool.acquire() as conn:
|
||||
rows = await conn.fetch("SELECT game_id, COUNT(*) AS turns FROM turns GROUP BY game_id")
|
||||
return {row["game_id"]: int(row["turns"]) for row in rows}
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Read legacy-layout games from a SQLite file (opened read-only)."""
|
||||
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
|
||||
from server.database.compact.games import GAME_COLUMNS
|
||||
from server.database.compact.records import GameBundle
|
||||
|
||||
from .convert import build_legacy_bundle, parse_json
|
||||
|
||||
_TURN_COLUMNS = {
|
||||
"turn": "turn", "observed_at": "observed_at", "my_move": "my_move",
|
||||
"my_thinking": "my_thinking_json", "board_state": "board_state_json",
|
||||
"food": "food_json", "hazards": "hazards_json",
|
||||
}
|
||||
_SNAKE_COLUMNS = {
|
||||
"snake_id": "snake_id", "turn": "turn", "snake_name": "snake_name",
|
||||
"health": "health", "length": "length", "head_x": "head_x", "head_y": "head_y",
|
||||
"body": "body_json", "is_you": "is_you", "latency": "latency",
|
||||
}
|
||||
|
||||
class LegacySqliteReader:
|
||||
def __init__(self, path:str|Path):
|
||||
self.path = Path(path)
|
||||
self._connection = sqlite3.connect(f"file:{self.path}?mode=ro", uri=True, timeout=60)
|
||||
self._connection.row_factory = sqlite3.Row
|
||||
self._connection.execute("PRAGMA query_only = ON")
|
||||
|
||||
def __enter__(self) -> "LegacySqliteReader":
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc) -> None:
|
||||
self.close()
|
||||
|
||||
def close(self) -> None:
|
||||
self._connection.close()
|
||||
|
||||
def _columns(self, table:str) -> set[str]:
|
||||
return {row[1] for row in self._connection.execute(f"PRAGMA table_info({table})")}
|
||||
|
||||
def _select(self, table:str, mapping:dict[str, str]) -> str:
|
||||
available = self._columns(table)
|
||||
return ", ".join(
|
||||
f"{source} AS {name}" if source in available else f"NULL AS {name}"
|
||||
for name, source in mapping.items()
|
||||
)
|
||||
|
||||
def has_legacy_tables(self) -> bool:
|
||||
return bool(self._columns("turns")) and bool(self._columns("snake_turns"))
|
||||
|
||||
def game_ids(self, since:str|None=None) -> list[str]:
|
||||
if since:
|
||||
rows = self._connection.execute(
|
||||
"SELECT game_id FROM games WHERE started_at >= ? ORDER BY started_at, game_id", (since,),
|
||||
)
|
||||
else:
|
||||
rows = self._connection.execute("SELECT game_id FROM games ORDER BY started_at, game_id")
|
||||
return [row[0] for row in rows]
|
||||
|
||||
def read_bundle(self, game_id:str) -> GameBundle|None:
|
||||
available = self._columns("games")
|
||||
selected = ", ".join(
|
||||
column if column in available
|
||||
else "quality_reasons_json AS quality_reasons" if column == "quality_reasons" and "quality_reasons_json" in available
|
||||
else f"NULL AS {column}"
|
||||
for column in GAME_COLUMNS
|
||||
)
|
||||
game = self._connection.execute(f"SELECT {selected} FROM games WHERE game_id = ?", (game_id,)).fetchone()
|
||||
if game is None:
|
||||
return None
|
||||
game = dict(game)
|
||||
game["quality_reasons"] = parse_json(game.get("quality_reasons"))
|
||||
|
||||
identities = []
|
||||
snake_columns = self._columns("game_snakes")
|
||||
if snake_columns:
|
||||
customizations = "customizations_json" if "customizations_json" in snake_columns else "NULL"
|
||||
identities = [
|
||||
dict(row) for row in self._connection.execute(f"""
|
||||
SELECT snake_id, snake_name, is_you, {customizations} AS customizations
|
||||
FROM game_snakes WHERE game_id = ?
|
||||
""", (game_id,))
|
||||
]
|
||||
turns = [
|
||||
dict(row) for row in self._connection.execute(
|
||||
f"SELECT {self._select('turns', _TURN_COLUMNS)} FROM turns WHERE game_id = ?", (game_id,),
|
||||
)
|
||||
]
|
||||
snakes = [
|
||||
dict(row) for row in self._connection.execute(
|
||||
f"SELECT {self._select('snake_turns', _SNAKE_COLUMNS)} FROM snake_turns WHERE game_id = ?", (game_id,),
|
||||
)
|
||||
]
|
||||
return build_legacy_bundle(game, identities, turns, snakes)
|
||||
|
||||
def turn_counts(self) -> dict[str, int]:
|
||||
return {
|
||||
row[0]: int(row[1])
|
||||
for row in self._connection.execute("SELECT game_id, COUNT(*) FROM turns GROUP BY game_id")
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
"""Normalized gameplay turn storage and replay hydration.
|
||||
|
||||
A turn is split into one board row plus one snake row per participating snake.
|
||||
Static snake identity belongs to ``game_snakes``. This avoids storing complete
|
||||
snake payloads in the board, snakes, you, and snake-turn columns simultaneously.
|
||||
"""
|
||||
|
||||
from typing import Callable
|
||||
|
||||
def compact_turn_json(board:dict) -> tuple[dict, list, dict, list, list]:
|
||||
"""Return compatibility JSON plus canonical food and hazard values."""
|
||||
return {}, [], {}, board.get("food", []), board.get("hazards", [])
|
||||
|
||||
def hydrate_replay_turns(game_row, turn_rows, snake_rows, decode_json:Callable) -> list[dict]:
|
||||
rows_by_turn:dict[int, list] = {}
|
||||
for row in snake_rows:
|
||||
rows_by_turn.setdefault(int(row["turn"]), []).append(row)
|
||||
|
||||
turns = []
|
||||
for row in turn_rows:
|
||||
turn = int(row["turn"])
|
||||
stored_board = decode_json(row["board_state_json"]) or {}
|
||||
food = decode_json(row["food_json"])
|
||||
hazards = decode_json(row["hazards_json"])
|
||||
|
||||
snakes = []
|
||||
api_snakes = []
|
||||
for snake_row in rows_by_turn.get(turn, []):
|
||||
body = decode_json(snake_row["body_json"]) or []
|
||||
customizations = decode_json(snake_row["customizations_json"]) or {}
|
||||
api_snake = {
|
||||
"id": snake_row["snake_id"],
|
||||
"name": snake_row["snake_name"],
|
||||
"health": snake_row["health"],
|
||||
"length": snake_row["length"],
|
||||
"head": {"x": snake_row["head_x"], "y": snake_row["head_y"]},
|
||||
"body": body,
|
||||
"customizations": customizations,
|
||||
}
|
||||
if snake_row["latency"] is not None:
|
||||
api_snake["latency"] = snake_row["latency"]
|
||||
api_snakes.append(api_snake)
|
||||
snakes.append({
|
||||
"snake_id": snake_row["snake_id"],
|
||||
"snake_name": snake_row["snake_name"],
|
||||
"health": snake_row["health"],
|
||||
"length": snake_row["length"],
|
||||
"head": api_snake["head"],
|
||||
"body": body,
|
||||
"customizations": customizations,
|
||||
"is_you": bool(snake_row["is_you"]),
|
||||
"inferred_move": snake_row["inferred_move"],
|
||||
"latency": snake_row["latency"],
|
||||
})
|
||||
|
||||
board = stored_board or {
|
||||
"width": game_row["width"],
|
||||
"height": game_row["height"],
|
||||
"food": food or [],
|
||||
"hazards": hazards or [],
|
||||
"snakes": api_snakes,
|
||||
}
|
||||
if food is None:
|
||||
food = board.get("food", [])
|
||||
if hazards is None:
|
||||
hazards = board.get("hazards", [])
|
||||
|
||||
you = decode_json(row["you_json"]) or {}
|
||||
if not you:
|
||||
you = next(
|
||||
(snake for snake in api_snakes if snake["id"] == game_row["your_snake_id"]),
|
||||
{},
|
||||
)
|
||||
|
||||
turns.append({
|
||||
"turn": turn,
|
||||
"observed_at": row["observed_at"],
|
||||
"my_move": row["my_move"],
|
||||
"my_thinking": decode_json(row["my_thinking_json"]),
|
||||
"board": board,
|
||||
"food": food or [],
|
||||
"hazards": hazards or [],
|
||||
"you": you,
|
||||
"snakes": snakes,
|
||||
})
|
||||
|
||||
return turns
|
||||
Reference in New Issue
Block a user