daniel156161/snake-python
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(database): store gameplay replays in compact form
Build and Push Docker Container / build-and-push (push) Successful in 3m52s
Details

Browse Source 
- Add compact storage package: codec, records, hydrate, replay blobs, cache.
- Running games write game_turns and snake_states keyed by integer ids.
- Encode moves as 1-byte codes and bodies as head plus step codes.
- Store food and hazards only when they changed since the previous turn.
- Strip reasoning fields that duplicate stored values and restore on read.
- Pack finished games into one zstd blob per game in game_replays.
- Merge late turns into packed blobs during the stale-game sweep.
- Extract SQLite and PostgreSQL schemas into dedicated modules.
- Add read/import game bundle and list_game_ids to the backend template.
- Add legacy readers and converter for the old turns/snake_turns layout.
- Add migrate_gameplay_to_compact.py with in-place or cross-DB migration.
- Verify every migrated game and keep legacy rows on failure.
- Support --since, --minimum-quality and guarded --drop-legacy.
- Add just recipe migrate-gameplay-postgres.
- Rewrite merge script to copy game by game via backend bundles.
- Move benchmark state loading into benchmark_states for both backends.
- Update PostgreSQL cleanup script to the new replay tables.
- Drop sqlite-zstd build stages from the Dockerfile.
- Remove old migration scripts, normalized_turn and their tests.
- Add tests for compact storage, schema, legacy migration and PostgreSQL.
- Document the compact layout and migration workflow in the README.
This commit is contained in:
daniel156161
2026-11-05 00:02:56 +01:00
parent 4403978537
commit 39a7a53873
41 changed files with 2779 additions and 2493 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/database/backend/PostgresqlGameplayBackend.py
+253 -488
View File
@@ -1,10 +1,8 @@
"""PostgreSQL gameplay backend using asyncpg.
JSON columns use the JSONB type so PostgreSQL stores them in a binary,
decomposed format and automatically compresses large values via TOAST
(Oversized-Attribute Storage Technique). No application-level
serialisation/deserialisation round-trip is needed for reads — asyncpg
decodes JSONB rows directly into Python dicts/lists.
Running games are stored as compact rows (``game_turns``/``snake_states``).
When a game ends it is packed into one zstd blob in ``game_replays`` and its
rows are removed. See ``server.database.compact`` for the encodings.
Connection: pass a DSN via the `dsn` constructor argument, e.g.
postgresql://user:password@host:5432/dbname
@@ -12,13 +10,22 @@ Connection: pass a DSN via the `dsn` constructor argument, e.g.
or set GAMEPLAY_DB_PG_DSN in the environment.
"""
import asyncio, logging, sqlite3, json, sys
import json, logging, sys
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import urlparse, urlunparse
from .Template import GameplayBackendTemplate
from server.database.normalized_turn import compact_turn_json
from .postgresql_schema import DROP_REDUNDANT_INDEXES, TABLES_DDL
from server.database.compact.cache import RunningGame, RunningGameCache
from server.database.compact.codec import (
decode_body, decode_move, decode_points, encode_body, encode_move, encode_points,
)
from server.database.compact.games import GAME_COLUMNS, normalize_game
from server.database.compact.records import (
GameBundle, SnakeIdentity, SnakeState, TurnRecord, merge_records, records_from_game_state,
)
from server.database.compact.replay_blob import REPLAY_FORMAT, pack_replay, unpack_replay
logger = logging.getLogger(__name__)
if not logger.handlers:
@@ -27,106 +34,46 @@ if not logger.handlers:
logger.addHandler(_handler)
logger.propagate = False
# DDL --------------------------------------------------------------------- #
_SCHEMA_LOCK_ID = 7_315_002
_DDL = """
CREATE TABLE IF NOT EXISTS games (
game_id TEXT PRIMARY KEY,
started_at TIMESTAMPTZ NOT NULL,
ended_at TIMESTAMPTZ,
width INTEGER,
height INTEGER,
source TEXT,
map_name TEXT,
ruleset_name TEXT,
ruleset_version TEXT,
your_snake_id TEXT,
your_snake_name TEXT,
your_snake_type TEXT,
your_snake_version TEXT,
game_type TEXT,
winner_name TEXT,
winner_you BOOLEAN NOT NULL DEFAULT FALSE,
final_turn INTEGER NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'running',
has_replay BOOLEAN NOT NULL DEFAULT TRUE,
quality_status TEXT NOT NULL DEFAULT 'retained',
quality_score INTEGER,
quality_tier TEXT,
quality_reasons JSONB
);
CREATE TABLE IF NOT EXISTS turns (
id BIGSERIAL PRIMARY KEY,
game_id TEXT NOT NULL REFERENCES games(game_id) ON DELETE CASCADE,
turn INTEGER NOT NULL,
observed_at TIMESTAMPTZ NOT NULL,
my_move TEXT,
my_thinking JSONB,
board_state JSONB NOT NULL,
snakes JSONB NOT NULL,
you JSONB NOT NULL,
food JSONB NOT NULL,
hazards JSONB NOT NULL,
UNIQUE (game_id, turn)
);
CREATE TABLE IF NOT EXISTS game_snakes (
game_id TEXT NOT NULL REFERENCES games(game_id) ON DELETE CASCADE,
snake_id TEXT NOT NULL,
snake_name TEXT,
is_you BOOLEAN NOT NULL DEFAULT FALSE,
customizations JSONB NOT NULL DEFAULT '{}'::jsonb,
PRIMARY KEY (game_id, snake_id)
);
CREATE TABLE IF NOT EXISTS snake_turns (
id BIGSERIAL PRIMARY KEY,
game_id TEXT NOT NULL REFERENCES games(game_id) ON DELETE CASCADE,
turn INTEGER NOT NULL,
snake_id TEXT NOT NULL,
snake_name TEXT,
health INTEGER,
length INTEGER,
head_x INTEGER,
head_y INTEGER,
body JSONB NOT NULL,
is_you BOOLEAN NOT NULL DEFAULT FALSE,
inferred_move TEXT,
latency TEXT,
UNIQUE (game_id, turn, snake_id)
);
CREATE INDEX IF NOT EXISTS idx_turns_game_turn ON turns(game_id, turn);
CREATE INDEX IF NOT EXISTS idx_games_status ON games(status);
CREATE INDEX IF NOT EXISTS idx_snake_turns_game_turn ON snake_turns(game_id, turn);
_GAME_SELECT = """
SELECT gid, game_id, started_at, ended_at, width, height, source, map_name,
ruleset_name, ruleset_version, game_type, your_snake_id, your_snake_name,
your_snake_type, your_snake_version, winner_name, winner_you, final_turn,
status, has_replay, quality_status, quality_score, quality_tier, quality_reasons
FROM games
"""
# Schema evolution: add new columns to existing tables (idempotent).
_ALTER_DDL = """
ALTER TABLE games ADD COLUMN IF NOT EXISTS game_type TEXT;
ALTER TABLE games ADD COLUMN IF NOT EXISTS your_snake_type TEXT;
ALTER TABLE games ADD COLUMN IF NOT EXISTS your_snake_version TEXT;
ALTER TABLE games ADD COLUMN IF NOT EXISTS winner_name TEXT;
ALTER TABLE games ADD COLUMN IF NOT EXISTS has_replay BOOLEAN NOT NULL DEFAULT TRUE;
ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_status TEXT NOT NULL DEFAULT 'retained';
ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_score INTEGER;
ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_tier TEXT;
ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_reasons JSONB;
ALTER TABLE turns ADD COLUMN IF NOT EXISTS my_thinking JSONB;
ALTER TABLE snake_turns ADD COLUMN IF NOT EXISTS latency TEXT;
ALTER TABLE game_snakes ADD COLUMN IF NOT EXISTS customizations JSONB NOT NULL DEFAULT '{}'::jsonb;
_UPSERT_SNAKES = """
INSERT INTO game_snakes (game_id, snake_id, snake_name, is_you, customizations)
SELECT $1, s.snake_id, s.snake_name, s.is_you, s.customizations::jsonb
FROM unnest($2::text[], $3::text[], $4::bool[], $5::text[])
AS s(snake_id, snake_name, is_you, customizations)
ON CONFLICT (game_id, snake_id) DO UPDATE SET
snake_name = EXCLUDED.snake_name,
is_you = EXCLUDED.is_you,
customizations = EXCLUDED.customizations
RETURNING snake_id, gsid
"""
# Force TOAST compression on the large JSONB columns so that even
# moderately-sized payloads get compressed on-disk.
_TOAST_DDL = """
ALTER TABLE turns ALTER COLUMN board_state SET STORAGE EXTENDED;
ALTER TABLE turns ALTER COLUMN snakes SET STORAGE EXTENDED;
ALTER TABLE turns ALTER COLUMN you SET STORAGE EXTENDED;
ALTER TABLE turns ALTER COLUMN food SET STORAGE EXTENDED;
ALTER TABLE turns ALTER COLUMN hazards SET STORAGE EXTENDED;
ALTER TABLE snake_turns ALTER COLUMN body SET STORAGE EXTENDED;
_UPSERT_TURN = """
INSERT INTO game_turns (gid, turn, observed_at, my_move, thinking, food, hazards)
VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (gid, turn) DO UPDATE SET
observed_at = EXCLUDED.observed_at,
my_move = EXCLUDED.my_move,
thinking = EXCLUDED.thinking,
food = EXCLUDED.food,
hazards = EXCLUDED.hazards
"""
_UPSERT_STATES = """
INSERT INTO snake_states (gsid, turn, health, latency, body)
SELECT * FROM unnest($1::int[], $2::int[], $3::smallint[], $4::smallint[], $5::bytea[])
ON CONFLICT (gsid, turn) DO UPDATE SET
health = EXCLUDED.health,
latency = EXCLUDED.latency,
body = EXCLUDED.body
"""
class PostgresqlGameplayBackend(GameplayBackendTemplate):
@@ -142,6 +89,7 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
self._max_size = max_size
self._sqlite_migration_path = sqlite_migration_path
self._pool = None # asyncpg.Pool, typed at runtime
self._running = RunningGameCache()
# ── DSN normalisation ──────────────────────────────────────────────────────
@@ -149,25 +97,16 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
@classmethod
def _ensure_db_name(cls, dsn:str) -> str:
"""Return *dsn* with a database name appended when none is present.
A DSN has no database name when its path component is empty or ``/``.
In that case ``battlesnake`` is appended so asyncpg gets a complete
connection string without the caller having to remember to add one.
"""
"""Return *dsn* with a database name appended when none is present."""
parsed = urlparse(dsn)
db = parsed.path.lstrip("/")
if db:
if parsed.path.lstrip("/"):
return dsn
new_path = f"/{cls._DEFAULT_DB_NAME}"
return urlunparse(parsed._replace(path=new_path))
return urlunparse(parsed._replace(path=f"/{cls._DEFAULT_DB_NAME}"))
# ── pool / schema ──────────────────────────────────────────────────────────
async def initialize(self) -> None:
"""Eagerly create the connection pool on startup so schema init and
SQLite migration run immediately rather than on the first game request."""
"""Eagerly create the pool so schema init and legacy SQLite import run on startup."""
await self._get_pool()
async def _get_pool(self):
@@ -194,27 +133,22 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
init=_init_conn,
)
await self._initialize_schema()
await self._maybe_migrate_from_sqlite()
await self._maybe_import_legacy_sqlite()
return self._pool
async def _ensure_database_exists(self, asyncpg, target_dsn:str) -> None:
"""Connect to the postgres maintenance DB and CREATE the target database
if it does not already exist. Uses a plain connection (not a pool) so
the CREATE DATABASE statement can run outside any transaction."""
"""CREATE the target database through the maintenance DB when missing."""
parsed = urlparse(target_dsn)
db_name = parsed.path.lstrip("/")
maintenance_dsn = urlunparse(parsed._replace(path="/postgres"))
try:
conn = await asyncpg.connect(dsn=maintenance_dsn)
except Exception:
# Fall back to connecting without specifying a database — some setups
# (e.g. Cloud SQL, managed PG) disallow direct access to 'postgres'.
# Some managed setups disallow direct access to 'postgres'.
maintenance_dsn = urlunparse(parsed._replace(path=""))
conn = await asyncpg.connect(dsn=maintenance_dsn)
try:
exists = await conn.fetchval(
"SELECT 1 FROM pg_database WHERE datname = $1", db_name
)
exists = await conn.fetchval("SELECT 1 FROM pg_database WHERE datname = $1", db_name)
if not exists:
await conn.execute(f'CREATE DATABASE "{db_name}"')
logger.info(f"PostgreSQL: created database '{db_name}'")
@@ -224,199 +158,143 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
async def _initialize_schema(self) -> None:
assert self._pool is not None
async with self._pool.acquire() as conn:
await conn.execute(_DDL)
await conn.execute(_ALTER_DDL)
# TOAST storage hints are idempotent; ignore errors on repeated runs.
# Several workers may start at once; serialize DDL.
await conn.execute("SELECT pg_advisory_lock($1)", _SCHEMA_LOCK_ID)
try:
await conn.execute(_TOAST_DDL)
except Exception as exc:
logger.debug(f"TOAST DDL skipped (likely already set): {exc}")
await conn.execute(TABLES_DDL)
finally:
await conn.execute("SELECT pg_advisory_unlock($1)", _SCHEMA_LOCK_ID)
for statement in DROP_REDUNDANT_INDEXES:
await conn.execute(statement)
# ── sqlite migration ───────────────────────────────────────────────────────
async def _maybe_migrate_from_sqlite(self) -> None:
async def _maybe_import_legacy_sqlite(self) -> None:
if not self._sqlite_migration_path:
return
src = Path(self._sqlite_migration_path)
if not src.exists():
source = Path(self._sqlite_migration_path)
if not source.exists():
return
logger.info(f"SQLite migration: found {src}, starting migration to PostgreSQL …")
from server.database.legacy.migrate import migrate_legacy # noqa: PLC0415
from server.database.legacy.reader_sqlite import LegacySqliteReader # noqa: PLC0415
logger.info(f"Legacy SQLite import: found {source}, importing into PostgreSQL …")
try:
games, turns, snake_turns = await asyncio.to_thread(self._read_sqlite_data_sync, str(src))
await self._insert_migrated_data(games, turns, snake_turns)
done_path = src.with_suffix(".migrated")
src.rename(done_path)
logger.info(
f"SQLite migration complete: {len(games)} games, {len(turns)} turns, "
f"{len(snake_turns)} snake_turns migrated. "
f"Source file renamed to {done_path.name}"
)
with LegacySqliteReader(source) as reader:
stats = await migrate_legacy(reader, self)
if stats.failed:
logger.warning(f"Legacy SQLite import incomplete ({stats.failed} games failed); source kept")
return
done_path = source.with_suffix(".migrated")
source.rename(done_path)
logger.info(f"Legacy SQLite import complete: {stats.games} games. Source renamed to {done_path.name}")
except Exception:
logger.exception("SQLite migration failed — PostgreSQL data is untouched, original SQLite file kept")
def _read_sqlite_data_sync(self, db_path:str) -> tuple[list[sqlite3.Row], list[sqlite3.Row], list[sqlite3.Row]]:
conn = sqlite3.connect(db_path, timeout=30, isolation_level=None)
conn.row_factory = sqlite3.Row
try:
game_columns = {
row["name"] for row in conn.execute("PRAGMA table_info(games)").fetchall()
}
if "winner_name" in game_columns:
winner_name_expression = "winner_name"
elif "winner_names_json" in game_columns:
winner_name_expression = "winner_names_json"
else:
winner_name_expression = "NULL"
games = conn.execute(f"""
SELECT game_id, started_at, ended_at, width, height, source, map_name,
ruleset_name, ruleset_version, your_snake_id, your_snake_name,
your_snake_type, your_snake_version, game_type,
{winner_name_expression} AS winner_name, winner_you, final_turn, status
FROM games
ORDER BY started_at ASC
""").fetchall()
turns = conn.execute("""
SELECT game_id, turn, observed_at, my_move, my_thinking_json,
board_state_json, snakes_json, you_json, food_json, hazards_json
FROM turns
ORDER BY game_id ASC, turn ASC
""").fetchall()
snake_turns = conn.execute("""
SELECT game_id, turn, snake_id, snake_name, health, length,
head_x, head_y, body_json, is_you, inferred_move, latency
FROM snake_turns
ORDER BY game_id ASC, turn ASC, snake_id ASC
""").fetchall()
finally:
conn.close()
return games, turns, snake_turns
def _parse_ts(self, value:str|None) -> datetime|None:
"""Parse an ISO-8601 TEXT timestamp from SQLite into a timezone-aware datetime."""
ts = self._parse_utc_timestamp(value)
return ts # already UTC-aware from base class helper
def _parse_json(self, value: str|None) -> object:
if not value:
return None
try:
return json.loads(value)
except (json.JSONDecodeError, TypeError):
return None
def _migrated_winner_name(self, value:str|None) -> str|None:
"""Accept both the current winner_name and legacy winner_names_json value."""
if not value:
return None
parsed = self._parse_json(value)
if isinstance(parsed, list):
return next((str(name) for name in parsed if name), None)
return value
async def _insert_migrated_data(self, games:list, turns:list, snake_turns:list) -> None:
assert self._pool is not None
async with self._pool.acquire() as conn:
async with conn.transaction():
# games ─────────────────────────────────────────────────────────────
# winner_name is TEXT — no cast needed.
await conn.executemany("""
INSERT INTO games (
game_id, started_at, ended_at, width, height, source, map_name,
ruleset_name, ruleset_version, your_snake_id, your_snake_name,
your_snake_type, your_snake_version, game_type,
winner_name, winner_you, final_turn, status
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18)
ON CONFLICT (game_id) DO NOTHING
""",
[
(
row["game_id"],
self._parse_ts(row["started_at"]),
self._parse_ts(row["ended_at"]),
row["width"],
row["height"],
row["source"],
row["map_name"],
row["ruleset_name"],
row["ruleset_version"],
row["your_snake_id"],
row["your_snake_name"],
row["your_snake_type"],
row["your_snake_version"],
row["game_type"],
self._migrated_winner_name(row["winner_name"]),
bool(row["winner_you"]),
row["final_turn"],
row["status"],
)
for row in games
],
)
await conn.executemany("""
INSERT INTO turns (
game_id, turn, observed_at, my_move, my_thinking,
board_state, snakes, you, food, hazards
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
ON CONFLICT (game_id, turn) DO NOTHING
""",
[
(
row["game_id"],
row["turn"],
self._parse_ts(row["observed_at"]),
row["my_move"],
self._parse_json(row["my_thinking_json"]),
self._parse_json(row["board_state_json"]),
self._parse_json(row["snakes_json"]),
self._parse_json(row["you_json"]),
self._parse_json(row["food_json"]),
self._parse_json(row["hazards_json"]),
)
for row in turns
],
)
# snake_turns
await conn.executemany("""
INSERT INTO snake_turns (
game_id, turn, snake_id, snake_name, health, length,
head_x, head_y, body, is_you, inferred_move, latency
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)
ON CONFLICT (game_id, turn, snake_id) DO NOTHING
""",
[
(
row["game_id"],
row["turn"],
row["snake_id"],
row["snake_name"],
row["health"],
row["length"],
row["head_x"],
row["head_y"],
self._parse_json(row["body_json"]),
bool(row["is_you"]),
row["inferred_move"],
row["latency"],
)
for row in snake_turns
],
)
logger.exception("Legacy SQLite import failed — original SQLite file kept")
# ── helpers ────────────────────────────────────────────────────────────────
def _utc_now_ts(self) -> datetime:
return datetime.now(timezone.utc)
# The pool init callback registers JSON/JSONB codecs so asyncpg automatically
# encodes Python dicts/lists on write and decodes them on read.
async def _running_game(self, conn, game_id:str) -> RunningGame:
game = self._running.get(game_id)
if game is None:
gid = await conn.fetchval("SELECT gid FROM games WHERE game_id = $1", game_id)
if gid is None:
raise LookupError(f"Turn for unknown game {game_id!r}")
game = RunningGame(gid=gid)
return game
async def _upsert_identities(self, conn, game_id:str, identities:list[SnakeIdentity]) -> dict[str, int]:
if not identities:
return {}
rows = await conn.fetch(
_UPSERT_SNAKES, game_id,
[identity.snake_id for identity in identities],
[identity.name for identity in identities],
[identity.is_you for identity in identities],
[json.dumps(identity.customizations or {}) for identity in identities],
)
return {row["snake_id"]: row["gsid"] for row in rows}
async def _write_rows(self, conn, gid:int, gsids:dict[str, int], turns:list[TurnRecord], states:list[SnakeState]) -> None:
if turns:
await conn.executemany(_UPSERT_TURN, [
(
gid, record.turn, record.observed_at or self._utc_now_ts(), encode_move(record.move),
record.thinking, encode_points(record.food), encode_points(record.hazards),
)
for record in turns
])
states = [state for state in states if state.snake_id in gsids]
if states:
await conn.execute(
_UPSERT_STATES,
[gsids[state.snake_id] for state in states],
[state.turn for state in states],
[state.health for state in states],
[state.latency for state in states],
[encode_body(state.body) for state in states],
)
async def _load_records(self, conn, game_id:str, gid:int) -> tuple[list[SnakeIdentity], list[TurnRecord], list[SnakeState]]:
identity_rows = await conn.fetch("""
SELECT snake_id, snake_name, is_you, customizations
FROM game_snakes WHERE game_id = $1
""", game_id)
identities = [
SnakeIdentity(row["snake_id"], row["snake_name"], bool(row["is_you"]), row["customizations"] or {})
for row in identity_rows
]
blob = await conn.fetchrow("SELECT format, data FROM game_replays WHERE gid = $1", gid)
turn_rows = await conn.fetch("""
SELECT turn, observed_at, my_move, thinking, food, hazards
FROM game_turns WHERE gid = $1 ORDER BY turn
""", gid)
state_rows = await conn.fetch("""
SELECT gs.snake_id, s.turn, s.health, s.latency, s.body
FROM game_snakes AS gs
JOIN snake_states AS s ON s.gsid = gs.gsid
WHERE gs.game_id = $1
""", game_id)
turns = [
TurnRecord(
turn=row["turn"], observed_at=row["observed_at"], move=decode_move(row["my_move"]),
thinking=row["thinking"], food=decode_points(row["food"]), hazards=decode_points(row["hazards"]),
)
for row in turn_rows
]
states = [
SnakeState(row["snake_id"], row["turn"], row["health"], row["latency"], decode_body(row["body"]))
for row in state_rows
]
if blob is not None:
packed_turns, packed_states = unpack_replay(blob["data"], blob["format"])
turns, states = merge_records(packed_turns, packed_states, turns, states)
return identities, turns, states
async def _store_replay(self, conn, game_id:str, gid:int, turns:list[TurnRecord], states:list[SnakeState]) -> None:
await conn.execute("""
INSERT INTO game_replays (gid, format, turn_count, data) VALUES ($1, $2, $3, $4)
ON CONFLICT (gid) DO UPDATE SET
format = EXCLUDED.format, turn_count = EXCLUDED.turn_count, data = EXCLUDED.data
""", gid, REPLAY_FORMAT, len(turns), pack_replay(turns, states))
await conn.execute("DELETE FROM game_turns WHERE gid = $1", gid)
await conn.execute("""
DELETE FROM snake_states
WHERE gsid IN (SELECT gsid FROM game_snakes WHERE game_id = $1)
""", game_id)
async def _pack_game(self, conn, game_id:str) -> None:
"""Move a game's rows into its replay blob (idempotent)."""
async with conn.transaction():
gid = await conn.fetchval("SELECT gid FROM games WHERE game_id = $1 FOR UPDATE", game_id)
if gid is None:
return
has_rows = await conn.fetchval("SELECT EXISTS (SELECT 1 FROM game_turns WHERE gid = $1)", gid)
if not has_rows:
return
_, turns, states = await self._load_records(conn, game_id, gid)
await self._store_replay(conn, game_id, gid, turns, states)
# ── write methods ──────────────────────────────────────────────────────────
@@ -465,111 +343,24 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
)
async def record_turn(self, game_state:dict, my_move:str|None, my_thinking:dict|None=None) -> None:
game = game_state.get("game", {})
board = game_state.get("board", {})
snakes = self._extract_snakes(game_state)
you = self._extract_you(game_state)
game_id = game.get("id")
turn = int(game_state.get("turn", 0))
board_json, snakes_json, you_json, food_json, hazards_json = compact_turn_json(board)
game_id = game_state.get("game", {}).get("id")
pool = await self._get_pool()
async with pool.acquire() as conn:
async with conn.transaction():
await conn.executemany("""
INSERT INTO game_snakes (
game_id, snake_id, snake_name, is_you, customizations
) VALUES ($1,$2,$3,$4,$5)
ON CONFLICT (game_id, snake_id) DO UPDATE SET
snake_name = EXCLUDED.snake_name,
is_you = EXCLUDED.is_you,
customizations = EXCLUDED.customizations
""",
[
(
game_id,
snake.get("id"),
snake.get("name"),
snake.get("id") == you.get("id"),
snake.get("customizations") or {},
)
for snake in snakes if snake.get("id") is not None
],
running = await self._running_game(conn, game_id)
record, states, identities, points = records_from_game_state(
game_state, my_move, my_thinking, running.points,
)
await conn.execute("""
INSERT INTO turns (
game_id, turn, observed_at, my_move, my_thinking,
board_state, snakes, you, food, hazards
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
ON CONFLICT (game_id, turn) DO UPDATE SET
observed_at = EXCLUDED.observed_at,
my_move = EXCLUDED.my_move,
my_thinking = EXCLUDED.my_thinking,
board_state = EXCLUDED.board_state,
snakes = EXCLUDED.snakes,
you = EXCLUDED.you,
food = EXCLUDED.food,
hazards = EXCLUDED.hazards
""",
game_id,
turn,
self._utc_now_ts(),
my_move,
my_thinking,
board_json,
snakes_json,
you_json,
food_json,
hazards_json,
)
previous_positions:dict[str, tuple[int, int]] = {}
if turn > 0:
prev_rows = await conn.fetch("""
SELECT snake_id, head_x, head_y
FROM snake_turns
WHERE game_id = $1 AND turn = $2
""",
game_id, turn - 1,
)
previous_positions = {
row["snake_id"]: (int(row["head_x"]), int(row["head_y"]))
for row in prev_rows
if row["head_x"] is not None and row["head_y"] is not None
}
you_id = you.get("id")
for params in self._build_snake_turn_params(snakes, you_id, game_id, turn, previous_positions):
(p_game_id, p_turn, p_snake_id, p_name, p_health, p_length,
p_head_x, p_head_y, p_body, p_is_you, p_inferred, p_latency) = params
await conn.execute("""
INSERT INTO snake_turns (
game_id, turn, snake_id, snake_name, health, length,
head_x, head_y, body, is_you, inferred_move, latency
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)
ON CONFLICT (game_id, turn, snake_id) DO UPDATE SET
snake_name = EXCLUDED.snake_name,
health = EXCLUDED.health,
length = EXCLUDED.length,
head_x = EXCLUDED.head_x,
head_y = EXCLUDED.head_y,
body = EXCLUDED.body,
is_you = EXCLUDED.is_you,
inferred_move = EXCLUDED.inferred_move,
latency = EXCLUDED.latency
""",
p_game_id, p_turn, p_snake_id, None, p_health, p_length,
p_head_x, p_head_y, p_body, False, p_inferred, p_latency,
)
await conn.execute("""
UPDATE games
SET final_turn = GREATEST(final_turn, $1)
WHERE game_id = $2
""",
turn, game_id,
missing = [identity for identity in identities if identity.snake_id not in running.snake_ids]
if missing:
running.snake_ids.update(await self._upsert_identities(conn, game_id, identities))
await self._write_rows(conn, running.gid, running.snake_ids, [record], states)
await conn.execute(
"UPDATE games SET final_turn = GREATEST(final_turn, $1) WHERE gid = $2",
record.turn, running.gid,
)
running.points = points
self._running.put(game_id, running)
async def record_game_end(self, game_state:dict) -> None:
game_id, winner_name, winner_you, turn = self._extract_game_end_params(game_state)
@@ -585,12 +376,10 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
status = 'finished'
WHERE game_id = $5
""",
self._utc_now_ts(),
winner_name,
winner_you,
turn,
game_id,
self._utc_now_ts(), winner_name, winner_you, turn, game_id,
)
await self._pack_game(conn, game_id)
self._running.drop(game_id)
# ── stale game finalization ────────────────────────────────────────────────
@@ -602,7 +391,7 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
pool = await self._get_pool()
async with pool.acquire() as conn:
rows = await conn.fetch("""
SELECT game_id, started_at, final_turn, your_snake_id
SELECT gid, game_id, started_at, final_turn, your_snake_id
FROM games
WHERE status = 'running'
ORDER BY started_at ASC
@@ -618,40 +407,9 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
continue
game_id = row["game_id"]
your_snake_id = row["your_snake_id"]
final_turn = int(row["final_turn"] or 0)
snake_rows = await conn.fetch("""
SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name) AS snake_name
FROM snake_turns AS st
LEFT JOIN game_snakes AS gs
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
WHERE st.game_id = $1 AND st.turn = $2
ORDER BY COALESCE(gs.is_you, st.is_you) DESC, snake_name ASC
""",
game_id, final_turn,
)
if len(snake_rows) == 0:
latest_row = await conn.fetchrow(
"SELECT MAX(turn) AS latest_turn FROM snake_turns WHERE game_id = $1",
game_id,
)
if latest_row is not None and latest_row["latest_turn"] is not None:
final_turn = int(latest_row["latest_turn"])
snake_rows = await conn.fetch("""
SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name) AS snake_name
FROM snake_turns AS st
LEFT JOIN game_snakes AS gs
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
WHERE st.game_id = $1 AND st.turn = $2
ORDER BY COALESCE(gs.is_you, st.is_you) DESC, snake_name ASC
""",
game_id, final_turn,
)
winner_you, survivor_name = self._calculate_survivor(snake_rows, your_snake_id)
identities, _, states = await self._load_records(conn, game_id, row["gid"])
final_turn, survivors = self._survivors(identities, states, int(row["final_turn"] or 0))
winner_you, survivor_name = self._calculate_survivor(survivors, row["your_snake_id"])
tag = await conn.execute("""
UPDATE games
@@ -662,14 +420,21 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
status = 'finished'
WHERE game_id = $5 AND status = 'running'
""",
self._utc_now_ts(),
survivor_name,
winner_you,
final_turn,
game_id,
self._utc_now_ts(), survivor_name, winner_you, final_turn, game_id,
)
if tag and tag.endswith("1"):
finalized += 1
await self._pack_game(conn, game_id)
self._running.drop(game_id)
# Turns that arrived after a game was packed are folded into its blob.
late = await conn.fetch("""
SELECT DISTINCT g.game_id
FROM game_turns AS t JOIN games AS g ON g.gid = t.gid
WHERE g.status = 'finished'
""")
for late_row in late:
await self._pack_game(conn, late_row["game_id"])
return finalized
@@ -734,51 +499,51 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate):
async def get_game_replay(self, game_id:str) -> dict|None:
pool = await self._get_pool()
async with pool.acquire() as conn:
game_row = await conn.fetchrow("""
SELECT game_id, started_at, ended_at, width, height, source, map_name,
ruleset_name, ruleset_version, game_type, your_snake_id, your_snake_name,
your_snake_type, your_snake_version,
winner_name, winner_you, final_turn, status
FROM games
WHERE game_id = $1 AND has_replay
""",
game_id,
)
game_row = await conn.fetchrow(_GAME_SELECT + " WHERE game_id = $1 AND has_replay", game_id)
if game_row is None:
return None
identities, turns, states = await self._load_records(conn, game_id, game_row["gid"])
turn_rows = await conn.fetch("""
SELECT turn, observed_at, my_move,
my_thinking AS my_thinking_json,
board_state AS board_state_json,
food AS food_json,
hazards AS hazards_json,
you AS you_json
FROM turns
WHERE game_id = $1
ORDER BY turn ASC
""",
game_id,
)
return self._build_game_replay_output(game_row, identities, turns, states)
snake_rows = await conn.fetch("""
SELECT st.turn, st.snake_id,
COALESCE(gs.snake_name, st.snake_name) AS snake_name,
st.health, st.length, st.head_x, st.head_y, st.body AS body_json,
COALESCE(gs.is_you, st.is_you) AS is_you,
COALESCE(gs.customizations, '{}'::jsonb) AS customizations_json,
st.inferred_move, st.latency
FROM snake_turns AS st
LEFT JOIN game_snakes AS gs
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
WHERE st.game_id = $1
ORDER BY st.turn ASC, is_you DESC, snake_name ASC
""",
game_id,
)
# ── bundles (migration / merge) ────────────────────────────────────────────
return self._build_game_replay_output(game_row, turn_rows, snake_rows, decode_json=lambda x: x)
async def list_game_ids(self) -> list[str]:
pool = await self._get_pool()
async with pool.acquire() as conn:
rows = await conn.fetch("SELECT game_id FROM games ORDER BY started_at, game_id")
return [row["game_id"] for row in rows]
async def read_game_bundle(self, game_id:str) -> GameBundle|None:
pool = await self._get_pool()
async with pool.acquire() as conn:
game_row = await conn.fetchrow(_GAME_SELECT + " WHERE game_id = $1", game_id)
if game_row is None:
return None
identities, turns, states = await self._load_records(conn, game_id, game_row["gid"])
return GameBundle(normalize_game(dict(game_row)), identities, turns, states)
async def import_game_bundle(self, bundle:GameBundle) -> None:
game = normalize_game(bundle.game)
columns = ", ".join(GAME_COLUMNS)
placeholders = ", ".join(f"${index}" for index in range(1, len(GAME_COLUMNS) + 1))
updates = ", ".join(f"{column} = EXCLUDED.{column}" for column in GAME_COLUMNS[1:])
pool = await self._get_pool()
async with pool.acquire() as conn:
async with conn.transaction():
gid = await conn.fetchval(
f"INSERT INTO games ({columns}) VALUES ({placeholders}) "
f"ON CONFLICT (game_id) DO UPDATE SET {updates} RETURNING gid",
*[game[column] for column in GAME_COLUMNS],
)
gsids = await self._upsert_identities(conn, game["game_id"], bundle.snakes)
if not bundle.turns or not game["has_replay"]:
return
if game["status"] == "finished":
await self._store_replay(conn, game["game_id"], gid, bundle.turns, bundle.states)
else:
await self._write_rows(conn, gid, gsids, bundle.turns, bundle.states)
self._running.drop(game["game_id"])
# ── lifecycle ──────────────────────────────────────────────────────────────
server/database/backend/SqliteGameplayBackend.py
+304 -384
View File
@@ -1,11 +1,24 @@
from quart_common.web.env import env_bool
"""SQLite gameplay backend with the same compact layout as PostgreSQL.
import asyncio, sqlite3, json, os, logging, sys
Running games use ``game_turns``/``snake_states``; finished games are packed
into one zstd blob per game in ``game_replays``.
"""
import asyncio, json, sqlite3, logging, sys
from datetime import datetime, timezone
from pathlib import Path
from server.database.backend.Template import GameplayBackendTemplate
from server.database.normalized_turn import compact_turn_json
from server.database.backend.sqlite_schema import create_schema
from server.database.compact.cache import RunningGame, RunningGameCache
from server.database.compact.codec import (
decode_body, decode_move, decode_points, encode_body, encode_move, encode_points,
)
from server.database.compact.games import GAME_COLUMNS, normalize_game
from server.database.compact.records import (
GameBundle, SnakeIdentity, SnakeState, TurnRecord, merge_records, records_from_game_state,
)
from server.database.compact.replay_blob import REPLAY_FORMAT, pack_replay, unpack_replay
logger = logging.getLogger(__name__)
if not logger.handlers:
@@ -14,37 +27,45 @@ if not logger.handlers:
logger.addHandler(_handler)
logger.propagate = False
_ZSTD_EXT = Path(os.environ.get("SQLITE_ZSTD_EXT", "/usr/local/lib/libsqlite_zstd.so")).expanduser().resolve()
_GAME_SELECT = """
SELECT gid, game_id, started_at, ended_at, width, height, source, map_name,
ruleset_name, ruleset_version, game_type, your_snake_id, your_snake_name,
your_snake_type, your_snake_version, winner_name, winner_you, final_turn,
status, has_replay, quality_status, quality_score, quality_tier, quality_reasons_json
FROM games
"""
def _game_from_row(row:sqlite3.Row) -> dict:
data = dict(row)
reasons = data.pop("quality_reasons_json", None)
data["quality_reasons"] = json.loads(reasons) if reasons else None
return normalize_game(data)
class SqliteGameplayBackend(GameplayBackendTemplate):
def __init__(self, db_path:str, busy_timeout_ms:int=5000, initialize_indexes:bool=True, journal_mode:str="WAL"):
def __init__(self, db_path:str, busy_timeout_ms:int=5000, journal_mode:str="WAL", read_only:bool=False):
self.db_path = db_path
self.busy_timeout_ms = max(1000, int(busy_timeout_ms))
self._zstd_available = False
self._initialize_indexes = initialize_indexes
self._journal_mode = journal_mode
self._initialize_database()
self._read_only = read_only
self._running = RunningGameCache()
if not read_only:
self._initialize_database()
# ── connection ─────────────────────────────────────────────────────────────
def _connect(self) -> sqlite3.Connection:
if self._read_only:
connection = sqlite3.connect(f"file:{self.db_path}?mode=ro", uri=True, isolation_level=None)
connection.row_factory = sqlite3.Row
connection.execute("PRAGMA query_only = ON")
return connection
connection = sqlite3.connect(
self.db_path,
timeout=max(1, self.busy_timeout_ms // 1000),
isolation_level=None,
)
connection.row_factory = sqlite3.Row
if _ZSTD_EXT.exists() and not env_bool('DISABLE_GAMEPLAY_DB_COMPRESSION', True):
try:
connection.enable_load_extension(True)
connection.load_extension(str(_ZSTD_EXT))
self._zstd_available = True
except sqlite3.OperationalError as e:
logger.warning(f"sqlite-zstd extension skipped: {e}")
finally:
connection.enable_load_extension(False)
connection.execute("PRAGMA foreign_keys = ON")
connection.execute(f"PRAGMA journal_mode = {self._journal_mode}")
connection.execute("PRAGMA synchronous = NORMAL")
@@ -53,159 +74,142 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
connection.execute(f"PRAGMA busy_timeout = {self.busy_timeout_ms}")
return connection
def _ensure_auto_vacuum_full(self, connection:sqlite3.Connection) -> None:
current = connection.execute("PRAGMA auto_vacuum").fetchone()[0]
if current != 1:
connection.execute("PRAGMA auto_vacuum = FULL")
connection.execute("VACUUM")
# ── schema setup ───────────────────────────────────────────────────────────
def _initialize_database(self) -> None:
Path(self.db_path).parent.mkdir(parents=True, exist_ok=True)
with self._connect() as connection:
self._ensure_auto_vacuum_full(connection)
connection.executescript("""
CREATE TABLE IF NOT EXISTS games (
game_id TEXT PRIMARY KEY,
started_at TEXT NOT NULL,
ended_at TEXT,
width INTEGER,
height INTEGER,
source TEXT,
map_name TEXT,
ruleset_name TEXT,
ruleset_version TEXT,
your_snake_id TEXT,
your_snake_name TEXT,
your_snake_type TEXT,
your_snake_version TEXT,
winner_name TEXT,
winner_you INTEGER NOT NULL DEFAULT 0,
final_turn INTEGER NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'running',
has_replay INTEGER NOT NULL DEFAULT 1,
quality_status TEXT NOT NULL DEFAULT 'retained',
quality_score INTEGER,
quality_tier TEXT,
quality_reasons_json TEXT
);
CREATE TABLE IF NOT EXISTS turns (
id INTEGER PRIMARY KEY AUTOINCREMENT,
game_id TEXT NOT NULL,
turn INTEGER NOT NULL,
observed_at TEXT NOT NULL,
my_move TEXT,
my_thinking_json TEXT,
board_state_json TEXT NOT NULL,
snakes_json TEXT NOT NULL,
you_json TEXT NOT NULL,
food_json TEXT NOT NULL,
hazards_json TEXT NOT NULL,
UNIQUE (game_id, turn),
FOREIGN KEY (game_id) REFERENCES games(game_id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS game_snakes (
game_id TEXT NOT NULL,
snake_id TEXT NOT NULL,
snake_name TEXT,
is_you INTEGER NOT NULL DEFAULT 0,
customizations_json TEXT NOT NULL DEFAULT '{}',
PRIMARY KEY (game_id, snake_id),
FOREIGN KEY (game_id) REFERENCES games(game_id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS snake_turns (
id INTEGER PRIMARY KEY AUTOINCREMENT,
game_id TEXT NOT NULL,
turn INTEGER NOT NULL,
snake_id TEXT NOT NULL,
snake_name TEXT,
health INTEGER,
length INTEGER,
head_x INTEGER,
head_y INTEGER,
body_json TEXT NOT NULL,
is_you INTEGER NOT NULL DEFAULT 0,
inferred_move TEXT,
UNIQUE (game_id, turn, snake_id),
FOREIGN KEY (game_id) REFERENCES games(game_id) ON DELETE CASCADE
);
""")
if self._initialize_indexes:
self._create_indexes_if_tables(connection)
self._ensure_column_exists(connection, "turns", "my_thinking_json", "TEXT")
self._ensure_column_exists(connection, "games", "your_snake_type", "TEXT")
self._ensure_column_exists(connection, "games", "your_snake_version", "TEXT")
self._ensure_column_exists(connection, "games", "game_type", "TEXT")
self._ensure_column_exists(connection, "snake_turns", "latency", "TEXT")
self._ensure_column_exists(connection, "game_snakes", "customizations_json", "TEXT NOT NULL DEFAULT '{}'")
self._ensure_column_exists(connection, "games", "winner_name", "TEXT")
self._ensure_column_exists(connection, "games", "has_replay", "INTEGER NOT NULL DEFAULT 1")
self._ensure_column_exists(connection, "games", "quality_status", "TEXT NOT NULL DEFAULT 'retained'")
self._ensure_column_exists(connection, "games", "quality_score", "INTEGER")
self._ensure_column_exists(connection, "games", "quality_tier", "TEXT")
self._ensure_column_exists(connection, "games", "quality_reasons_json", "TEXT")
if self._zstd_available:
self._enable_zstd_compression(connection)
connection = self._connect()
try:
if connection.execute("PRAGMA auto_vacuum").fetchone()[0] != 1:
connection.execute("PRAGMA auto_vacuum = FULL")
connection.execute("VACUUM")
create_schema(connection)
connection.execute("PRAGMA optimize")
finally:
connection.close()
def _create_indexes_if_tables(self, connection:sqlite3.Connection) -> None:
real_tables = {
row[0] for row in connection.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()
}
indexes = [
("idx_turns_game_turn", "turns", "game_id, turn"),
("idx_games_status", "games", "status"),
("idx_snake_turns_game_turn", "snake_turns", "game_id, turn"),
]
for idx_name, table, cols in indexes:
if table in real_tables:
connection.execute(f"CREATE INDEX IF NOT EXISTS {idx_name} ON {table}({cols})")
# ── record helpers ─────────────────────────────────────────────────────────
def _ensure_column_exists(self, connection:sqlite3.Connection, table_name:str, column_name:str, column_type:str) -> None:
obj = connection.execute(
"SELECT type FROM sqlite_master WHERE name = ?", (table_name,)
).fetchone()
def _running_game(self, connection:sqlite3.Connection, game_id:str) -> RunningGame:
game = self._running.get(game_id)
if game is None:
row = connection.execute("SELECT gid FROM games WHERE game_id = ?", (game_id,)).fetchone()
if row is None:
raise LookupError(f"Turn for unknown game {game_id!r}")
game = RunningGame(gid=row["gid"])
return game
if obj and obj["type"] == "view":
underlying = f"_{table_name}_zstd"
exists = connection.execute(
"SELECT 1 FROM sqlite_master WHERE name = ? AND type = 'table'", (underlying,)
def _upsert_identities(self, connection:sqlite3.Connection, game_id:str, identities:list[SnakeIdentity]) -> dict[str, int]:
gsids = {}
for identity in identities:
row = connection.execute("""
INSERT INTO game_snakes (game_id, snake_id, snake_name, is_you, customizations_json, gsid)
VALUES (?, ?, ?, ?, ?, (SELECT COALESCE(MAX(gsid), 0) + 1 FROM game_snakes))
ON CONFLICT(game_id, snake_id) DO UPDATE SET
snake_name = excluded.snake_name,
is_you = excluded.is_you,
customizations_json = excluded.customizations_json
RETURNING gsid
""",
(
game_id, identity.snake_id, identity.name, 1 if identity.is_you else 0,
self._to_json(identity.customizations or {}),
),
).fetchone()
if not exists:
return
actual_table = underlying
else:
actual_table = table_name
gsids[identity.snake_id] = row["gsid"]
return gsids
existing = connection.execute(f"PRAGMA table_info({actual_table})").fetchall()
if any(row["name"] == column_name for row in existing):
return
connection.execute(f"ALTER TABLE {actual_table} ADD COLUMN {column_name} {column_type}")
def _enable_zstd_compression(self, connection:sqlite3.Connection) -> None:
compressed_columns = [
("turns", "board_state_json"),
("turns", "snakes_json"),
("turns", "you_json"),
("turns", "food_json"),
("turns", "hazards_json"),
("snake_turns", "body_json"),
]
for table, column in compressed_columns:
try:
connection.execute(
"SELECT zstd_enable_transparent(?)",
[json.dumps({"table": table, "column": column, "compression_level": 6, "dict_chooser": "'a'"})],
def _write_rows(self, connection:sqlite3.Connection, gid:int, gsids:dict[str, int], turns:list[TurnRecord], states:list[SnakeState]) -> None:
connection.executemany("""
INSERT INTO game_turns (gid, turn, observed_at, my_move, thinking_json, food, hazards)
VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(gid, turn) DO UPDATE SET
observed_at = excluded.observed_at,
my_move = excluded.my_move,
thinking_json = excluded.thinking_json,
food = excluded.food,
hazards = excluded.hazards
""",
[
(
gid, record.turn, self._ts_to_str(record.observed_at) or self._utc_now(),
encode_move(record.move),
self._to_json(record.thinking) if record.thinking is not None else None,
encode_points(record.food), encode_points(record.hazards),
)
except sqlite3.OperationalError:
pass
connection.execute("SELECT zstd_incremental_maintenance(null, 1)")
for record in turns
],
)
connection.executemany("""
INSERT INTO snake_states (gsid, turn, health, latency, body) VALUES (?, ?, ?, ?, ?)
ON CONFLICT(gsid, turn) DO UPDATE SET
health = excluded.health,
latency = excluded.latency,
body = excluded.body
""",
[
(gsids[state.snake_id], state.turn, state.health, state.latency, encode_body(state.body))
for state in states if state.snake_id in gsids
],
)
def _load_records(self, connection:sqlite3.Connection, game_id:str, gid:int) -> tuple[list[SnakeIdentity], list[TurnRecord], list[SnakeState]]:
identities = [
SnakeIdentity(row["snake_id"], row["snake_name"], bool(row["is_you"]), self._from_json(row["customizations_json"]) or {})
for row in connection.execute("""
SELECT snake_id, snake_name, is_you, customizations_json
FROM game_snakes WHERE game_id = ?
""", (game_id,))
]
turns = [
TurnRecord(
turn=row["turn"], observed_at=row["observed_at"], move=decode_move(row["my_move"]),
thinking=self._from_json(row["thinking_json"]),
food=decode_points(row["food"]), hazards=decode_points(row["hazards"]),
)
for row in connection.execute("""
SELECT turn, observed_at, my_move, thinking_json, food, hazards
FROM game_turns WHERE gid = ? ORDER BY turn
""", (gid,))
]
states = [
SnakeState(row["snake_id"], row["turn"], row["health"], row["latency"], decode_body(row["body"]))
for row in connection.execute("""
SELECT gs.snake_id, s.turn, s.health, s.latency, s.body
FROM game_snakes AS gs
JOIN snake_states AS s ON s.gsid = gs.gsid
WHERE gs.game_id = ?
""", (game_id,))
]
blob = connection.execute("SELECT format, data FROM game_replays WHERE gid = ?", (gid,)).fetchone()
if blob is not None:
packed_turns, packed_states = unpack_replay(blob["data"], blob["format"])
turns, states = merge_records(packed_turns, packed_states, turns, states)
return identities, turns, states
def _store_replay(self, connection:sqlite3.Connection, game_id:str, gid:int, turns:list[TurnRecord], states:list[SnakeState]) -> None:
connection.execute("""
INSERT INTO game_replays (gid, format, turn_count, data) VALUES (?, ?, ?, ?)
ON CONFLICT(gid) DO UPDATE SET
format = excluded.format, turn_count = excluded.turn_count, data = excluded.data
""", (gid, REPLAY_FORMAT, len(turns), pack_replay(turns, states)))
connection.execute("DELETE FROM game_turns WHERE gid = ?", (gid,))
connection.execute(
"DELETE FROM snake_states WHERE gsid IN (SELECT gsid FROM game_snakes WHERE game_id = ?)",
(game_id,),
)
def _pack_game(self, connection:sqlite3.Connection, game_id:str) -> None:
connection.execute("BEGIN IMMEDIATE")
try:
row = connection.execute("SELECT gid FROM games WHERE game_id = ?", (game_id,)).fetchone()
if row is not None and connection.execute(
"SELECT 1 FROM game_turns WHERE gid = ? LIMIT 1", (row["gid"],),
).fetchone():
_, turns, states = self._load_records(connection, game_id, row["gid"])
self._store_replay(connection, game_id, row["gid"], turns, states)
connection.execute("COMMIT")
except Exception:
connection.execute("ROLLBACK")
raise
# ── sync write methods ─────────────────────────────────────────────────────
@@ -216,13 +220,15 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
ruleset = game.get("ruleset", {})
game_type = self._derive_game_type(board, ruleset)
with self._connect() as connection:
connection = self._connect()
try:
connection.execute("""
INSERT INTO games (
game_id, started_at, width, height, source, map_name,
ruleset_name, ruleset_version, your_snake_id, your_snake_name,
your_snake_type, your_snake_version, game_type, status
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'running')
your_snake_type, your_snake_version, game_type, status, gid
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'running',
(SELECT COALESCE(MAX(gid), 0) + 1 FROM games))
ON CONFLICT(game_id) DO UPDATE SET
width = excluded.width,
height = excluded.height,
@@ -238,166 +244,71 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
status = 'running'
""",
(
game.get("id"),
self._utc_now(),
board.get("width"),
board.get("height"),
game.get("source"),
game.get("map"),
ruleset.get("name"),
ruleset.get("version"),
you.get("id"),
you.get("name"),
snake_type,
snake_version,
game_type,
game.get("id"), self._utc_now(), board.get("width"), board.get("height"),
game.get("source"), game.get("map"), ruleset.get("name"), ruleset.get("version"),
you.get("id"), you.get("name"), snake_type, snake_version, game_type,
),
)
connection.execute("PRAGMA wal_checkpoint(PASSIVE)")
connection.execute("PRAGMA optimize")
finally:
connection.close()
def _record_turn_sync(self, game_state:dict, my_move:str|None, my_thinking:dict|None) -> None:
game = game_state.get("game", {})
board = game_state.get("board", {})
snakes = self._extract_snakes(game_state)
you = self._extract_you(game_state)
game_id = game.get("id")
turn = int(game_state.get("turn", 0))
board_json, snakes_json, you_json, food_json, hazards_json = compact_turn_json(board)
with self._connect() as connection:
connection.executemany("""
INSERT INTO game_snakes (
game_id, snake_id, snake_name, is_you, customizations_json
) VALUES (?, ?, ?, ?, ?)
ON CONFLICT(game_id, snake_id) DO UPDATE SET
snake_name = excluded.snake_name,
is_you = excluded.is_you,
customizations_json = excluded.customizations_json
""",
[
(
game_id,
snake.get("id"),
snake.get("name"),
1 if snake.get("id") == you.get("id") else 0,
self._to_json(snake.get("customizations") or {}),
)
for snake in snakes if snake.get("id") is not None
],
)
connection.execute("""
INSERT INTO turns (
game_id, turn, observed_at, my_move, my_thinking_json,
board_state_json, snakes_json, you_json, food_json, hazards_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(game_id, turn) DO UPDATE SET
observed_at = excluded.observed_at,
my_move = excluded.my_move,
my_thinking_json = excluded.my_thinking_json,
board_state_json = excluded.board_state_json,
snakes_json = excluded.snakes_json,
you_json = excluded.you_json,
food_json = excluded.food_json,
hazards_json = excluded.hazards_json
""",
(
game_id,
turn,
self._utc_now(),
my_move,
self._to_json(my_thinking) if my_thinking is not None else None,
self._to_json(board_json),
self._to_json(snakes_json),
self._to_json(you_json),
self._to_json(food_json),
self._to_json(hazards_json),
),
)
previous_positions: dict[str, tuple[int, int]] = {}
if turn > 0:
previous_rows = connection.execute("""
SELECT snake_id, head_x, head_y
FROM snake_turns
WHERE game_id = ? AND turn = ?
""",
(game_id, turn - 1),
).fetchall()
previous_positions = {
row["snake_id"]: (int(row["head_x"]), int(row["head_y"]))
for row in previous_rows
if row["head_x"] is not None and row["head_y"] is not None
}
you_id = you.get("id")
for params in self._build_snake_turn_params(snakes, you_id, game_id, turn, previous_positions):
(p_game_id, p_turn, p_snake_id, p_name, p_health, p_length,
p_head_x, p_head_y, p_body, p_is_you, p_inferred, p_latency) = params
connection.execute("""
INSERT INTO snake_turns (
game_id, turn, snake_id, snake_name, health, length,
head_x, head_y, body_json, is_you, inferred_move, latency
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(game_id, turn, snake_id) DO UPDATE SET
snake_name = excluded.snake_name,
health = excluded.health,
length = excluded.length,
head_x = excluded.head_x,
head_y = excluded.head_y,
body_json = excluded.body_json,
is_you = excluded.is_you,
inferred_move = excluded.inferred_move,
latency = excluded.latency
""",
(
p_game_id, p_turn, p_snake_id, None, p_health, p_length,
p_head_x, p_head_y, self._to_json(p_body),
0,
p_inferred, p_latency,
),
game_id = game_state.get("game", {}).get("id")
connection = self._connect()
try:
connection.execute("BEGIN IMMEDIATE")
try:
running = self._running_game(connection, game_id)
record, states, identities, points = records_from_game_state(
game_state, my_move, my_thinking, running.points,
)
connection.execute("""
UPDATE games
SET final_turn = CASE WHEN ? > final_turn THEN ? ELSE final_turn END
WHERE game_id = ?
""",
(turn, turn, game_id),
)
if any(identity.snake_id not in running.snake_ids for identity in identities):
running.snake_ids.update(self._upsert_identities(connection, game_id, identities))
self._write_rows(connection, running.gid, running.snake_ids, [record], states)
connection.execute(
"UPDATE games SET final_turn = MAX(final_turn, ?) WHERE gid = ?",
(record.turn, running.gid),
)
connection.execute("COMMIT")
except Exception:
connection.execute("ROLLBACK")
raise
finally:
connection.close()
running.points = points
self._running.put(game_id, running)
def _record_game_end_sync(self, game_state:dict) -> None:
game_id, winner_name, winner_you, turn = self._extract_game_end_params(game_state)
with self._connect() as connection:
connection = self._connect()
try:
connection.execute("""
UPDATE games
SET ended_at = ?,
winner_name = ?,
winner_you = ?,
final_turn = CASE WHEN ? > final_turn THEN ? ELSE final_turn END,
final_turn = MAX(final_turn, ?),
status = 'finished'
WHERE game_id = ?
""",
(
self._utc_now(),
winner_name,
1 if winner_you else 0,
turn,
turn,
game_id,
),
(self._utc_now(), winner_name, 1 if winner_you else 0, turn, game_id),
)
self._pack_game(connection, game_id)
finally:
connection.close()
self._running.drop(game_id)
def _finalize_stale_running_games_sync(self, stale_after_seconds:int=600) -> int:
threshold = max(60, int(stale_after_seconds))
now_utc = datetime.now(timezone.utc)
finalized = 0
with self._connect() as connection:
connection = self._connect()
try:
rows = connection.execute("""
SELECT game_id, started_at, final_turn, your_snake_id
SELECT gid, game_id, started_at, final_turn, your_snake_id
FROM games
WHERE status = 'running'
ORDER BY started_at ASC
@@ -405,73 +316,47 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
for row in rows:
started_at = self._parse_utc_timestamp(row["started_at"])
if started_at is None:
continue
if (now_utc - started_at).total_seconds() < threshold:
if started_at is None or (now_utc - started_at).total_seconds() < threshold:
continue
game_id = row["game_id"]
your_snake_id = row["your_snake_id"]
final_turn = int(row["final_turn"] or 0)
snake_rows = connection.execute("""
SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name) AS snake_name
FROM snake_turns AS st
LEFT JOIN game_snakes AS gs
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
WHERE st.game_id = ? AND st.turn = ?
ORDER BY COALESCE(gs.is_you, st.is_you) DESC, snake_name ASC
""",
(game_id, final_turn),
).fetchall()
if len(snake_rows) == 0:
latest_row = connection.execute(
"SELECT MAX(turn) AS latest_turn FROM snake_turns WHERE game_id = ?",
(game_id,),
).fetchone()
if latest_row is not None and latest_row["latest_turn"] is not None:
final_turn = int(latest_row["latest_turn"])
snake_rows = connection.execute("""
SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name) AS snake_name
FROM snake_turns AS st
LEFT JOIN game_snakes AS gs
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
WHERE st.game_id = ? AND st.turn = ?
ORDER BY COALESCE(gs.is_you, st.is_you) DESC, snake_name ASC
""",
(game_id, final_turn),
).fetchall()
winner_you, survivor_name = self._calculate_survivor(snake_rows, your_snake_id)
identities, _, states = self._load_records(connection, game_id, row["gid"])
final_turn, survivors = self._survivors(identities, states, int(row["final_turn"] or 0))
winner_you, survivor_name = self._calculate_survivor(survivors, row["your_snake_id"])
result = connection.execute("""
UPDATE games
SET ended_at = ?,
winner_name = ?,
winner_you = ?,
final_turn = CASE WHEN ? > final_turn THEN ? ELSE final_turn END,
final_turn = MAX(final_turn, ?),
status = 'finished'
WHERE game_id = ? AND status = 'running'
""",
(
self._utc_now(),
survivor_name,
1 if winner_you else 0,
final_turn,
final_turn,
game_id,
),
(self._utc_now(), survivor_name, 1 if winner_you else 0, final_turn, game_id),
)
if result.rowcount > 0:
finalized += 1
self._pack_game(connection, game_id)
self._running.drop(game_id)
late = connection.execute("""
SELECT DISTINCT g.game_id
FROM game_turns AS t JOIN games AS g ON g.gid = t.gid
WHERE g.status = 'finished'
""").fetchall()
for late_row in late:
self._pack_game(connection, late_row["game_id"])
finally:
connection.close()
return finalized
# ── sync read methods ──────────────────────────────────────────────────────
def _get_summary_sync(self, recent_limit:int=15) -> dict:
with self._connect() as connection:
connection = self._connect()
try:
totals = connection.execute("""
SELECT
COUNT(*) AS total_games,
@@ -505,11 +390,14 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
""",
(max(1, int(recent_limit)),),
).fetchall()
finally:
connection.close()
return self._build_summary_output(totals, by_type, recent)
def _list_games_sync(self, limit:int=50) -> list[dict]:
with self._connect() as connection:
connection = self._connect()
try:
rows = connection.execute("""
SELECT game_id, started_at, ended_at, map_name, source, ruleset_name, game_type,
your_snake_name, your_snake_type, your_snake_version,
@@ -521,52 +409,75 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
""",
(max(1, int(limit)),),
).fetchall()
finally:
connection.close()
return self._build_game_list_output(rows)
def _get_game_replay_sync(self, game_id:str) -> dict|None:
with self._connect() as connection:
game_row = connection.execute("""
SELECT game_id, started_at, ended_at, width, height, source, map_name,
ruleset_name, ruleset_version, game_type, your_snake_id, your_snake_name,
your_snake_type, your_snake_version,
winner_name, winner_you, final_turn, status
FROM games
WHERE game_id = ? AND has_replay = 1
""",
(game_id,),
).fetchone()
connection = self._connect()
try:
game_row = connection.execute(_GAME_SELECT + " WHERE game_id = ? AND has_replay = 1", (game_id,)).fetchone()
if game_row is None:
return None
identities, turns, states = self._load_records(connection, game_id, game_row["gid"])
finally:
connection.close()
turn_rows = connection.execute("""
SELECT turn, observed_at, my_move, my_thinking_json,
board_state_json, food_json, hazards_json, you_json
FROM turns
WHERE game_id = ?
ORDER BY turn ASC
""",
(game_id,),
).fetchall()
return self._build_game_replay_output(game_row, identities, turns, states)
snake_rows = connection.execute("""
SELECT st.turn, st.snake_id,
COALESCE(gs.snake_name, st.snake_name) AS snake_name,
st.health, st.length, st.head_x, st.head_y, st.body_json,
COALESCE(gs.is_you, st.is_you) AS is_you,
COALESCE(gs.customizations_json, '{}') AS customizations_json,
st.inferred_move, st.latency
FROM snake_turns AS st
LEFT JOIN game_snakes AS gs
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
WHERE st.game_id = ?
ORDER BY st.turn ASC, is_you DESC, snake_name ASC
""",
(game_id,),
).fetchall()
# ── bundles (migration / merge) ────────────────────────────────────────────
return self._build_game_replay_output(game_row, turn_rows, snake_rows, decode_json=self._from_json)
def list_game_ids_sync(self) -> list[str]:
connection = self._connect()
try:
return [row[0] for row in connection.execute("SELECT game_id FROM games ORDER BY started_at, game_id")]
finally:
connection.close()
def read_game_bundle_sync(self, game_id:str) -> GameBundle|None:
connection = self._connect()
try:
row = connection.execute(_GAME_SELECT + " WHERE game_id = ?", (game_id,)).fetchone()
if row is None:
return None
identities, turns, states = self._load_records(connection, game_id, row["gid"])
finally:
connection.close()
return GameBundle(_game_from_row(row), identities, turns, states)
def import_game_bundle_sync(self, bundle:GameBundle) -> None:
game = normalize_game(bundle.game)
columns = [column for column in GAME_COLUMNS if column != "quality_reasons"] + ["quality_reasons_json"]
values = [game[column] for column in GAME_COLUMNS if column != "quality_reasons"]
values.append(self._to_json(game["quality_reasons"]) if game["quality_reasons"] is not None else None)
values = [self._ts_to_str(value) if isinstance(value, datetime) else value for value in values]
updates = ", ".join(f"{column} = excluded.{column}" for column in columns[1:])
connection = self._connect()
try:
connection.execute("BEGIN IMMEDIATE")
try:
connection.execute(
f"INSERT INTO games ({', '.join(columns)}, gid) "
f"VALUES ({', '.join('?' for _ in columns)}, (SELECT COALESCE(MAX(gid), 0) + 1 FROM games)) "
f"ON CONFLICT(game_id) DO UPDATE SET {updates}",
values,
)
gid = connection.execute("SELECT gid FROM games WHERE game_id = ?", (game["game_id"],)).fetchone()["gid"]
gsids = self._upsert_identities(connection, game["game_id"], bundle.snakes)
if bundle.turns and game["has_replay"]:
if game["status"] == "finished":
self._store_replay(connection, game["game_id"], gid, bundle.turns, bundle.states)
else:
self._write_rows(connection, gid, gsids, bundle.turns, bundle.states)
connection.execute("COMMIT")
except Exception:
connection.execute("ROLLBACK")
raise
finally:
connection.close()
self._running.drop(game["game_id"])
# ── public async interface ─────────────────────────────────────────────────
@@ -591,5 +502,14 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
async def get_game_replay(self, game_id:str) -> dict|None:
return await asyncio.to_thread(self._get_game_replay_sync, game_id)
async def list_game_ids(self) -> list[str]:
return await asyncio.to_thread(self.list_game_ids_sync)
async def read_game_bundle(self, game_id:str) -> GameBundle|None:
return await asyncio.to_thread(self.read_game_bundle_sync, game_id)
async def import_game_bundle(self, bundle:GameBundle) -> None:
await asyncio.to_thread(self.import_game_bundle_sync, bundle)
async def close(self) -> None:
return None
server/database/backend/Template.py
+47 -64
View File
@@ -2,7 +2,8 @@ import json
from datetime import datetime, timezone
from typing import Any
from server.database.normalized_turn import hydrate_replay_turns
from server.database.compact.hydrate import hydrate_replay_turns
from server.database.compact.records import GameBundle, SnakeIdentity, SnakeState, TurnRecord
class GameplayBackendTemplate:
"""Abstract base for gameplay database backends.
@@ -40,6 +41,17 @@ class GameplayBackendTemplate:
async def get_game_replay(self, game_id:str) -> dict|None:
raise NotImplementedError
async def read_game_bundle(self, game_id:str) -> GameBundle|None:
"""Return all stored data of one game (used by migrations and merges)."""
raise NotImplementedError
async def import_game_bundle(self, bundle:GameBundle) -> None:
"""Store a complete game; finished games are packed into a replay blob."""
raise NotImplementedError
async def list_game_ids(self) -> list[str]:
raise NotImplementedError
async def close(self) -> None:
return None
@@ -79,22 +91,6 @@ class GameplayBackendTemplate:
def _extract_you(self, game_state:dict) -> dict:
return dict(game_state.get("you", {}))
def _infer_direction(self, old_head:tuple[int, int]|None, new_head:tuple[int, int]|None) -> str|None:
if old_head is None or new_head is None:
return None
dx = new_head[0] - old_head[0]
dy = new_head[1] - old_head[1]
if dx == 1 and dy == 0:
return "right"
if dx == -1 and dy == 0:
return "left"
if dx == 0 and dy == 1:
return "up"
if dx == 0 and dy == -1:
return "down"
return None
def _derive_game_type(self, board:dict, ruleset:dict) -> str:
if len(board.get("snakes", [])) == 2:
return "duel"
@@ -159,14 +155,21 @@ class GameplayBackendTemplate:
"status": row["status"],
} for row in rows]
def _build_game_replay_output(self, game_row, turn_rows, snake_rows, decode_json) -> dict:
"""Build the full replay dict.
decode_json: callable applied to raw column values that may be JSON strings
(SQLite) or already-decoded objects (PostgreSQL). Pass self._from_json for
SQLite; pass (lambda x: x) for PostgreSQL.
"""
hydrated_turns = hydrate_replay_turns(game_row, turn_rows, snake_rows, decode_json)
def _build_game_replay_output(
self,
game_row,
identities:list[SnakeIdentity],
turns:list[TurnRecord],
states:list[SnakeState],
) -> dict:
game = {
"width": game_row["width"],
"height": game_row["height"],
"your_snake_id": game_row["your_snake_id"],
}
hydrated_turns = hydrate_replay_turns(
game, {identity.snake_id: identity for identity in identities}, turns, states,
)
return {
"game": {
@@ -197,45 +200,6 @@ class GameplayBackendTemplate:
# ── shared write-path helpers ──────────────────────────────────────────────
def _build_snake_turn_params(
self,
snakes:list[dict],
you_id:str|None,
game_id:str|None,
turn:int,
previous_positions:dict,
) -> list[tuple]:
"""Return one parameter tuple per snake for a snake_turns INSERT."""
result = []
for snake in snakes:
snake_id = snake.get("id")
if snake_id is None:
continue
head = snake.get("head", {})
head_x = head.get("x")
head_y = head.get("y")
new_head = (
(int(head_x), int(head_y))
if head_x is not None and head_y is not None
else None
)
inferred = self._infer_direction(previous_positions.get(snake_id), new_head)
result.append((
game_id,
turn,
snake_id,
snake.get("name"),
snake.get("health"),
snake.get("length"),
head_x,
head_y,
snake.get("body", []),
snake_id == you_id,
inferred,
snake.get("latency"),
))
return result
def _extract_game_end_params(self, game_state:dict) -> tuple:
"""Return (game_id, winner_name, winner_you, turn) from a game_end payload."""
game = game_state.get("game", {})
@@ -249,6 +213,25 @@ class GameplayBackendTemplate:
turn = int(game_state.get("turn", 0))
return game_id, winner_name, winner_you, turn
def _survivors(self, identities:list[SnakeIdentity], states:list[SnakeState], final_turn:int) -> tuple[int, list[dict]]:
"""Snakes alive at the final (or latest stored) turn, you first, then by name."""
turns = {state.turn for state in states}
if final_turn not in turns and turns:
final_turn = max(turns)
by_id = {identity.snake_id: identity for identity in identities}
alive = []
for state in states:
if state.turn != final_turn:
continue
identity = by_id.get(state.snake_id)
alive.append({
"snake_id": state.snake_id,
"snake_name": identity.name if identity else None,
"is_you": bool(identity and identity.is_you),
})
alive.sort(key=lambda row: (not row["is_you"], row["snake_name"] is None, row["snake_name"] or ""))
return final_turn, alive
def _calculate_survivor(self, snake_rows, your_snake_id:str|None) -> tuple[bool, str|None]:
"""Return (winner_you, survivor_name) from a list of snake_turns rows."""
survivor_ids = [s["snake_id"] for s in snake_rows if s["snake_id"]]
server/database/backend/postgresql_schema.py
+106
View File
@@ -0,0 +1,106 @@
"""PostgreSQL schema for compact gameplay storage.
Finished games live in ``game_replays`` as one zstd blob per game. Running
games use ``game_turns`` and ``snake_states`` keyed by small integer ids
(``games.gid`` / ``game_snakes.gsid``) instead of repeating text ids per row.
"""
TABLES_DDL = """
CREATE TABLE IF NOT EXISTS games (
game_id TEXT PRIMARY KEY,
started_at TIMESTAMPTZ NOT NULL,
ended_at TIMESTAMPTZ,
width INTEGER,
height INTEGER,
source TEXT,
map_name TEXT,
ruleset_name TEXT,
ruleset_version TEXT,
your_snake_id TEXT,
your_snake_name TEXT,
your_snake_type TEXT,
your_snake_version TEXT,
game_type TEXT,
winner_name TEXT,
winner_you BOOLEAN NOT NULL DEFAULT FALSE,
final_turn INTEGER NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'running',
has_replay BOOLEAN NOT NULL DEFAULT TRUE,
quality_status TEXT NOT NULL DEFAULT 'retained',
quality_score INTEGER,
quality_tier TEXT,
quality_reasons JSONB
);
CREATE TABLE IF NOT EXISTS game_snakes (
game_id TEXT NOT NULL REFERENCES games(game_id) ON DELETE CASCADE,
snake_id TEXT NOT NULL,
snake_name TEXT,
is_you BOOLEAN NOT NULL DEFAULT FALSE,
customizations JSONB NOT NULL DEFAULT '{}'::jsonb,
PRIMARY KEY (game_id, snake_id)
);
ALTER TABLE games ADD COLUMN IF NOT EXISTS game_type TEXT;
ALTER TABLE games ADD COLUMN IF NOT EXISTS your_snake_type TEXT;
ALTER TABLE games ADD COLUMN IF NOT EXISTS your_snake_version TEXT;
ALTER TABLE games ADD COLUMN IF NOT EXISTS winner_name TEXT;
ALTER TABLE games ADD COLUMN IF NOT EXISTS has_replay BOOLEAN NOT NULL DEFAULT TRUE;
ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_status TEXT NOT NULL DEFAULT 'retained';
ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_score INTEGER;
ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_tier TEXT;
ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_reasons JSONB;
ALTER TABLE game_snakes ADD COLUMN IF NOT EXISTS customizations JSONB NOT NULL DEFAULT '{}'::jsonb;
ALTER TABLE games ADD COLUMN IF NOT EXISTS gid INTEGER GENERATED BY DEFAULT AS IDENTITY;
ALTER TABLE game_snakes ADD COLUMN IF NOT EXISTS gsid INTEGER GENERATED BY DEFAULT AS IDENTITY;
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'games_gid_key') THEN
ALTER TABLE games ADD CONSTRAINT games_gid_key UNIQUE (gid);
END IF;
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'game_snakes_gsid_key') THEN
ALTER TABLE game_snakes ADD CONSTRAINT game_snakes_gsid_key UNIQUE (gsid);
END IF;
END $$;
CREATE TABLE IF NOT EXISTS game_turns (
gid INTEGER NOT NULL REFERENCES games(gid) ON DELETE CASCADE,
turn INTEGER NOT NULL,
observed_at TIMESTAMPTZ NOT NULL,
my_move SMALLINT,
thinking JSONB,
food BYTEA,
hazards BYTEA,
PRIMARY KEY (gid, turn)
);
CREATE TABLE IF NOT EXISTS snake_states (
gsid INTEGER NOT NULL REFERENCES game_snakes(gsid) ON DELETE CASCADE,
turn INTEGER NOT NULL,
health SMALLINT,
latency SMALLINT,
body BYTEA NOT NULL,
PRIMARY KEY (gsid, turn)
);
CREATE TABLE IF NOT EXISTS game_replays (
gid INTEGER PRIMARY KEY REFERENCES games(gid) ON DELETE CASCADE,
format SMALLINT NOT NULL,
turn_count INTEGER NOT NULL,
data BYTEA NOT NULL
);
-- Replay blobs are already zstd-compressed; skip TOAST compression attempts.
ALTER TABLE game_replays ALTER COLUMN data SET STORAGE EXTERNAL;
CREATE INDEX IF NOT EXISTS idx_games_status ON games(status);
"""
# Superseded by the UNIQUE constraints of the legacy turns/snake_turns tables.
# CONCURRENTLY avoids blocking writes; each must run outside a transaction.
DROP_REDUNDANT_INDEXES = (
"DROP INDEX CONCURRENTLY IF EXISTS idx_turns_game_turn",
"DROP INDEX CONCURRENTLY IF EXISTS idx_snake_turns_game_turn",
)
server/database/backend/sqlite_schema.py
+114
View File
@@ -0,0 +1,114 @@
"""SQLite schema for compact gameplay storage (mirrors postgresql_schema)."""
import sqlite3
BASE_TABLES = """
CREATE TABLE IF NOT EXISTS games (
game_id TEXT PRIMARY KEY,
started_at TEXT NOT NULL,
ended_at TEXT,
width INTEGER,
height INTEGER,
source TEXT,
map_name TEXT,
ruleset_name TEXT,
ruleset_version TEXT,
your_snake_id TEXT,
your_snake_name TEXT,
your_snake_type TEXT,
your_snake_version TEXT,
game_type TEXT,
winner_name TEXT,
winner_you INTEGER NOT NULL DEFAULT 0,
final_turn INTEGER NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'running',
has_replay INTEGER NOT NULL DEFAULT 1,
quality_status TEXT NOT NULL DEFAULT 'retained',
quality_score INTEGER,
quality_tier TEXT,
quality_reasons_json TEXT,
gid INTEGER
);
CREATE TABLE IF NOT EXISTS game_snakes (
game_id TEXT NOT NULL,
snake_id TEXT NOT NULL,
snake_name TEXT,
is_you INTEGER NOT NULL DEFAULT 0,
customizations_json TEXT NOT NULL DEFAULT '{}',
gsid INTEGER,
PRIMARY KEY (game_id, snake_id),
FOREIGN KEY (game_id) REFERENCES games(game_id) ON DELETE CASCADE
);
"""
# Columns added over time; older databases get them via ALTER TABLE.
LEGACY_COLUMNS = (
("games", "your_snake_type", "TEXT"),
("games", "your_snake_version", "TEXT"),
("games", "game_type", "TEXT"),
("games", "winner_name", "TEXT"),
("games", "has_replay", "INTEGER NOT NULL DEFAULT 1"),
("games", "quality_status", "TEXT NOT NULL DEFAULT 'retained'"),
("games", "quality_score", "INTEGER"),
("games", "quality_tier", "TEXT"),
("games", "quality_reasons_json", "TEXT"),
("games", "gid", "INTEGER"),
("game_snakes", "customizations_json", "TEXT NOT NULL DEFAULT '{}'"),
("game_snakes", "gsid", "INTEGER"),
)
COMPACT_TABLES = """
CREATE UNIQUE INDEX IF NOT EXISTS games_gid_key ON games(gid);
CREATE UNIQUE INDEX IF NOT EXISTS game_snakes_gsid_key ON game_snakes(gsid);
CREATE INDEX IF NOT EXISTS idx_games_status ON games(status);
UPDATE games
SET gid = rowid + (SELECT COALESCE(MAX(gid), 0) FROM games)
WHERE gid IS NULL;
UPDATE game_snakes
SET gsid = rowid + (SELECT COALESCE(MAX(gsid), 0) FROM game_snakes)
WHERE gsid IS NULL;
CREATE TABLE IF NOT EXISTS game_turns (
gid INTEGER NOT NULL REFERENCES games(gid) ON DELETE CASCADE,
turn INTEGER NOT NULL,
observed_at TEXT NOT NULL,
my_move INTEGER,
thinking_json TEXT,
food BLOB,
hazards BLOB,
PRIMARY KEY (gid, turn)
) WITHOUT ROWID;
CREATE TABLE IF NOT EXISTS snake_states (
gsid INTEGER NOT NULL REFERENCES game_snakes(gsid) ON DELETE CASCADE,
turn INTEGER NOT NULL,
health INTEGER,
latency INTEGER,
body BLOB NOT NULL,
PRIMARY KEY (gsid, turn)
) WITHOUT ROWID;
CREATE TABLE IF NOT EXISTS game_replays (
gid INTEGER PRIMARY KEY REFERENCES games(gid) ON DELETE CASCADE,
format INTEGER NOT NULL,
turn_count INTEGER NOT NULL,
data BLOB NOT NULL
);
DROP INDEX IF EXISTS idx_turns_game_turn;
DROP INDEX IF EXISTS idx_snake_turns_game_turn;
"""
def ensure_column(connection:sqlite3.Connection, table:str, column:str, column_type:str) -> None:
existing = {row[1] for row in connection.execute(f"PRAGMA table_info({table})")}
if column not in existing:
connection.execute(f"ALTER TABLE {table} ADD COLUMN {column} {column_type}")
def create_schema(connection:sqlite3.Connection) -> None:
connection.executescript(BASE_TABLES)
for table, column, column_type in LEGACY_COLUMNS:
ensure_column(connection, table, column, column_type)
connection.executescript(COMPACT_TABLES)