feat(database): store gameplay replays in compact form
Build and Push Docker Container / build-and-push (push) Successful in 3m52s
Build and Push Docker Container / build-and-push (push) Successful in 3m52s
- Add compact storage package: codec, records, hydrate, replay blobs, cache. - Running games write game_turns and snake_states keyed by integer ids. - Encode moves as 1-byte codes and bodies as head plus step codes. - Store food and hazards only when they changed since the previous turn. - Strip reasoning fields that duplicate stored values and restore on read. - Pack finished games into one zstd blob per game in game_replays. - Merge late turns into packed blobs during the stale-game sweep. - Extract SQLite and PostgreSQL schemas into dedicated modules. - Add read/import game bundle and list_game_ids to the backend template. - Add legacy readers and converter for the old turns/snake_turns layout. - Add migrate_gameplay_to_compact.py with in-place or cross-DB migration. - Verify every migrated game and keep legacy rows on failure. - Support --since, --minimum-quality and guarded --drop-legacy. - Add just recipe migrate-gameplay-postgres. - Rewrite merge script to copy game by game via backend bundles. - Move benchmark state loading into benchmark_states for both backends. - Update PostgreSQL cleanup script to the new replay tables. - Drop sqlite-zstd build stages from the Dockerfile. - Remove old migration scripts, normalized_turn and their tests. - Add tests for compact storage, schema, legacy migration and PostgreSQL. - Document the compact layout and migration workflow in the README.
This commit is contained in:
@@ -1,11 +1,24 @@
|
||||
from quart_common.web.env import env_bool
|
||||
"""SQLite gameplay backend with the same compact layout as PostgreSQL.
|
||||
|
||||
import asyncio, sqlite3, json, os, logging, sys
|
||||
Running games use ``game_turns``/``snake_states``; finished games are packed
|
||||
into one zstd blob per game in ``game_replays``.
|
||||
"""
|
||||
|
||||
import asyncio, json, sqlite3, logging, sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from server.database.backend.Template import GameplayBackendTemplate
|
||||
from server.database.normalized_turn import compact_turn_json
|
||||
from server.database.backend.sqlite_schema import create_schema
|
||||
from server.database.compact.cache import RunningGame, RunningGameCache
|
||||
from server.database.compact.codec import (
|
||||
decode_body, decode_move, decode_points, encode_body, encode_move, encode_points,
|
||||
)
|
||||
from server.database.compact.games import GAME_COLUMNS, normalize_game
|
||||
from server.database.compact.records import (
|
||||
GameBundle, SnakeIdentity, SnakeState, TurnRecord, merge_records, records_from_game_state,
|
||||
)
|
||||
from server.database.compact.replay_blob import REPLAY_FORMAT, pack_replay, unpack_replay
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
if not logger.handlers:
|
||||
@@ -14,37 +27,45 @@ if not logger.handlers:
|
||||
logger.addHandler(_handler)
|
||||
logger.propagate = False
|
||||
|
||||
_ZSTD_EXT = Path(os.environ.get("SQLITE_ZSTD_EXT", "/usr/local/lib/libsqlite_zstd.so")).expanduser().resolve()
|
||||
_GAME_SELECT = """
|
||||
SELECT gid, game_id, started_at, ended_at, width, height, source, map_name,
|
||||
ruleset_name, ruleset_version, game_type, your_snake_id, your_snake_name,
|
||||
your_snake_type, your_snake_version, winner_name, winner_you, final_turn,
|
||||
status, has_replay, quality_status, quality_score, quality_tier, quality_reasons_json
|
||||
FROM games
|
||||
"""
|
||||
|
||||
def _game_from_row(row:sqlite3.Row) -> dict:
|
||||
data = dict(row)
|
||||
reasons = data.pop("quality_reasons_json", None)
|
||||
data["quality_reasons"] = json.loads(reasons) if reasons else None
|
||||
return normalize_game(data)
|
||||
|
||||
class SqliteGameplayBackend(GameplayBackendTemplate):
|
||||
def __init__(self, db_path:str, busy_timeout_ms:int=5000, initialize_indexes:bool=True, journal_mode:str="WAL"):
|
||||
def __init__(self, db_path:str, busy_timeout_ms:int=5000, journal_mode:str="WAL", read_only:bool=False):
|
||||
self.db_path = db_path
|
||||
self.busy_timeout_ms = max(1000, int(busy_timeout_ms))
|
||||
self._zstd_available = False
|
||||
self._initialize_indexes = initialize_indexes
|
||||
self._journal_mode = journal_mode
|
||||
self._initialize_database()
|
||||
self._read_only = read_only
|
||||
self._running = RunningGameCache()
|
||||
if not read_only:
|
||||
self._initialize_database()
|
||||
|
||||
# ── connection ─────────────────────────────────────────────────────────────
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
if self._read_only:
|
||||
connection = sqlite3.connect(f"file:{self.db_path}?mode=ro", uri=True, isolation_level=None)
|
||||
connection.row_factory = sqlite3.Row
|
||||
connection.execute("PRAGMA query_only = ON")
|
||||
return connection
|
||||
|
||||
connection = sqlite3.connect(
|
||||
self.db_path,
|
||||
timeout=max(1, self.busy_timeout_ms // 1000),
|
||||
isolation_level=None,
|
||||
)
|
||||
connection.row_factory = sqlite3.Row
|
||||
|
||||
if _ZSTD_EXT.exists() and not env_bool('DISABLE_GAMEPLAY_DB_COMPRESSION', True):
|
||||
try:
|
||||
connection.enable_load_extension(True)
|
||||
connection.load_extension(str(_ZSTD_EXT))
|
||||
self._zstd_available = True
|
||||
except sqlite3.OperationalError as e:
|
||||
logger.warning(f"sqlite-zstd extension skipped: {e}")
|
||||
finally:
|
||||
connection.enable_load_extension(False)
|
||||
|
||||
connection.execute("PRAGMA foreign_keys = ON")
|
||||
connection.execute(f"PRAGMA journal_mode = {self._journal_mode}")
|
||||
connection.execute("PRAGMA synchronous = NORMAL")
|
||||
@@ -53,159 +74,142 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
|
||||
connection.execute(f"PRAGMA busy_timeout = {self.busy_timeout_ms}")
|
||||
return connection
|
||||
|
||||
def _ensure_auto_vacuum_full(self, connection:sqlite3.Connection) -> None:
|
||||
current = connection.execute("PRAGMA auto_vacuum").fetchone()[0]
|
||||
if current != 1:
|
||||
connection.execute("PRAGMA auto_vacuum = FULL")
|
||||
connection.execute("VACUUM")
|
||||
|
||||
# ── schema setup ───────────────────────────────────────────────────────────
|
||||
|
||||
def _initialize_database(self) -> None:
|
||||
Path(self.db_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
with self._connect() as connection:
|
||||
self._ensure_auto_vacuum_full(connection)
|
||||
connection.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS games (
|
||||
game_id TEXT PRIMARY KEY,
|
||||
started_at TEXT NOT NULL,
|
||||
ended_at TEXT,
|
||||
width INTEGER,
|
||||
height INTEGER,
|
||||
source TEXT,
|
||||
map_name TEXT,
|
||||
ruleset_name TEXT,
|
||||
ruleset_version TEXT,
|
||||
your_snake_id TEXT,
|
||||
your_snake_name TEXT,
|
||||
your_snake_type TEXT,
|
||||
your_snake_version TEXT,
|
||||
winner_name TEXT,
|
||||
winner_you INTEGER NOT NULL DEFAULT 0,
|
||||
final_turn INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL DEFAULT 'running',
|
||||
has_replay INTEGER NOT NULL DEFAULT 1,
|
||||
quality_status TEXT NOT NULL DEFAULT 'retained',
|
||||
quality_score INTEGER,
|
||||
quality_tier TEXT,
|
||||
quality_reasons_json TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS turns (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
game_id TEXT NOT NULL,
|
||||
turn INTEGER NOT NULL,
|
||||
observed_at TEXT NOT NULL,
|
||||
my_move TEXT,
|
||||
my_thinking_json TEXT,
|
||||
board_state_json TEXT NOT NULL,
|
||||
snakes_json TEXT NOT NULL,
|
||||
you_json TEXT NOT NULL,
|
||||
food_json TEXT NOT NULL,
|
||||
hazards_json TEXT NOT NULL,
|
||||
UNIQUE (game_id, turn),
|
||||
FOREIGN KEY (game_id) REFERENCES games(game_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS game_snakes (
|
||||
game_id TEXT NOT NULL,
|
||||
snake_id TEXT NOT NULL,
|
||||
snake_name TEXT,
|
||||
is_you INTEGER NOT NULL DEFAULT 0,
|
||||
customizations_json TEXT NOT NULL DEFAULT '{}',
|
||||
PRIMARY KEY (game_id, snake_id),
|
||||
FOREIGN KEY (game_id) REFERENCES games(game_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS snake_turns (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
game_id TEXT NOT NULL,
|
||||
turn INTEGER NOT NULL,
|
||||
snake_id TEXT NOT NULL,
|
||||
snake_name TEXT,
|
||||
health INTEGER,
|
||||
length INTEGER,
|
||||
head_x INTEGER,
|
||||
head_y INTEGER,
|
||||
body_json TEXT NOT NULL,
|
||||
is_you INTEGER NOT NULL DEFAULT 0,
|
||||
inferred_move TEXT,
|
||||
UNIQUE (game_id, turn, snake_id),
|
||||
FOREIGN KEY (game_id) REFERENCES games(game_id) ON DELETE CASCADE
|
||||
);
|
||||
""")
|
||||
if self._initialize_indexes:
|
||||
self._create_indexes_if_tables(connection)
|
||||
self._ensure_column_exists(connection, "turns", "my_thinking_json", "TEXT")
|
||||
self._ensure_column_exists(connection, "games", "your_snake_type", "TEXT")
|
||||
self._ensure_column_exists(connection, "games", "your_snake_version", "TEXT")
|
||||
self._ensure_column_exists(connection, "games", "game_type", "TEXT")
|
||||
self._ensure_column_exists(connection, "snake_turns", "latency", "TEXT")
|
||||
self._ensure_column_exists(connection, "game_snakes", "customizations_json", "TEXT NOT NULL DEFAULT '{}'")
|
||||
self._ensure_column_exists(connection, "games", "winner_name", "TEXT")
|
||||
self._ensure_column_exists(connection, "games", "has_replay", "INTEGER NOT NULL DEFAULT 1")
|
||||
self._ensure_column_exists(connection, "games", "quality_status", "TEXT NOT NULL DEFAULT 'retained'")
|
||||
self._ensure_column_exists(connection, "games", "quality_score", "INTEGER")
|
||||
self._ensure_column_exists(connection, "games", "quality_tier", "TEXT")
|
||||
self._ensure_column_exists(connection, "games", "quality_reasons_json", "TEXT")
|
||||
if self._zstd_available:
|
||||
self._enable_zstd_compression(connection)
|
||||
connection = self._connect()
|
||||
try:
|
||||
if connection.execute("PRAGMA auto_vacuum").fetchone()[0] != 1:
|
||||
connection.execute("PRAGMA auto_vacuum = FULL")
|
||||
connection.execute("VACUUM")
|
||||
create_schema(connection)
|
||||
connection.execute("PRAGMA optimize")
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def _create_indexes_if_tables(self, connection:sqlite3.Connection) -> None:
|
||||
real_tables = {
|
||||
row[0] for row in connection.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table'"
|
||||
).fetchall()
|
||||
}
|
||||
indexes = [
|
||||
("idx_turns_game_turn", "turns", "game_id, turn"),
|
||||
("idx_games_status", "games", "status"),
|
||||
("idx_snake_turns_game_turn", "snake_turns", "game_id, turn"),
|
||||
]
|
||||
for idx_name, table, cols in indexes:
|
||||
if table in real_tables:
|
||||
connection.execute(f"CREATE INDEX IF NOT EXISTS {idx_name} ON {table}({cols})")
|
||||
# ── record helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
def _ensure_column_exists(self, connection:sqlite3.Connection, table_name:str, column_name:str, column_type:str) -> None:
|
||||
obj = connection.execute(
|
||||
"SELECT type FROM sqlite_master WHERE name = ?", (table_name,)
|
||||
).fetchone()
|
||||
def _running_game(self, connection:sqlite3.Connection, game_id:str) -> RunningGame:
|
||||
game = self._running.get(game_id)
|
||||
if game is None:
|
||||
row = connection.execute("SELECT gid FROM games WHERE game_id = ?", (game_id,)).fetchone()
|
||||
if row is None:
|
||||
raise LookupError(f"Turn for unknown game {game_id!r}")
|
||||
game = RunningGame(gid=row["gid"])
|
||||
return game
|
||||
|
||||
if obj and obj["type"] == "view":
|
||||
underlying = f"_{table_name}_zstd"
|
||||
exists = connection.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE name = ? AND type = 'table'", (underlying,)
|
||||
def _upsert_identities(self, connection:sqlite3.Connection, game_id:str, identities:list[SnakeIdentity]) -> dict[str, int]:
|
||||
gsids = {}
|
||||
for identity in identities:
|
||||
row = connection.execute("""
|
||||
INSERT INTO game_snakes (game_id, snake_id, snake_name, is_you, customizations_json, gsid)
|
||||
VALUES (?, ?, ?, ?, ?, (SELECT COALESCE(MAX(gsid), 0) + 1 FROM game_snakes))
|
||||
ON CONFLICT(game_id, snake_id) DO UPDATE SET
|
||||
snake_name = excluded.snake_name,
|
||||
is_you = excluded.is_you,
|
||||
customizations_json = excluded.customizations_json
|
||||
RETURNING gsid
|
||||
""",
|
||||
(
|
||||
game_id, identity.snake_id, identity.name, 1 if identity.is_you else 0,
|
||||
self._to_json(identity.customizations or {}),
|
||||
),
|
||||
).fetchone()
|
||||
if not exists:
|
||||
return
|
||||
actual_table = underlying
|
||||
else:
|
||||
actual_table = table_name
|
||||
gsids[identity.snake_id] = row["gsid"]
|
||||
return gsids
|
||||
|
||||
existing = connection.execute(f"PRAGMA table_info({actual_table})").fetchall()
|
||||
if any(row["name"] == column_name for row in existing):
|
||||
return
|
||||
connection.execute(f"ALTER TABLE {actual_table} ADD COLUMN {column_name} {column_type}")
|
||||
|
||||
def _enable_zstd_compression(self, connection:sqlite3.Connection) -> None:
|
||||
compressed_columns = [
|
||||
("turns", "board_state_json"),
|
||||
("turns", "snakes_json"),
|
||||
("turns", "you_json"),
|
||||
("turns", "food_json"),
|
||||
("turns", "hazards_json"),
|
||||
("snake_turns", "body_json"),
|
||||
]
|
||||
for table, column in compressed_columns:
|
||||
try:
|
||||
connection.execute(
|
||||
"SELECT zstd_enable_transparent(?)",
|
||||
[json.dumps({"table": table, "column": column, "compression_level": 6, "dict_chooser": "'a'"})],
|
||||
def _write_rows(self, connection:sqlite3.Connection, gid:int, gsids:dict[str, int], turns:list[TurnRecord], states:list[SnakeState]) -> None:
|
||||
connection.executemany("""
|
||||
INSERT INTO game_turns (gid, turn, observed_at, my_move, thinking_json, food, hazards)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(gid, turn) DO UPDATE SET
|
||||
observed_at = excluded.observed_at,
|
||||
my_move = excluded.my_move,
|
||||
thinking_json = excluded.thinking_json,
|
||||
food = excluded.food,
|
||||
hazards = excluded.hazards
|
||||
""",
|
||||
[
|
||||
(
|
||||
gid, record.turn, self._ts_to_str(record.observed_at) or self._utc_now(),
|
||||
encode_move(record.move),
|
||||
self._to_json(record.thinking) if record.thinking is not None else None,
|
||||
encode_points(record.food), encode_points(record.hazards),
|
||||
)
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
connection.execute("SELECT zstd_incremental_maintenance(null, 1)")
|
||||
for record in turns
|
||||
],
|
||||
)
|
||||
connection.executemany("""
|
||||
INSERT INTO snake_states (gsid, turn, health, latency, body) VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(gsid, turn) DO UPDATE SET
|
||||
health = excluded.health,
|
||||
latency = excluded.latency,
|
||||
body = excluded.body
|
||||
""",
|
||||
[
|
||||
(gsids[state.snake_id], state.turn, state.health, state.latency, encode_body(state.body))
|
||||
for state in states if state.snake_id in gsids
|
||||
],
|
||||
)
|
||||
|
||||
def _load_records(self, connection:sqlite3.Connection, game_id:str, gid:int) -> tuple[list[SnakeIdentity], list[TurnRecord], list[SnakeState]]:
|
||||
identities = [
|
||||
SnakeIdentity(row["snake_id"], row["snake_name"], bool(row["is_you"]), self._from_json(row["customizations_json"]) or {})
|
||||
for row in connection.execute("""
|
||||
SELECT snake_id, snake_name, is_you, customizations_json
|
||||
FROM game_snakes WHERE game_id = ?
|
||||
""", (game_id,))
|
||||
]
|
||||
turns = [
|
||||
TurnRecord(
|
||||
turn=row["turn"], observed_at=row["observed_at"], move=decode_move(row["my_move"]),
|
||||
thinking=self._from_json(row["thinking_json"]),
|
||||
food=decode_points(row["food"]), hazards=decode_points(row["hazards"]),
|
||||
)
|
||||
for row in connection.execute("""
|
||||
SELECT turn, observed_at, my_move, thinking_json, food, hazards
|
||||
FROM game_turns WHERE gid = ? ORDER BY turn
|
||||
""", (gid,))
|
||||
]
|
||||
states = [
|
||||
SnakeState(row["snake_id"], row["turn"], row["health"], row["latency"], decode_body(row["body"]))
|
||||
for row in connection.execute("""
|
||||
SELECT gs.snake_id, s.turn, s.health, s.latency, s.body
|
||||
FROM game_snakes AS gs
|
||||
JOIN snake_states AS s ON s.gsid = gs.gsid
|
||||
WHERE gs.game_id = ?
|
||||
""", (game_id,))
|
||||
]
|
||||
blob = connection.execute("SELECT format, data FROM game_replays WHERE gid = ?", (gid,)).fetchone()
|
||||
if blob is not None:
|
||||
packed_turns, packed_states = unpack_replay(blob["data"], blob["format"])
|
||||
turns, states = merge_records(packed_turns, packed_states, turns, states)
|
||||
return identities, turns, states
|
||||
|
||||
def _store_replay(self, connection:sqlite3.Connection, game_id:str, gid:int, turns:list[TurnRecord], states:list[SnakeState]) -> None:
|
||||
connection.execute("""
|
||||
INSERT INTO game_replays (gid, format, turn_count, data) VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(gid) DO UPDATE SET
|
||||
format = excluded.format, turn_count = excluded.turn_count, data = excluded.data
|
||||
""", (gid, REPLAY_FORMAT, len(turns), pack_replay(turns, states)))
|
||||
connection.execute("DELETE FROM game_turns WHERE gid = ?", (gid,))
|
||||
connection.execute(
|
||||
"DELETE FROM snake_states WHERE gsid IN (SELECT gsid FROM game_snakes WHERE game_id = ?)",
|
||||
(game_id,),
|
||||
)
|
||||
|
||||
def _pack_game(self, connection:sqlite3.Connection, game_id:str) -> None:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
row = connection.execute("SELECT gid FROM games WHERE game_id = ?", (game_id,)).fetchone()
|
||||
if row is not None and connection.execute(
|
||||
"SELECT 1 FROM game_turns WHERE gid = ? LIMIT 1", (row["gid"],),
|
||||
).fetchone():
|
||||
_, turns, states = self._load_records(connection, game_id, row["gid"])
|
||||
self._store_replay(connection, game_id, row["gid"], turns, states)
|
||||
connection.execute("COMMIT")
|
||||
except Exception:
|
||||
connection.execute("ROLLBACK")
|
||||
raise
|
||||
|
||||
# ── sync write methods ─────────────────────────────────────────────────────
|
||||
|
||||
@@ -216,13 +220,15 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
|
||||
ruleset = game.get("ruleset", {})
|
||||
game_type = self._derive_game_type(board, ruleset)
|
||||
|
||||
with self._connect() as connection:
|
||||
connection = self._connect()
|
||||
try:
|
||||
connection.execute("""
|
||||
INSERT INTO games (
|
||||
game_id, started_at, width, height, source, map_name,
|
||||
ruleset_name, ruleset_version, your_snake_id, your_snake_name,
|
||||
your_snake_type, your_snake_version, game_type, status
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'running')
|
||||
your_snake_type, your_snake_version, game_type, status, gid
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'running',
|
||||
(SELECT COALESCE(MAX(gid), 0) + 1 FROM games))
|
||||
ON CONFLICT(game_id) DO UPDATE SET
|
||||
width = excluded.width,
|
||||
height = excluded.height,
|
||||
@@ -238,166 +244,71 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
|
||||
status = 'running'
|
||||
""",
|
||||
(
|
||||
game.get("id"),
|
||||
self._utc_now(),
|
||||
board.get("width"),
|
||||
board.get("height"),
|
||||
game.get("source"),
|
||||
game.get("map"),
|
||||
ruleset.get("name"),
|
||||
ruleset.get("version"),
|
||||
you.get("id"),
|
||||
you.get("name"),
|
||||
snake_type,
|
||||
snake_version,
|
||||
game_type,
|
||||
game.get("id"), self._utc_now(), board.get("width"), board.get("height"),
|
||||
game.get("source"), game.get("map"), ruleset.get("name"), ruleset.get("version"),
|
||||
you.get("id"), you.get("name"), snake_type, snake_version, game_type,
|
||||
),
|
||||
)
|
||||
connection.execute("PRAGMA wal_checkpoint(PASSIVE)")
|
||||
connection.execute("PRAGMA optimize")
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def _record_turn_sync(self, game_state:dict, my_move:str|None, my_thinking:dict|None) -> None:
|
||||
game = game_state.get("game", {})
|
||||
board = game_state.get("board", {})
|
||||
snakes = self._extract_snakes(game_state)
|
||||
you = self._extract_you(game_state)
|
||||
game_id = game.get("id")
|
||||
turn = int(game_state.get("turn", 0))
|
||||
|
||||
board_json, snakes_json, you_json, food_json, hazards_json = compact_turn_json(board)
|
||||
|
||||
with self._connect() as connection:
|
||||
connection.executemany("""
|
||||
INSERT INTO game_snakes (
|
||||
game_id, snake_id, snake_name, is_you, customizations_json
|
||||
) VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(game_id, snake_id) DO UPDATE SET
|
||||
snake_name = excluded.snake_name,
|
||||
is_you = excluded.is_you,
|
||||
customizations_json = excluded.customizations_json
|
||||
""",
|
||||
[
|
||||
(
|
||||
game_id,
|
||||
snake.get("id"),
|
||||
snake.get("name"),
|
||||
1 if snake.get("id") == you.get("id") else 0,
|
||||
self._to_json(snake.get("customizations") or {}),
|
||||
)
|
||||
for snake in snakes if snake.get("id") is not None
|
||||
],
|
||||
)
|
||||
connection.execute("""
|
||||
INSERT INTO turns (
|
||||
game_id, turn, observed_at, my_move, my_thinking_json,
|
||||
board_state_json, snakes_json, you_json, food_json, hazards_json
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(game_id, turn) DO UPDATE SET
|
||||
observed_at = excluded.observed_at,
|
||||
my_move = excluded.my_move,
|
||||
my_thinking_json = excluded.my_thinking_json,
|
||||
board_state_json = excluded.board_state_json,
|
||||
snakes_json = excluded.snakes_json,
|
||||
you_json = excluded.you_json,
|
||||
food_json = excluded.food_json,
|
||||
hazards_json = excluded.hazards_json
|
||||
""",
|
||||
(
|
||||
game_id,
|
||||
turn,
|
||||
self._utc_now(),
|
||||
my_move,
|
||||
self._to_json(my_thinking) if my_thinking is not None else None,
|
||||
self._to_json(board_json),
|
||||
self._to_json(snakes_json),
|
||||
self._to_json(you_json),
|
||||
self._to_json(food_json),
|
||||
self._to_json(hazards_json),
|
||||
),
|
||||
)
|
||||
|
||||
previous_positions: dict[str, tuple[int, int]] = {}
|
||||
if turn > 0:
|
||||
previous_rows = connection.execute("""
|
||||
SELECT snake_id, head_x, head_y
|
||||
FROM snake_turns
|
||||
WHERE game_id = ? AND turn = ?
|
||||
""",
|
||||
(game_id, turn - 1),
|
||||
).fetchall()
|
||||
previous_positions = {
|
||||
row["snake_id"]: (int(row["head_x"]), int(row["head_y"]))
|
||||
for row in previous_rows
|
||||
if row["head_x"] is not None and row["head_y"] is not None
|
||||
}
|
||||
|
||||
you_id = you.get("id")
|
||||
for params in self._build_snake_turn_params(snakes, you_id, game_id, turn, previous_positions):
|
||||
(p_game_id, p_turn, p_snake_id, p_name, p_health, p_length,
|
||||
p_head_x, p_head_y, p_body, p_is_you, p_inferred, p_latency) = params
|
||||
connection.execute("""
|
||||
INSERT INTO snake_turns (
|
||||
game_id, turn, snake_id, snake_name, health, length,
|
||||
head_x, head_y, body_json, is_you, inferred_move, latency
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(game_id, turn, snake_id) DO UPDATE SET
|
||||
snake_name = excluded.snake_name,
|
||||
health = excluded.health,
|
||||
length = excluded.length,
|
||||
head_x = excluded.head_x,
|
||||
head_y = excluded.head_y,
|
||||
body_json = excluded.body_json,
|
||||
is_you = excluded.is_you,
|
||||
inferred_move = excluded.inferred_move,
|
||||
latency = excluded.latency
|
||||
""",
|
||||
(
|
||||
p_game_id, p_turn, p_snake_id, None, p_health, p_length,
|
||||
p_head_x, p_head_y, self._to_json(p_body),
|
||||
0,
|
||||
p_inferred, p_latency,
|
||||
),
|
||||
game_id = game_state.get("game", {}).get("id")
|
||||
connection = self._connect()
|
||||
try:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
running = self._running_game(connection, game_id)
|
||||
record, states, identities, points = records_from_game_state(
|
||||
game_state, my_move, my_thinking, running.points,
|
||||
)
|
||||
|
||||
connection.execute("""
|
||||
UPDATE games
|
||||
SET final_turn = CASE WHEN ? > final_turn THEN ? ELSE final_turn END
|
||||
WHERE game_id = ?
|
||||
""",
|
||||
(turn, turn, game_id),
|
||||
)
|
||||
if any(identity.snake_id not in running.snake_ids for identity in identities):
|
||||
running.snake_ids.update(self._upsert_identities(connection, game_id, identities))
|
||||
self._write_rows(connection, running.gid, running.snake_ids, [record], states)
|
||||
connection.execute(
|
||||
"UPDATE games SET final_turn = MAX(final_turn, ?) WHERE gid = ?",
|
||||
(record.turn, running.gid),
|
||||
)
|
||||
connection.execute("COMMIT")
|
||||
except Exception:
|
||||
connection.execute("ROLLBACK")
|
||||
raise
|
||||
finally:
|
||||
connection.close()
|
||||
running.points = points
|
||||
self._running.put(game_id, running)
|
||||
|
||||
def _record_game_end_sync(self, game_state:dict) -> None:
|
||||
game_id, winner_name, winner_you, turn = self._extract_game_end_params(game_state)
|
||||
|
||||
with self._connect() as connection:
|
||||
connection = self._connect()
|
||||
try:
|
||||
connection.execute("""
|
||||
UPDATE games
|
||||
SET ended_at = ?,
|
||||
winner_name = ?,
|
||||
winner_you = ?,
|
||||
final_turn = CASE WHEN ? > final_turn THEN ? ELSE final_turn END,
|
||||
final_turn = MAX(final_turn, ?),
|
||||
status = 'finished'
|
||||
WHERE game_id = ?
|
||||
""",
|
||||
(
|
||||
self._utc_now(),
|
||||
winner_name,
|
||||
1 if winner_you else 0,
|
||||
turn,
|
||||
turn,
|
||||
game_id,
|
||||
),
|
||||
(self._utc_now(), winner_name, 1 if winner_you else 0, turn, game_id),
|
||||
)
|
||||
self._pack_game(connection, game_id)
|
||||
finally:
|
||||
connection.close()
|
||||
self._running.drop(game_id)
|
||||
|
||||
def _finalize_stale_running_games_sync(self, stale_after_seconds:int=600) -> int:
|
||||
threshold = max(60, int(stale_after_seconds))
|
||||
now_utc = datetime.now(timezone.utc)
|
||||
finalized = 0
|
||||
|
||||
with self._connect() as connection:
|
||||
connection = self._connect()
|
||||
try:
|
||||
rows = connection.execute("""
|
||||
SELECT game_id, started_at, final_turn, your_snake_id
|
||||
SELECT gid, game_id, started_at, final_turn, your_snake_id
|
||||
FROM games
|
||||
WHERE status = 'running'
|
||||
ORDER BY started_at ASC
|
||||
@@ -405,73 +316,47 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
|
||||
|
||||
for row in rows:
|
||||
started_at = self._parse_utc_timestamp(row["started_at"])
|
||||
if started_at is None:
|
||||
continue
|
||||
if (now_utc - started_at).total_seconds() < threshold:
|
||||
if started_at is None or (now_utc - started_at).total_seconds() < threshold:
|
||||
continue
|
||||
|
||||
game_id = row["game_id"]
|
||||
your_snake_id = row["your_snake_id"]
|
||||
final_turn = int(row["final_turn"] or 0)
|
||||
|
||||
snake_rows = connection.execute("""
|
||||
SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name) AS snake_name
|
||||
FROM snake_turns AS st
|
||||
LEFT JOIN game_snakes AS gs
|
||||
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
|
||||
WHERE st.game_id = ? AND st.turn = ?
|
||||
ORDER BY COALESCE(gs.is_you, st.is_you) DESC, snake_name ASC
|
||||
""",
|
||||
(game_id, final_turn),
|
||||
).fetchall()
|
||||
|
||||
if len(snake_rows) == 0:
|
||||
latest_row = connection.execute(
|
||||
"SELECT MAX(turn) AS latest_turn FROM snake_turns WHERE game_id = ?",
|
||||
(game_id,),
|
||||
).fetchone()
|
||||
if latest_row is not None and latest_row["latest_turn"] is not None:
|
||||
final_turn = int(latest_row["latest_turn"])
|
||||
snake_rows = connection.execute("""
|
||||
SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name) AS snake_name
|
||||
FROM snake_turns AS st
|
||||
LEFT JOIN game_snakes AS gs
|
||||
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
|
||||
WHERE st.game_id = ? AND st.turn = ?
|
||||
ORDER BY COALESCE(gs.is_you, st.is_you) DESC, snake_name ASC
|
||||
""",
|
||||
(game_id, final_turn),
|
||||
).fetchall()
|
||||
|
||||
winner_you, survivor_name = self._calculate_survivor(snake_rows, your_snake_id)
|
||||
identities, _, states = self._load_records(connection, game_id, row["gid"])
|
||||
final_turn, survivors = self._survivors(identities, states, int(row["final_turn"] or 0))
|
||||
winner_you, survivor_name = self._calculate_survivor(survivors, row["your_snake_id"])
|
||||
|
||||
result = connection.execute("""
|
||||
UPDATE games
|
||||
SET ended_at = ?,
|
||||
winner_name = ?,
|
||||
winner_you = ?,
|
||||
final_turn = CASE WHEN ? > final_turn THEN ? ELSE final_turn END,
|
||||
final_turn = MAX(final_turn, ?),
|
||||
status = 'finished'
|
||||
WHERE game_id = ? AND status = 'running'
|
||||
""",
|
||||
(
|
||||
self._utc_now(),
|
||||
survivor_name,
|
||||
1 if winner_you else 0,
|
||||
final_turn,
|
||||
final_turn,
|
||||
game_id,
|
||||
),
|
||||
(self._utc_now(), survivor_name, 1 if winner_you else 0, final_turn, game_id),
|
||||
)
|
||||
if result.rowcount > 0:
|
||||
finalized += 1
|
||||
self._pack_game(connection, game_id)
|
||||
self._running.drop(game_id)
|
||||
|
||||
late = connection.execute("""
|
||||
SELECT DISTINCT g.game_id
|
||||
FROM game_turns AS t JOIN games AS g ON g.gid = t.gid
|
||||
WHERE g.status = 'finished'
|
||||
""").fetchall()
|
||||
for late_row in late:
|
||||
self._pack_game(connection, late_row["game_id"])
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
return finalized
|
||||
|
||||
# ── sync read methods ──────────────────────────────────────────────────────
|
||||
|
||||
def _get_summary_sync(self, recent_limit:int=15) -> dict:
|
||||
with self._connect() as connection:
|
||||
connection = self._connect()
|
||||
try:
|
||||
totals = connection.execute("""
|
||||
SELECT
|
||||
COUNT(*) AS total_games,
|
||||
@@ -505,11 +390,14 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
|
||||
""",
|
||||
(max(1, int(recent_limit)),),
|
||||
).fetchall()
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
return self._build_summary_output(totals, by_type, recent)
|
||||
|
||||
def _list_games_sync(self, limit:int=50) -> list[dict]:
|
||||
with self._connect() as connection:
|
||||
connection = self._connect()
|
||||
try:
|
||||
rows = connection.execute("""
|
||||
SELECT game_id, started_at, ended_at, map_name, source, ruleset_name, game_type,
|
||||
your_snake_name, your_snake_type, your_snake_version,
|
||||
@@ -521,52 +409,75 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
|
||||
""",
|
||||
(max(1, int(limit)),),
|
||||
).fetchall()
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
return self._build_game_list_output(rows)
|
||||
|
||||
def _get_game_replay_sync(self, game_id:str) -> dict|None:
|
||||
with self._connect() as connection:
|
||||
game_row = connection.execute("""
|
||||
SELECT game_id, started_at, ended_at, width, height, source, map_name,
|
||||
ruleset_name, ruleset_version, game_type, your_snake_id, your_snake_name,
|
||||
your_snake_type, your_snake_version,
|
||||
winner_name, winner_you, final_turn, status
|
||||
FROM games
|
||||
WHERE game_id = ? AND has_replay = 1
|
||||
""",
|
||||
(game_id,),
|
||||
).fetchone()
|
||||
|
||||
connection = self._connect()
|
||||
try:
|
||||
game_row = connection.execute(_GAME_SELECT + " WHERE game_id = ? AND has_replay = 1", (game_id,)).fetchone()
|
||||
if game_row is None:
|
||||
return None
|
||||
identities, turns, states = self._load_records(connection, game_id, game_row["gid"])
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
turn_rows = connection.execute("""
|
||||
SELECT turn, observed_at, my_move, my_thinking_json,
|
||||
board_state_json, food_json, hazards_json, you_json
|
||||
FROM turns
|
||||
WHERE game_id = ?
|
||||
ORDER BY turn ASC
|
||||
""",
|
||||
(game_id,),
|
||||
).fetchall()
|
||||
return self._build_game_replay_output(game_row, identities, turns, states)
|
||||
|
||||
snake_rows = connection.execute("""
|
||||
SELECT st.turn, st.snake_id,
|
||||
COALESCE(gs.snake_name, st.snake_name) AS snake_name,
|
||||
st.health, st.length, st.head_x, st.head_y, st.body_json,
|
||||
COALESCE(gs.is_you, st.is_you) AS is_you,
|
||||
COALESCE(gs.customizations_json, '{}') AS customizations_json,
|
||||
st.inferred_move, st.latency
|
||||
FROM snake_turns AS st
|
||||
LEFT JOIN game_snakes AS gs
|
||||
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
|
||||
WHERE st.game_id = ?
|
||||
ORDER BY st.turn ASC, is_you DESC, snake_name ASC
|
||||
""",
|
||||
(game_id,),
|
||||
).fetchall()
|
||||
# ── bundles (migration / merge) ────────────────────────────────────────────
|
||||
|
||||
return self._build_game_replay_output(game_row, turn_rows, snake_rows, decode_json=self._from_json)
|
||||
def list_game_ids_sync(self) -> list[str]:
|
||||
connection = self._connect()
|
||||
try:
|
||||
return [row[0] for row in connection.execute("SELECT game_id FROM games ORDER BY started_at, game_id")]
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def read_game_bundle_sync(self, game_id:str) -> GameBundle|None:
|
||||
connection = self._connect()
|
||||
try:
|
||||
row = connection.execute(_GAME_SELECT + " WHERE game_id = ?", (game_id,)).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
identities, turns, states = self._load_records(connection, game_id, row["gid"])
|
||||
finally:
|
||||
connection.close()
|
||||
return GameBundle(_game_from_row(row), identities, turns, states)
|
||||
|
||||
def import_game_bundle_sync(self, bundle:GameBundle) -> None:
|
||||
game = normalize_game(bundle.game)
|
||||
columns = [column for column in GAME_COLUMNS if column != "quality_reasons"] + ["quality_reasons_json"]
|
||||
values = [game[column] for column in GAME_COLUMNS if column != "quality_reasons"]
|
||||
values.append(self._to_json(game["quality_reasons"]) if game["quality_reasons"] is not None else None)
|
||||
values = [self._ts_to_str(value) if isinstance(value, datetime) else value for value in values]
|
||||
updates = ", ".join(f"{column} = excluded.{column}" for column in columns[1:])
|
||||
|
||||
connection = self._connect()
|
||||
try:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
connection.execute(
|
||||
f"INSERT INTO games ({', '.join(columns)}, gid) "
|
||||
f"VALUES ({', '.join('?' for _ in columns)}, (SELECT COALESCE(MAX(gid), 0) + 1 FROM games)) "
|
||||
f"ON CONFLICT(game_id) DO UPDATE SET {updates}",
|
||||
values,
|
||||
)
|
||||
gid = connection.execute("SELECT gid FROM games WHERE game_id = ?", (game["game_id"],)).fetchone()["gid"]
|
||||
gsids = self._upsert_identities(connection, game["game_id"], bundle.snakes)
|
||||
if bundle.turns and game["has_replay"]:
|
||||
if game["status"] == "finished":
|
||||
self._store_replay(connection, game["game_id"], gid, bundle.turns, bundle.states)
|
||||
else:
|
||||
self._write_rows(connection, gid, gsids, bundle.turns, bundle.states)
|
||||
connection.execute("COMMIT")
|
||||
except Exception:
|
||||
connection.execute("ROLLBACK")
|
||||
raise
|
||||
finally:
|
||||
connection.close()
|
||||
self._running.drop(game["game_id"])
|
||||
|
||||
# ── public async interface ─────────────────────────────────────────────────
|
||||
|
||||
@@ -591,5 +502,14 @@ class SqliteGameplayBackend(GameplayBackendTemplate):
|
||||
async def get_game_replay(self, game_id:str) -> dict|None:
|
||||
return await asyncio.to_thread(self._get_game_replay_sync, game_id)
|
||||
|
||||
async def list_game_ids(self) -> list[str]:
|
||||
return await asyncio.to_thread(self.list_game_ids_sync)
|
||||
|
||||
async def read_game_bundle(self, game_id:str) -> GameBundle|None:
|
||||
return await asyncio.to_thread(self.read_game_bundle_sync, game_id)
|
||||
|
||||
async def import_game_bundle(self, bundle:GameBundle) -> None:
|
||||
await asyncio.to_thread(self.import_game_bundle_sync, bundle)
|
||||
|
||||
async def close(self) -> None:
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user