From 39a7a53873bbcc131e93630529ebf0d0660b54c0 Mon Sep 17 00:00:00 2001 From: Daniel Dolezal Date: Thu, 5 Nov 2026 00:02:56 +0100 Subject: [PATCH] feat(database): store gameplay replays in compact form - Add compact storage package: codec, records, hydrate, replay blobs, cache. - Running games write game_turns and snake_states keyed by integer ids. - Encode moves as 1-byte codes and bodies as head plus step codes. - Store food and hazards only when they changed since the previous turn. - Strip reasoning fields that duplicate stored values and restore on read. - Pack finished games into one zstd blob per game in game_replays. - Merge late turns into packed blobs during the stale-game sweep. - Extract SQLite and PostgreSQL schemas into dedicated modules. - Add read/import game bundle and list_game_ids to the backend template. - Add legacy readers and converter for the old turns/snake_turns layout. - Add migrate_gameplay_to_compact.py with in-place or cross-DB migration. - Verify every migrated game and keep legacy rows on failure. - Support --since, --minimum-quality and guarded --drop-legacy. - Add just recipe migrate-gameplay-postgres. - Rewrite merge script to copy game by game via backend bundles. - Move benchmark state loading into benchmark_states for both backends. - Update PostgreSQL cleanup script to the new replay tables. - Drop sqlite-zstd build stages from the Dockerfile. - Remove old migration scripts, normalized_turn and their tests. - Add tests for compact storage, schema, legacy migration and PostgreSQL. - Document the compact layout and migration workflow in the README. --- Dockerfile | 32 - README.md | 77 +- justfile | 11 + scripts/benchmark_snakes_from_db.py | 52 +- scripts/cleanup_postgresql_gameplay.py | 34 +- scripts/merge_gameplay_databases.py | 254 ++---- scripts/migrate_gameplay_database.py | 355 --------- scripts/migrate_gameplay_to_compact.py | 127 +++ scripts/migrate_sqlite_to_postgresql.py | 207 ----- .../backend/PostgresqlGameplayBackend.py | 741 ++++++------------ .../database/backend/SqliteGameplayBackend.py | 688 +++++++--------- server/database/backend/Template.py | 111 ++- server/database/backend/postgresql_schema.py | 106 +++ server/database/backend/sqlite_schema.py | 114 +++ server/database/benchmark_states.py | 129 +-- server/database/compact/__init__.py | 1 + server/database/compact/cache.py | 33 + server/database/compact/codec.py | 146 ++++ server/database/compact/games.py | 28 + server/database/compact/hydrate.py | 100 +++ server/database/compact/quality.py | 23 + server/database/compact/records.py | 136 ++++ server/database/compact/replay_blob.py | 63 ++ server/database/compact/thinking.py | 57 ++ server/database/legacy/__init__.py | 1 + server/database/legacy/convert.py | 98 +++ server/database/legacy/migrate.py | 84 ++ server/database/legacy/reader_postgresql.py | 69 ++ server/database/legacy/reader_sqlite.py | 100 +++ server/database/normalized_turn.py | 87 -- tests/test_BenchmarkStates.py | 49 +- tests/test_CompactStorage.py | 124 +++ tests/test_GameplayDatabase.py | 177 +++-- tests/test_GameplaySchema.py | 120 +++ tests/test_LegacyMigration.py | 176 +++++ tests/test_MergeGameplayDatabases.py | 144 ++-- tests/test_MigrateGameplayDatabase.py | 137 ---- tests/test_MigrateSqliteToPostgresql.py | 34 - tests/test_PostgresqlGameplayBackend.py | 90 +++ tests/test_PostgresqlGameplayMigration.py | 65 -- tests/test_ZstdCompression.py | 92 --- 41 files changed, 2779 insertions(+), 2493 deletions(-) delete mode 100644 scripts/migrate_gameplay_database.py create mode 100644 scripts/migrate_gameplay_to_compact.py delete mode 100644 scripts/migrate_sqlite_to_postgresql.py create mode 100644 server/database/backend/postgresql_schema.py create mode 100644 server/database/backend/sqlite_schema.py create mode 100644 server/database/compact/__init__.py create mode 100644 server/database/compact/cache.py create mode 100644 server/database/compact/codec.py create mode 100644 server/database/compact/games.py create mode 100644 server/database/compact/hydrate.py create mode 100644 server/database/compact/quality.py create mode 100644 server/database/compact/records.py create mode 100644 server/database/compact/replay_blob.py create mode 100644 server/database/compact/thinking.py create mode 100644 server/database/legacy/__init__.py create mode 100644 server/database/legacy/convert.py create mode 100644 server/database/legacy/migrate.py create mode 100644 server/database/legacy/reader_postgresql.py create mode 100644 server/database/legacy/reader_sqlite.py delete mode 100644 server/database/normalized_turn.py create mode 100644 tests/test_CompactStorage.py create mode 100644 tests/test_GameplaySchema.py create mode 100644 tests/test_LegacyMigration.py delete mode 100644 tests/test_MigrateGameplayDatabase.py delete mode 100644 tests/test_MigrateSqliteToPostgresql.py create mode 100644 tests/test_PostgresqlGameplayBackend.py delete mode 100644 tests/test_PostgresqlGameplayMigration.py delete mode 100644 tests/test_ZstdCompression.py diff --git a/Dockerfile b/Dockerfile index 7de1fce..63a18a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,37 +1,5 @@ -# Stage 1 — compile sqlite-zstd + SQLite 3.49.1 -FROM ghcr.io/astral-sh/uv:python3.15-trixie-slim AS sqlite-zstd-builder - -RUN apt-get update && apt-get install -y --no-install-recommends \ - curl unzip git libzstd-dev pkg-config build-essential ca-certificates && \ - rm -rf /var/lib/apt/lists/* - -# Compile SQLite 3.49.1 as a shared library so the runtime image can use it -RUN curl -fsSL https://www.sqlite.org/2025/sqlite-amalgamation-3490100.zip -o /tmp/sqlite.zip && \ - unzip /tmp/sqlite.zip -d /tmp/ && \ - cd /tmp/sqlite-amalgamation-3490100 && \ - gcc -O2 -shared -fPIC -o /usr/local/lib/libsqlite3.so.0 sqlite3.c -ldl -lpthread - -# Install Rust and compile sqlite-zstd (uses bundled SQLite 3.49.1 — no patching needed) -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal -ENV PATH="/root/.cargo/bin:${PATH}" - -RUN git clone --depth=1 --branch v0.3.5 https://github.com/phiresky/sqlite-zstd.git /tmp/sqlite-zstd && \ - cd /tmp/sqlite-zstd && \ - cargo build --release --features build_extension && \ - cp target/release/libsqlite_zstd.so /usr/local/lib/libsqlite_zstd.so - -# Stage 2 — runtime image FROM ghcr.io/astral-sh/uv:python3.15-trixie-slim -RUN apt-get update && apt-get install -y --no-install-recommends libzstd1 && \ - rm -rf /var/lib/apt/lists/* - -# Replace system SQLite 3.46.1 with 3.49.1 so it matches the extension -COPY --from=sqlite-zstd-builder /usr/local/lib/libsqlite3.so.0 /usr/local/lib/libsqlite3.so.0 -RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/local.conf && ldconfig - -COPY --from=sqlite-zstd-builder /usr/local/lib/libsqlite_zstd.so /usr/local/lib/libsqlite_zstd.so - # Install app COPY . /app WORKDIR /app diff --git a/README.md b/README.md index 7e27751..0866457 100644 --- a/README.md +++ b/README.md @@ -107,8 +107,9 @@ python scripts/benchmark_snakes_from_db.py \ --samples 100 ``` -The benchmark opens SQLite read-only and reports mean, median, p95, and maximum -move latency. Increase `--samples` for a broader but slower comparison. +The benchmark samples every `--stride`-th stored turn from the newest games +(SQLite is opened read-only; a `postgresql://` DSN works too) and reports mean, +median, p95, and maximum move latency. Increase `--samples` for a broader but slower comparison. Run the deterministic CI-friendly arena benchmark without a gameplay database: ```sh @@ -135,39 +136,52 @@ both snake servers with gameplay persistence disabled, adds the engine identity header required by the API, and shuts them down when finished. ## Compact gameplay database -New gameplay turns use normalized storage: the turn row stores food, hazards, -move, and thinking data once; snake identity is stored once per game in -`game_snakes`; and changing snake state/body data lives in `snake_turns`. Replay -loading rebuilds the normal Battlesnake board payload. +Gameplay storage keeps game results in `games` and snake identity (name, +customizations) once per game in `game_snakes`. Replay data is stored compactly: -Create and verify a separate compact copy of an existing SQLite database. By -default, replay-heavy rows are retained for games rated `medium` or `high` by +- **Running games** write one `game_turns` row per turn and one `snake_states` + row per snake and turn, keyed by small integer ids (`games.gid`, + `game_snakes.gsid`) instead of repeating text ids. Moves are 1-byte codes, + latency is an integer, bodies are a head position plus one step code per + segment, and food/hazards are only stored when they changed since the previous + turn. Reasoning fields that equal stored values (`turn`, `move`, `head`, + `health`, `length`, `food_count`, `board_size`, `snakes`) are stripped and + restored exactly on read. +- **Finished games** are packed into one zstd-compressed blob per game in + `game_replays`, and their rows are deleted. Turns arriving after a game was + packed are merged into the blob by the stale-game sweep. + +Replay loading rebuilds the normal Battlesnake board payload, so the dashboard +is unchanged. Head position, length, and inferred moves are derived from the +stored bodies. + +### Migrate the legacy layout +Databases written by older versions store replays in `turns`/`snake_turns`. +Migrate them in place (SQLite path or `postgresql://` DSN). Every game is read +back and compared after writing; failed games are reported and keep their +legacy rows: +```sh +python scripts/migrate_gameplay_to_compact.py --source /path/to/gameplay.sqlite3 +python scripts/migrate_gameplay_to_compact.py --source /path/to/gameplay.sqlite3 --drop-legacy +``` + +`--destination` writes to a different database instead, for example from +SQLite into PostgreSQL. `--minimum-quality medium` re-rates replays by structural completeness, valid moves, thinking coverage, game length, move -diversity, opponent data, and terminal outcome. All game-result rows remain -stored, so historical win/loss rates stay persistent when low-quality replay -data is removed. -```sh -python scripts/migrate_gameplay_database.py \ - --source /path/to/gameplay.sqlite3 \ - --destination /path/to/gameplay.compact.sqlite3 \ - --minimum-quality medium -``` +diversity, opponent data, and terminal outcome, and keeps only replays of that +tier or better. All game-result rows remain stored, so historical win/loss +rates stay persistent. `--drop-legacy` only drops the old tables when every +legacy replay exists in compact form. -After reviewing the compact copy, `--replace` renames the original to a -timestamped backup and puts the verified compact database at the original path. -Stop all writers before using it: -```sh -python scripts/migrate_gameplay_database.py \ - --source /path/to/gameplay.sqlite3 \ - --replace -``` - -The migration never modifies the source in place. It verifies row counts and -runs SQLite's `integrity_check` before any replacement. +For a live PostgreSQL database: run the migration while the old version keeps +recording, deploy the new version, run the migration again with +`--since ` to pick up games recorded meanwhile, +then run `--drop-legacy`. A PostgreSQL server configured with a SQLite +`GAMEPLAY_DB_PATH` imports that legacy file automatically on startup. ### Record new games while cleanup runs Point the running server at a temporary delta database while the old database -is being compacted. After stopping the writer and flushing the delta database, +is being migrated or cleaned. After stopping the writer and flushing the delta database, merge it into the cleaned copy: ```sh python scripts/merge_gameplay_databases.py \ @@ -177,8 +191,9 @@ python scripts/merge_gameplay_databases.py \ --minimum-quality medium ``` -The merger keeps all game results, quality-rates delta games, regenerates -numeric turn IDs, and verifies row counts, foreign keys, and database integrity. +The merger copies game by game, keeps all game results, quality-rates delta +games, assigns fresh integer ids, and verifies game, snake, turn, and snake-state +counts. Identical game IDs are skipped; conflicting duplicates abort the merge. After reviewing the result, `--replace-base` backs up and replaces the cleaned base. Stop the delta writer before the final merge and file swap. diff --git a/justfile b/justfile index 4ef4fb1..ebb05f2 100644 --- a/justfile +++ b/justfile @@ -90,6 +90,17 @@ cleanup-gameplay-postgres older_than_days="30" execute="false" vacuum="false": if [ "{{vacuum}}" = "true" ]; then args+=(--vacuum); fi PYTHONPATH="{{justfile_directory()}}" uv run python "{{justfile_directory()}}/scripts/cleanup_postgresql_gameplay.py" "${args[@]}" +# Migrate the legacy turns/snake_turns layout to compact storage (in place) +migrate-gameplay-postgres since="" drop_legacy="false": + #!/usr/bin/env bash + set -euo pipefail + + : "${GAMEPLAY_DB_PG_DSN:?Set GAMEPLAY_DB_PG_DSN in .env or the environment}" + args=(--source "$GAMEPLAY_DB_PG_DSN") + if [ -n "{{since}}" ]; then args+=(--since "{{since}}"); fi + if [ "{{drop_legacy}}" = "true" ]; then args+=(--drop-legacy); fi + PYTHONPATH="{{justfile_directory()}}" uv run python "{{justfile_directory()}}/scripts/migrate_gameplay_to_compact.py" "${args[@]}" + bench-snake-tournament games="20" gametype="standard" map="standard" output="": #!/usr/bin/env bash set -euo pipefail diff --git a/scripts/benchmark_snakes_from_db.py b/scripts/benchmark_snakes_from_db.py index 8307730..bf804ba 100644 --- a/scripts/benchmark_snakes_from_db.py +++ b/scripts/benchmark_snakes_from_db.py @@ -1,17 +1,14 @@ #!/usr/bin/env python3 -"""Benchmark snake move latency against sampled states from gameplay SQLite.""" +"""Benchmark snake move latency against sampled states from a gameplay database.""" import argparse from pathlib import Path -import sqlite3 from statistics import mean, median import sys from time import perf_counter sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from server.database.benchmark_states import ( - _build_state, is_postgresql_source, load_postgresql_states, -) +from server.database.benchmark_states import load_states lazy from server.GameBoard import GameBoard lazy from snakes import SnakeBuilder @@ -20,51 +17,6 @@ def percentile(values: list[float], quantile: float) -> float: index = min(len(ordered) - 1, round((len(ordered) - 1) * quantile)) return ordered[index] -def load_states(db_path: str, samples: int, stride: int) -> list[tuple[dict, dict]]: - if is_postgresql_source(db_path): - return load_postgresql_states(db_path, samples, stride) - - connection = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) - connection.execute("PRAGMA query_only = ON") - max_id = int(connection.execute("SELECT max(id) FROM turns").fetchone()[0] or 0) - if max_id == 0: - return [] - - states: list[tuple[dict, dict]] = [] - next_id = max(1, max_id - (samples - 1) * stride) - query = """ - SELECT t.id, t.board_state_json, t.you_json, t.food_json, t.hazards_json, - g.your_snake_id, g.your_snake_name, g.width, g.height, - g.game_id, g.source, g.map_name, - g.ruleset_name, g.ruleset_version, t.turn - FROM turns AS t - JOIN games AS g ON g.game_id = t.game_id - WHERE t.id >= ? - ORDER BY t.id - LIMIT 1 - """ - snake_query = """ - SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name), - st.health, st.length, st.head_x, st.head_y, st.body_json, - COALESCE(gs.customizations_json, '{}') - FROM snake_turns AS st - LEFT JOIN game_snakes AS gs - ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id - WHERE st.game_id = ? AND st.turn = ? - ORDER BY st.id - """ - while len(states) < samples and next_id <= max_id: - row = connection.execute(query, (next_id,)).fetchone() - if row is None: - break - snake_rows = connection.execute(snake_query, (row[9], row[14])).fetchall() - state = _build_state(row, snake_rows) - if state is not None: - states.append(state) - next_id = int(row[0]) + stride - connection.close() - return states - def benchmark(snake_name: str, states: list[tuple[dict, dict]], repeat: int) -> dict: durations: list[float] = [] moves = 0 diff --git a/scripts/cleanup_postgresql_gameplay.py b/scripts/cleanup_postgresql_gameplay.py index a1091df..2e5a9c1 100644 --- a/scripts/cleanup_postgresql_gameplay.py +++ b/scripts/cleanup_postgresql_gameplay.py @@ -21,20 +21,20 @@ async def cleanup(dsn: str, older_than_days: int, dry_run: bool, vacuum: bool) - AND COALESCE(ended_at, started_at) < $1 """, cutoff)) rows = await connection.fetchrow(""" + WITH candidates AS ( + SELECT game_id, gid FROM games + WHERE has_replay + AND quality_status = 'low_quality' + AND COALESCE(ended_at, started_at) < $1 + ) SELECT - (SELECT count(*) FROM turns t JOIN games g USING (game_id) - WHERE g.has_replay AND g.quality_status = 'low_quality' - AND COALESCE(g.ended_at, g.started_at) < $1) AS turns, - (SELECT count(*) FROM snake_turns t JOIN games g USING (game_id) - WHERE g.has_replay AND g.quality_status = 'low_quality' - AND COALESCE(g.ended_at, g.started_at) < $1) AS snake_turns, - (SELECT count(*) FROM game_snakes t JOIN games g USING (game_id) - WHERE g.has_replay AND g.quality_status = 'low_quality' - AND COALESCE(g.ended_at, g.started_at) < $1) AS game_snakes + (SELECT count(*) FROM game_replays r JOIN candidates c USING (gid)) AS replays, + (SELECT count(*) FROM game_turns t JOIN candidates c USING (gid)) AS turns, + (SELECT count(*) FROM game_snakes s JOIN candidates c USING (game_id)) AS game_snakes """, cutoff) print( f"candidates before {cutoff.isoformat()}: games={candidates:,}, " - f"turns={rows['turns']:,}, snake_turns={rows['snake_turns']:,}, " + f"replays={rows['replays']:,}, running turns={rows['turns']:,}, " f"game_snakes={rows['game_snakes']:,}" ) if dry_run or candidates == 0: @@ -50,8 +50,13 @@ async def cleanup(dsn: str, older_than_days: int, dry_run: bool, vacuum: bool) - FOR UPDATE """, cutoff) ids = [row["game_id"] for row in game_ids] - await connection.execute("DELETE FROM snake_turns WHERE game_id = ANY($1::text[])", ids) - await connection.execute("DELETE FROM turns WHERE game_id = ANY($1::text[])", ids) + # snake_states rows cascade from game_snakes. + await connection.execute(""" + DELETE FROM game_replays WHERE gid IN (SELECT gid FROM games WHERE game_id = ANY($1::text[])) + """, ids) + await connection.execute(""" + DELETE FROM game_turns WHERE gid IN (SELECT gid FROM games WHERE game_id = ANY($1::text[])) + """, ids) await connection.execute("DELETE FROM game_snakes WHERE game_id = ANY($1::text[])", ids) await connection.execute(""" UPDATE games @@ -62,8 +67,9 @@ async def cleanup(dsn: str, older_than_days: int, dry_run: bool, vacuum: bool) - if vacuum: await connection.execute("VACUUM (ANALYZE) games") await connection.execute("VACUUM (ANALYZE) game_snakes") - await connection.execute("VACUUM (ANALYZE) turns") - await connection.execute("VACUUM (ANALYZE) snake_turns") + await connection.execute("VACUUM (ANALYZE) game_replays") + await connection.execute("VACUUM (ANALYZE) game_turns") + await connection.execute("VACUUM (ANALYZE) snake_states") print("vacuum/analyze complete") finally: await connection.close() diff --git a/scripts/merge_gameplay_databases.py b/scripts/merge_gameplay_databases.py index ee04fb8..560ec52 100755 --- a/scripts/merge_gameplay_databases.py +++ b/scripts/merge_gameplay_databases.py @@ -1,40 +1,23 @@ #!/usr/bin/env python3 -"""Safely merge a cleaned gameplay database with a temporary delta database. +"""Safely merge a gameplay SQLite database with a temporary delta database. -The base and delta are opened read-only. A new destination is created, numeric -row IDs are regenerated, delta games are quality-rated, and all result rows are -kept even when their replay is excluded. The base can be replaced only after -verification and a timestamped backup. +Base and delta are opened read-only and copied game by game into a new +destination, which assigns fresh integer ids. Delta games are quality-rated; +all game results are kept even when their replay is excluded. The base can be +replaced only after verification and a timestamped backup. """ import argparse from collections import Counter from datetime import datetime, timezone -import json from pathlib import Path -import sqlite3 import sys from time import perf_counter sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from scripts.migrate_gameplay_database import ( - analyze_quality, column_or_null, create_destination, object_columns, - open_source, retained_game_ids, -) -from server.database.game_quality import quality_meets_minimum +from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend +from server.database.legacy.migrate import apply_quality -GAME_COLUMNS = ( - "game_id", "started_at", "ended_at", "width", "height", "source", - "map_name", "ruleset_name", "ruleset_version", "your_snake_id", - "your_snake_name", "your_snake_type", "your_snake_version", "game_type", - "winner_name", "winner_you", "final_turn", "status", "has_replay", - "quality_status", "quality_score", "quality_tier", "quality_reasons_json", -) -GAME_DEFAULTS = { - "winner_you": "0 AS winner_you", "final_turn": "0 AS final_turn", - "status": "'running' AS status", "has_replay": "1 AS has_replay", - "quality_status": "'retained' AS quality_status", -} CORE_GAME_FIELDS = ( "game_id", "width", "height", "source", "map_name", "ruleset_name", "ruleset_version", "your_snake_id", "your_snake_name", "your_snake_type", @@ -42,26 +25,13 @@ CORE_GAME_FIELDS = ( "final_turn", "status", ) -def select_expression(columns:set[str], name:str) -> str: - if name in columns: - return name - return GAME_DEFAULTS.get(name, f"NULL AS {name}") - -def read_games(connection:sqlite3.Connection) -> dict[str, sqlite3.Row]: - columns = object_columns(connection, "games") - selected = ", ".join(select_expression(columns, name) for name in GAME_COLUMNS) - return { - row["game_id"]: row - for row in connection.execute(f"SELECT {selected} FROM games") - } - -def duplicate_ids(base_games:dict[str, sqlite3.Row], delta_games:dict[str, sqlite3.Row]) -> set[str]: - duplicates = set(base_games) & set(delta_games) +def duplicate_ids(base:SqliteGameplayBackend, delta:SqliteGameplayBackend, base_ids:list[str], delta_ids:list[str]) -> set[str]: + duplicates = set(base_ids) & set(delta_ids) conflicts = [] for game_id in duplicates: - base_signature = tuple(base_games[game_id][name] for name in CORE_GAME_FIELDS) - delta_signature = tuple(delta_games[game_id][name] for name in CORE_GAME_FIELDS) - if base_signature != delta_signature: + base_game = base.read_game_bundle_sync(game_id).game + delta_game = delta.read_game_bundle_sync(game_id).game + if any(base_game[name] != delta_game[name] for name in CORE_GAME_FIELDS): conflicts.append(game_id) if conflicts: examples = ", ".join(sorted(conflicts)[:5]) @@ -71,131 +41,32 @@ def duplicate_ids(base_games:dict[str, sqlite3.Row], delta_games:dict[str, sqlit ) return duplicates -def insert_games( - destination:sqlite3.Connection, - rows:dict[str, sqlite3.Row], - excluded:set[str], - quality:dict[str, object]|None=None, - minimum_quality:str="medium", -) -> tuple[int, set[str], Counter]: - placeholders = ",".join("?" for _ in GAME_COLUMNS) - sql = f"INSERT INTO games ({','.join(GAME_COLUMNS)}) VALUES ({placeholders})" - values = [] - replay_ids:set[str] = set() +def copy_games(source:SqliteGameplayBackend, destination:SqliteGameplayBackend, game_ids:list[str], minimum_quality:str|None=None) -> tuple[Counter, Counter]: + counts:Counter = Counter() tiers:Counter = Counter() + for game_id in game_ids: + bundle = source.read_game_bundle_sync(game_id) + if minimum_quality is not None: + apply_quality(bundle, minimum_quality) + tiers[bundle.game["quality_tier"]] += 1 + destination.import_game_bundle_sync(bundle) + counts["games"] += 1 + counts["game_snakes"] += len(bundle.snakes) + counts["turns"] += len(bundle.turns) + counts["snake_states"] += len(bundle.states) + return counts, tiers - for game_id, row in rows.items(): - if game_id in excluded: - continue - output = [row[name] for name in GAME_COLUMNS] - if quality is not None: - result = quality[game_id] - keep_replay = quality_meets_minimum(result.tier, minimum_quality) - replacements = { - "has_replay": int(keep_replay), - "quality_status": "retained" if keep_replay else "low_quality", - "quality_score": result.score, - "quality_tier": result.tier, - "quality_reasons_json": json.dumps(result.reasons, separators=(",", ":")), - } - output = [replacements.get(name, row[name]) for name in GAME_COLUMNS] - tiers[result.tier] += 1 - if keep_replay: - replay_ids.add(game_id) - elif bool(row["has_replay"]): - replay_ids.add(game_id) - values.append(tuple(output)) - - destination.executemany(sql, values) - return len(values), replay_ids, tiers - -def copy_game_snakes(source:sqlite3.Connection, destination:sqlite3.Connection, allowed:set[str], batch_size:int) -> int: - has_table = source.execute( - "SELECT 1 FROM sqlite_master WHERE type='table' AND name='game_snakes'" - ).fetchone() - if has_table: - columns = object_columns(source, "game_snakes") - customizations = ( - "COALESCE(customizations_json, '{}') AS customizations_json" - if "customizations_json" in columns else "'{}' AS customizations_json" - ) - cursor = source.execute(f""" - SELECT game_id, snake_id, snake_name, is_you, {customizations} - FROM game_snakes ORDER BY game_id, snake_id - """) - else: - cursor = source.execute(""" - SELECT game_id, snake_id, MAX(snake_name), MAX(is_you), - '{}' AS customizations_json - FROM snake_turns GROUP BY game_id, snake_id ORDER BY game_id, snake_id - """) - sql = """ - INSERT INTO game_snakes ( - game_id,snake_id,snake_name,is_you,customizations_json - ) VALUES (?,?,?,?,?) - """ - count = 0 - while rows := cursor.fetchmany(batch_size): - values = [tuple(row) for row in rows if row[0] in allowed] - destination.executemany(sql, values) - count += len(values) - return count - -def copy_turns(source:sqlite3.Connection, destination:sqlite3.Connection, allowed:set[str], batch_size:int) -> int: - columns = object_columns(source, "turns") - names = ( - "game_id", "turn", "observed_at", "my_move", "my_thinking_json", - "board_state_json", "snakes_json", "you_json", "food_json", "hazards_json", - ) - defaults = { - "my_thinking_json": "NULL AS my_thinking_json", "board_state_json": "'{}' AS board_state_json", - "snakes_json": "'[]' AS snakes_json", "you_json": "'{}' AS you_json", - "food_json": "'[]' AS food_json", "hazards_json": "'[]' AS hazards_json", - } - selected = ",".join(name if name in columns else defaults[name] for name in names) - cursor = source.execute(f"SELECT {selected} FROM turns ORDER BY id") - sql = f"INSERT INTO turns ({','.join(names)}) VALUES ({','.join('?' for _ in names)})" - count = 0 - while rows := cursor.fetchmany(batch_size): - values = [tuple(row) for row in rows if row[0] in allowed] - destination.executemany(sql, values) - count += len(values) - return count - -def copy_snake_turns(source:sqlite3.Connection, destination:sqlite3.Connection, allowed:set[str], batch_size:int) -> int: - columns = object_columns(source, "snake_turns") - names = ( - "game_id", "turn", "snake_id", "snake_name", "health", "length", - "head_x", "head_y", "body_json", "is_you", "inferred_move", "latency", - ) - selected = ",".join(column_or_null(columns, name) for name in names) - cursor = source.execute(f"SELECT {selected} FROM snake_turns ORDER BY id") - sql = f"INSERT INTO snake_turns ({','.join(names)}) VALUES ({','.join('?' for _ in names)})" - count = 0 - while rows := cursor.fetchmany(batch_size): - values = [tuple(row) for row in rows if row[0] in allowed] - destination.executemany(sql, values) - count += len(values) - return count - -def copy_replays(source, destination, allowed:set[str], batch_size:int) -> dict[str, int]: - return { - "game_snakes": copy_game_snakes(source, destination, allowed, batch_size), - "turns": copy_turns(source, destination, allowed, batch_size), - "snake_turns": copy_snake_turns(source, destination, allowed, batch_size), - } - -def verify(destination:sqlite3.Connection, expected:dict[str, int]) -> None: - for table, count in expected.items(): - actual = int(destination.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]) - if actual != count: - raise RuntimeError(f"{table} count mismatch: expected {count}, got {actual}") - foreign_keys = destination.execute("PRAGMA foreign_key_check").fetchall() - if foreign_keys: - raise RuntimeError(f"Foreign-key verification failed with {len(foreign_keys)} errors") - integrity = destination.execute("PRAGMA integrity_check").fetchone()[0] - if integrity != "ok": - raise RuntimeError(f"Integrity check failed: {integrity}") +def verify(destination:SqliteGameplayBackend, expected:Counter) -> None: + actual:Counter = Counter() + for game_id in destination.list_game_ids_sync(): + bundle = destination.read_game_bundle_sync(game_id) + actual["games"] += 1 + actual["game_snakes"] += len(bundle.snakes) + actual["turns"] += len(bundle.turns) + actual["snake_states"] += len(bundle.states) + for key, count in expected.items(): + if actual[key] != count: + raise RuntimeError(f"{key} count mismatch: expected {count}, got {actual[key]}") def replace_base(base_path:Path, destination_path:Path) -> Path: timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") @@ -214,8 +85,6 @@ def main() -> None: parser.add_argument("--delta", required=True, type=Path, help="Database written during cleanup") parser.add_argument("--destination", required=True, type=Path) parser.add_argument("--minimum-quality", choices=("low", "medium", "high"), default="medium") - parser.add_argument("--batch-size", type=int, default=10_000) - parser.add_argument("--busy-timeout-ms", type=int, default=60_000) parser.add_argument("--replace-base", action="store_true", help="Replace base after verification and keep a timestamped backup") args = parser.parse_args() @@ -224,51 +93,32 @@ def main() -> None: destination_path = args.destination.expanduser().resolve() if len({base_path, delta_path, destination_path}) != 3: raise SystemExit("Base, delta, and destination must be different paths") + if destination_path.exists(): + raise SystemExit(f"Destination already exists: {destination_path}") started = perf_counter() - base = open_source(base_path) - delta = open_source(delta_path) - destination = create_destination(destination_path, max(1000, args.busy_timeout_ms)) - try: - base_games = read_games(base) - delta_games = read_games(delta) - duplicates = duplicate_ids(base_games, delta_games) - delta_quality = analyze_quality(delta) + base = SqliteGameplayBackend(str(base_path), read_only=True) + delta = SqliteGameplayBackend(str(delta_path), read_only=True) + base_ids = base.list_game_ids_sync() + delta_ids = delta.list_game_ids_sync() + duplicates = duplicate_ids(base, delta, base_ids, delta_ids) - base_count, base_replays, _ = insert_games(destination, base_games, set()) - delta_count, delta_replays, tiers = insert_games( - destination, delta_games, duplicates, delta_quality, args.minimum_quality, + destination = SqliteGameplayBackend(str(destination_path), journal_mode="DELETE") + try: + base_counts, _ = copy_games(base, destination, base_ids) + delta_counts, tiers = copy_games( + delta, destination, [game_id for game_id in delta_ids if game_id not in duplicates], + args.minimum_quality, ) - base_rows = copy_replays(base, destination, base_replays, max(1, args.batch_size)) - delta_rows = copy_replays(delta, destination, delta_replays - duplicates, max(1, args.batch_size)) - destination.commit() - destination.executescript(""" - CREATE INDEX IF NOT EXISTS idx_turns_game_turn ON turns(game_id, turn); - CREATE INDEX IF NOT EXISTS idx_games_status ON games(status); - CREATE INDEX IF NOT EXISTS idx_snake_turns_game_turn ON snake_turns(game_id, turn); - """) - destination.execute("PRAGMA foreign_keys = ON") - expected = { - "games": base_count + delta_count, - "game_snakes": base_rows["game_snakes"] + delta_rows["game_snakes"], - "turns": base_rows["turns"] + delta_rows["turns"], - "snake_turns": base_rows["snake_turns"] + delta_rows["snake_turns"], - } - verify(destination, expected) + verify(destination, base_counts + delta_counts) except Exception: - destination.close() - base.close() - delta.close() for suffix in ("", "-wal", "-shm"): Path(f"{destination_path}{suffix}").unlink(missing_ok=True) raise - destination.close() - base.close() - delta.close() - print(f"merged: base games={base_count:,}, delta games={delta_count:,}, identical duplicates skipped={len(duplicates):,}") + print(f"merged: base games={base_counts['games']:,}, delta games={delta_counts['games']:,}, identical duplicates skipped={len(duplicates):,}") print(f"delta quality: {dict(tiers)}") - print(f"verified: {expected}; elapsed: {perf_counter() - started:.1f}s") + print(f"verified: {dict(base_counts + delta_counts)}; elapsed: {perf_counter() - started:.1f}s") if args.replace_base: backup = replace_base(base_path, destination_path) print(f"base replaced; backup kept at: {backup}") diff --git a/scripts/migrate_gameplay_database.py b/scripts/migrate_gameplay_database.py deleted file mode 100644 index 4534ba6..0000000 --- a/scripts/migrate_gameplay_database.py +++ /dev/null @@ -1,355 +0,0 @@ -#!/usr/bin/env python3 -"""Create a compact, normalized copy of a gameplay SQLite database. - -The source is always opened read-only. The destination is written separately, -verified, and can optionally replace the source after a timestamped backup is -created. No in-place schema rewrite is performed. -""" -import argparse -from datetime import datetime, timezone -import json -from pathlib import Path -import sqlite3 -import sys -from time import perf_counter - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend -from server.database.game_quality import GameQualityInput, quality_meets_minimum, rate_game_quality - -def open_source(path:Path) -> sqlite3.Connection: - connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=60) - connection.row_factory = sqlite3.Row - connection.execute("PRAGMA query_only = ON") - return connection - -def object_columns(connection:sqlite3.Connection, name:str) -> set[str]: - return {row[1] for row in connection.execute(f"PRAGMA table_info({name})")} - -def column_or_null(columns:set[str], name:str) -> str: - return name if name in columns else f"NULL AS {name}" - -def create_destination(path:Path, busy_timeout_ms:int) -> sqlite3.Connection: - if path.exists(): - raise FileExistsError(f"Destination already exists: {path}") - SqliteGameplayBackend( - str(path), busy_timeout_ms=busy_timeout_ms, - initialize_indexes=False, journal_mode="DELETE", - ) - connection = sqlite3.connect(str(path), timeout=max(1, busy_timeout_ms // 1000)) - connection.execute("PRAGMA foreign_keys = OFF") - connection.execute("PRAGMA synchronous = OFF") - connection.execute("PRAGMA locking_mode = EXCLUSIVE") - connection.execute("PRAGMA temp_store = MEMORY") - connection.execute("PRAGMA cache_size = -262144") - connection.execute(f"PRAGMA busy_timeout = {busy_timeout_ms}") - return connection - -def analyze_quality(source:sqlite3.Connection) -> dict[str, object]: - games = { - row["game_id"]: row - for row in source.execute(""" - SELECT game_id, status, final_turn, winner_name - FROM games - """) - } - aggregates = { - row["game_id"]: row - for row in source.execute(""" - SELECT t.game_id, - COUNT(*) AS turn_rows, - MIN(t.turn) AS min_turn, - MAX(t.turn) AS max_turn, - SUM(CASE WHEN t.my_move IN ('up','down','left','right') THEN 1 ELSE 0 END) AS valid_moves, - SUM(CASE WHEN t.my_thinking_json IS NOT NULL AND t.my_thinking_json NOT IN ('', '{}', 'null') THEN 1 ELSE 0 END) AS thinking_rows, - COUNT(DISTINCT t.my_move) AS distinct_moves - FROM turns AS t - GROUP BY t.game_id - """) - } - snake_counts = { - row["game_id"]: int(row["snake_turn_rows"]) - for row in source.execute(""" - SELECT game_id, COUNT(*) AS snake_turn_rows - FROM snake_turns GROUP BY game_id - """) - } - - quality = {} - for game_id, game in games.items(): - aggregate = aggregates.get(game_id) - quality[game_id] = rate_game_quality(GameQualityInput( - status=game["status"], - final_turn=int(game["final_turn"] or 0), - turn_rows=int(aggregate["turn_rows"] if aggregate else 0), - min_turn=int(aggregate["min_turn"]) if aggregate and aggregate["min_turn"] is not None else None, - max_turn=int(aggregate["max_turn"]) if aggregate and aggregate["max_turn"] is not None else None, - valid_moves=int(aggregate["valid_moves"] if aggregate else 0), - thinking_rows=int(aggregate["thinking_rows"] if aggregate else 0), - distinct_moves=int(aggregate["distinct_moves"] if aggregate else 0), - snake_turn_rows=snake_counts.get(game_id, 0), - winner_name=game["winner_name"], - )) - return quality - -def copy_games(source:sqlite3.Connection, destination:sqlite3.Connection, batch_size:int, quality:dict[str, object], minimum_quality:str) -> tuple[int, int]: - columns = object_columns(source, "games") - selected = [ - "game_id", "started_at", "ended_at", "width", "height", "source", "map_name", - "ruleset_name", "ruleset_version", "your_snake_id", "your_snake_name", - column_or_null(columns, "your_snake_type"), column_or_null(columns, "your_snake_version"), - column_or_null(columns, "game_type"), column_or_null(columns, "winner_name"), - "winner_you", "final_turn", "status", - ] - cursor = source.execute(f"SELECT {', '.join(selected)} FROM games ORDER BY game_id") - sql = """ - INSERT INTO games ( - game_id, started_at, ended_at, width, height, source, map_name, - ruleset_name, ruleset_version, your_snake_id, your_snake_name, - your_snake_type, your_snake_version, game_type, winner_name, - winner_you, final_turn, status, has_replay, quality_status, - quality_score, quality_tier, quality_reasons_json - ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) - """ - count = 0 - retained = 0 - while rows := cursor.fetchmany(batch_size): - values = [] - for row in rows: - game_quality = quality[row[0]] - keep_replay = quality_meets_minimum(game_quality.tier, minimum_quality) - values.append(( - *tuple(row), 1 if keep_replay else 0, - "retained" if keep_replay else "low_quality", - game_quality.score, game_quality.tier, - json.dumps(game_quality.reasons, separators=(",", ":")), - )) - retained += int(keep_replay) - destination.executemany(sql, values) - count += len(rows) - return count, retained - -def retained_game_ids(quality:dict[str, object], minimum_quality:str) -> set[str]: - return { - game_id for game_id, result in quality.items() - if quality_meets_minimum(result.tier, minimum_quality) - } - -def copy_game_snakes(source:sqlite3.Connection, destination:sqlite3.Connection, batch_size:int, retained_ids:set[str]) -> int: - has_game_snakes = source.execute(""" - SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'game_snakes' - """).fetchone() is not None - sql = """ - INSERT OR IGNORE INTO game_snakes ( - game_id, snake_id, snake_name, is_you, customizations_json - ) VALUES (?, ?, ?, ?, ?) - """ - count = 0 - if has_game_snakes: - columns = object_columns(source, "game_snakes") - customizations = ( - "COALESCE(customizations_json, '{}') AS customizations_json" - if "customizations_json" in columns else "'{}' AS customizations_json" - ) - cursor = source.execute(f""" - SELECT game_id, snake_id, snake_name, is_you, {customizations} - FROM game_snakes ORDER BY game_id, snake_id - """) - while rows := cursor.fetchmany(batch_size): - retained_rows = [tuple(row) for row in rows if row[0] in retained_ids] - before = destination.total_changes - destination.executemany(sql, retained_rows) - count += destination.total_changes - before - - # Older databases can contain an empty or only partially populated - # game_snakes table. Always synthesize missing identities from snake_turns. - cursor = source.execute(""" - SELECT game_id, snake_id, MAX(snake_name), MAX(is_you), - '{}' AS customizations_json - FROM snake_turns - GROUP BY game_id, snake_id - ORDER BY game_id, snake_id - """) - while rows := cursor.fetchmany(batch_size): - retained_rows = [tuple(row) for row in rows if row[0] in retained_ids] - before = destination.total_changes - destination.executemany(sql, retained_rows) - count += destination.total_changes - before - return count - -def decode_json(value:str|None, fallback): - if not value: - return fallback - try: - return json.loads(value) - except (json.JSONDecodeError, TypeError): - return fallback - -def compact_board(row:sqlite3.Row) -> tuple[str, str, str]: - board = decode_json(row["board_state_json"], {}) - food = board.get("food") if isinstance(board, dict) else None - hazards = board.get("hazards") if isinstance(board, dict) else None - if food is None: - food = decode_json(row["food_json"], []) - if hazards is None: - hazards = decode_json(row["hazards_json"], []) - compact = json.dumps({}, separators=(",", ":")) - return ( - compact, - json.dumps(food or [], separators=(",", ":")), - json.dumps(hazards or [], separators=(",", ":")), - ) - -def copy_turns(source:sqlite3.Connection, destination:sqlite3.Connection, batch_size:int, retained_ids:set[str]) -> int: - cursor = source.execute(""" - SELECT t.id, t.game_id, t.turn, t.observed_at, t.my_move, t.my_thinking_json, - t.board_state_json, t.food_json, t.hazards_json - FROM turns AS t - ORDER BY t.id - """) - sql = """ - INSERT INTO turns ( - id, game_id, turn, observed_at, my_move, my_thinking_json, - board_state_json, snakes_json, you_json, food_json, hazards_json - ) VALUES (?,?,?,?,?,?,?,'[]','{}',?,?) - """ - count = 0 - while rows := cursor.fetchmany(batch_size): - values = [] - for row in rows: - if row["game_id"] not in retained_ids: - continue - compact, food, hazards = compact_board(row) - values.append(( - row["id"], row["game_id"], row["turn"], row["observed_at"], - row["my_move"], row["my_thinking_json"], compact, food, hazards, - )) - destination.executemany(sql, values) - count += len(values) - if count % max(batch_size, 100_000) == 0: - print(f"turns: {count:,}", flush=True) - return count - -def copy_snake_turns(source:sqlite3.Connection, destination:sqlite3.Connection, batch_size:int, retained_ids:set[str]) -> int: - columns = object_columns(source, "snake_turns") - latency = column_or_null(columns, "latency") - cursor = source.execute(f""" - SELECT st.id, st.game_id, st.turn, st.snake_id, st.health, st.length, - st.head_x, st.head_y, st.body_json, st.inferred_move, st.{latency} - FROM snake_turns AS st - ORDER BY st.id - """) - sql = """ - INSERT INTO snake_turns ( - id, game_id, turn, snake_id, snake_name, health, length, - head_x, head_y, body_json, is_you, inferred_move, latency - ) VALUES (?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, 0, ?, ?) - """ - count = 0 - while rows := cursor.fetchmany(batch_size): - destination.executemany(sql, [ - ( - row["id"], row["game_id"], row["turn"], row["snake_id"], - row["health"], row["length"], row["head_x"], row["head_y"], - row["body_json"], row["inferred_move"], row["latency"], - ) - for row in rows if row["game_id"] in retained_ids - ]) - count += sum(1 for row in rows if row["game_id"] in retained_ids) - if count % max(batch_size, 100_000) == 0: - print(f"snake_turns: {count:,}", flush=True) - return count - -def verify(source:sqlite3.Connection, destination:sqlite3.Connection, retained_ids:set[str]) -> dict[str, int]: - result = {} - retained_turns = sum( - 1 for row in source.execute("SELECT game_id FROM turns") - if row["game_id"] in retained_ids - ) - retained_snake_turns = sum( - 1 for row in source.execute("SELECT game_id FROM snake_turns") - if row["game_id"] in retained_ids - ) - expected = { - "games": int(source.execute("SELECT COUNT(*) FROM games").fetchone()[0]), - "turns": retained_turns, - "snake_turns": retained_snake_turns, - } - for table, source_count in expected.items(): - destination_count = int(destination.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]) - if source_count != destination_count: - raise RuntimeError(f"{table} count mismatch: {source_count} != {destination_count}") - result[table] = destination_count - integrity = destination.execute("PRAGMA integrity_check").fetchone()[0] - if integrity != "ok": - raise RuntimeError(f"Destination integrity check failed: {integrity}") - return result - -def replace_with_backup(source_path:Path, destination_path:Path) -> Path: - timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") - backup = source_path.with_name(f"{source_path.name}.backup-{timestamp}") - source_path.rename(backup) - try: - destination_path.rename(source_path) - except Exception: - backup.rename(source_path) - raise - return backup - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--source", required=True, type=Path) - parser.add_argument("--destination", type=Path) - parser.add_argument("--batch-size", type=int, default=10_000) - parser.add_argument("--busy-timeout-ms", type=int, default=60_000) - parser.add_argument("--minimum-quality", choices=("low", "medium", "high"), default="medium", help="Minimum quality tier whose replay rows are retained") - parser.add_argument("--replace", action="store_true", help="Backup source and replace it after verification") - args = parser.parse_args() - - source_path = args.source.expanduser().resolve() - destination_path = (args.destination or source_path.with_name(f"{source_path.stem}.compact{source_path.suffix}")).expanduser().resolve() - if source_path == destination_path: - raise SystemExit("Source and destination must be different paths") - - started = perf_counter() - source = open_source(source_path) - destination = create_destination(destination_path, max(1000, args.busy_timeout_ms)) - try: - quality = analyze_quality(source) - retained_ids = retained_game_ids(quality, args.minimum_quality) - games, retained_games = copy_games(source, destination, max(1, args.batch_size), quality, args.minimum_quality) - game_snakes = copy_game_snakes(source, destination, max(1, args.batch_size), retained_ids) - turns = copy_turns(source, destination, max(1, args.batch_size), retained_ids) - snake_turns = copy_snake_turns(source, destination, max(1, args.batch_size), retained_ids) - destination.commit() - destination.executescript(""" - CREATE INDEX IF NOT EXISTS idx_turns_game_turn ON turns(game_id, turn); - CREATE INDEX IF NOT EXISTS idx_games_status ON games(status); - CREATE INDEX IF NOT EXISTS idx_snake_turns_game_turn ON snake_turns(game_id, turn); - """) - destination.execute("PRAGMA foreign_keys = ON") - counts = verify(source, destination, retained_ids) - except Exception: - destination.close() - source.close() - for suffix in ("", "-wal", "-shm"): - Path(f"{destination_path}{suffix}").unlink(missing_ok=True) - raise - destination.close() - source.close() - - source_bytes = source_path.stat().st_size - destination_bytes = destination_path.stat().st_size - print(f"copied: game rates={games:,}, retained replays={retained_games:,}, game_snakes={game_snakes:,}, turns={turns:,}, snake_turns={snake_turns:,}") - print(f"verified: {counts}; size {source_bytes / 2**30:.2f} GiB -> {destination_bytes / 2**30:.2f} GiB") - print(f"elapsed: {perf_counter() - started:.1f}s") - - if args.replace: - backup = replace_with_backup(source_path, destination_path) - print(f"source replaced; backup kept at: {backup}") - else: - print(f"compact database created at: {destination_path}") - -if __name__ == "__main__": - main() diff --git a/scripts/migrate_gameplay_to_compact.py b/scripts/migrate_gameplay_to_compact.py new file mode 100644 index 0000000..7a365ef --- /dev/null +++ b/scripts/migrate_gameplay_to_compact.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Migrate gameplay data from the legacy turns/snake_turns layout to compact storage. + +Source and destination are a SQLite path or a postgresql:// DSN. Without +--destination the migration runs in place (same database, new tables). Every +game is read back and compared after writing; legacy tables are only removed +with --drop-legacy, and only when every legacy replay exists in compact form. + +Online PostgreSQL procedure: + 1. run the migration while the old bot version keeps recording, + 2. deploy the new version (it records into the compact tables), + 3. run again with --since to pick up games recorded meanwhile, + 4. run with --drop-legacy. +""" +import argparse +import asyncio +from pathlib import Path +import sqlite3 +import sys +from time import perf_counter + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from server.database.benchmark_states import is_postgresql_source +from server.database.backend.PostgresqlGameplayBackend import PostgresqlGameplayBackend +from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend +from server.database.legacy.migrate import migrate_legacy +from server.database.legacy.reader_postgresql import LegacyPostgresqlReader +from server.database.legacy.reader_sqlite import LegacySqliteReader + +_MISSING_REPLAYS = """ + SELECT COUNT(*) FROM games AS g + WHERE g.has_replay + AND EXISTS (SELECT 1 FROM turns AS t WHERE t.game_id = g.game_id) + AND NOT EXISTS (SELECT 1 FROM game_replays AS r WHERE r.gid = g.gid) + AND NOT EXISTS (SELECT 1 FROM game_turns AS c WHERE c.gid = g.gid) +""" + +def _progress(done:int, total:int, stats) -> None: + print( + f"{done:,}/{total:,} games — replays={stats.replays:,} turns={stats.turns:,} " + f"snake states={stats.states:,} failed={stats.failed:,}", + flush=True, + ) + +async def _drop_legacy(destination:str) -> None: + if is_postgresql_source(destination): + backend = PostgresqlGameplayBackend(destination) + pool = await backend._get_pool() + try: + async with pool.acquire() as conn: + missing = int(await conn.fetchval(_MISSING_REPLAYS)) + if missing: + raise SystemExit(f"{missing} games still lack a compact replay; legacy tables kept") + await conn.execute("DROP TABLE snake_turns") + await conn.execute("DROP TABLE turns") + finally: + await backend.close() + else: + connection = sqlite3.connect(destination) + try: + missing = int(connection.execute(_MISSING_REPLAYS.replace("g.has_replay", "g.has_replay = 1")).fetchone()[0]) + if missing: + raise SystemExit(f"{missing} games still lack a compact replay; legacy tables kept") + connection.execute("DROP TABLE snake_turns") + connection.execute("DROP TABLE turns") + connection.commit() + connection.execute("VACUUM") + finally: + connection.close() + print("legacy tables turns and snake_turns dropped") + +async def migrate(args:argparse.Namespace) -> None: + source = args.source + destination = args.destination or source + if args.drop_legacy: + await _drop_legacy(destination) + return + + if is_postgresql_source(destination): + backend = PostgresqlGameplayBackend(destination) + await backend.initialize() + else: + backend = SqliteGameplayBackend(destination) + + if is_postgresql_source(source): + reader = await LegacyPostgresqlReader.connect(source) + close_reader = reader.close + else: + reader = LegacySqliteReader(source) + async def close_reader() -> None: + reader.close() + + started = perf_counter() + try: + stats = await migrate_legacy( + reader, backend, since=args.since, minimum_quality=args.minimum_quality, + verify=not args.no_verify, progress=_progress, + ) + finally: + await close_reader() + await backend.close() + + print( + f"migrated {stats.games:,} games ({stats.replays:,} replays, {stats.turns:,} turns, " + f"{stats.states:,} snake states) in {perf_counter() - started:.1f}s" + ) + if args.minimum_quality: + print(f"replays dropped below '{args.minimum_quality}' quality: {stats.dropped_replays:,}") + if stats.failed: + print(f"FAILED games ({stats.failed}); their legacy rows are untouched:", file=sys.stderr) + for failure in stats.failures[:50]: + print(f" {failure}", file=sys.stderr) + raise SystemExit(1) + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--source", required=True, help="SQLite path or postgresql:// DSN with legacy tables") + parser.add_argument("--destination", help="SQLite path or postgresql:// DSN (default: in place)") + parser.add_argument("--since", help="Only games started at or after this ISO timestamp") + parser.add_argument("--minimum-quality", choices=("low", "medium", "high"), help="Re-rate replays and drop those below this tier") + parser.add_argument("--no-verify", action="store_true", help="Skip reading each game back after writing") + parser.add_argument("--drop-legacy", action="store_true", help="Drop turns/snake_turns after checking every replay was migrated") + asyncio.run(migrate(parser.parse_args())) + +if __name__ == "__main__": + main() diff --git a/scripts/migrate_sqlite_to_postgresql.py b/scripts/migrate_sqlite_to_postgresql.py deleted file mode 100644 index 057eaca..0000000 --- a/scripts/migrate_sqlite_to_postgresql.py +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env python3 -"""Stream a normalized Battlesnake SQLite database into PostgreSQL. - -The source is opened read-only. Rows are copied in bounded batches through -temporary PostgreSQL tables, then inserted idempotently with ON CONFLICT. The -script verifies source/inserted row counts and never modifies the SQLite file. -""" -import argparse -import asyncio -from datetime import datetime -import json -from pathlib import Path -import sqlite3 -from time import perf_counter - -from server.database.backend.PostgresqlGameplayBackend import PostgresqlGameplayBackend - -TABLES = ( - ( - "games", - ( - "game_id", "started_at", "ended_at", "width", "height", "source", - "map_name", "ruleset_name", "ruleset_version", "your_snake_id", - "your_snake_name", "your_snake_type", "your_snake_version", "game_type", - "winner_name", "winner_you", "final_turn", "status", "has_replay", - "quality_status", "quality_score", "quality_tier", "quality_reasons", - ), - "game_id", - ), - ( - "game_snakes", - ("game_id", "snake_id", "snake_name", "is_you", "customizations"), - "game_id, snake_id", - ), - ( - "turns", - ( - "game_id", "turn", "observed_at", "my_move", "my_thinking", - "board_state", "snakes", "you", "food", "hazards", - ), - "game_id, turn", - ), - ( - "snake_turns", - ( - "game_id", "turn", "snake_id", "snake_name", "health", "length", - "head_x", "head_y", "body", "is_you", "inferred_move", "latency", - ), - "game_id, turn, snake_id", - ), -) - -SQLITE_SELECTS = { - "games": """ - SELECT game_id, started_at, ended_at, width, height, source, map_name, - ruleset_name, ruleset_version, your_snake_id, your_snake_name, - your_snake_type, your_snake_version, game_type, winner_name, winner_you, - final_turn, status, has_replay, quality_status, quality_score, - quality_tier, quality_reasons_json - FROM games ORDER BY game_id - """, - "game_snakes": """ - SELECT game_id, snake_id, snake_name, is_you, customizations_json - FROM game_snakes ORDER BY game_id, snake_id - """, - "turns": """ - SELECT game_id, turn, observed_at, my_move, my_thinking_json, - board_state_json, snakes_json, you_json, food_json, hazards_json - FROM turns ORDER BY id - """, - "snake_turns": """ - SELECT game_id, turn, snake_id, snake_name, health, length, head_x, head_y, - body_json, is_you, inferred_move, latency - FROM snake_turns ORDER BY id - """, -} - -TIMESTAMP_FIELDS = {"started_at", "ended_at", "observed_at"} -BOOLEAN_FIELDS = {"winner_you", "has_replay", "is_you"} -JSON_FIELDS = { - "quality_reasons", "customizations", "my_thinking", "board_state", - "snakes", "you", "food", "hazards", "body", -} -JSON_DEFAULTS = { - "customizations": {}, "board_state": {}, "snakes": [], "you": {}, - "food": [], "hazards": [], "body": [], -} - -def parse_timestamp(value: str | None) -> datetime | None: - if not value: - return None - parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) - return parsed - -def parse_json(value: str | None, default: object = None) -> object: - if value in (None, ""): - return default - try: - return json.loads(value) - except (json.JSONDecodeError, TypeError) as exc: - raise ValueError(f"Invalid JSON value: {str(value)[:120]}") from exc - -def transform_row(columns: tuple[str, ...], row: sqlite3.Row) -> tuple: - output = [] - for column, value in zip(columns, row, strict=True): - if column in TIMESTAMP_FIELDS: - value = parse_timestamp(value) - elif column in BOOLEAN_FIELDS: - value = bool(value) - elif column in JSON_FIELDS: - parsed = parse_json(value, JSON_DEFAULTS.get(column)) - value = None if parsed is None else json.dumps(parsed, separators=(",", ":")) - output.append(value) - return tuple(output) - -def open_source(path: Path) -> sqlite3.Connection: - connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=60) - connection.row_factory = sqlite3.Row - return connection - -async def import_table(pool, source: sqlite3.Connection, table: str, - columns: tuple[str, ...], conflict_columns: str, - batch_size: int) -> tuple[int, int]: - source_count = int(source.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]) - async with pool.acquire() as connection: - before = int(await connection.fetchval(f"SELECT COUNT(*) FROM {table}")) - stage = f"migration_{table}" - await connection.execute( - f"CREATE TEMP TABLE {stage} (LIKE {table} INCLUDING DEFAULTS)" - ) - json_columns = [column for column in columns if column in JSON_FIELDS] - for column in json_columns: - await connection.execute( - f"ALTER TABLE {stage} ALTER COLUMN {column} TYPE TEXT USING {column}::text" - ) - cursor = source.execute(SQLITE_SELECTS[table]) - copied = 0 - while rows := cursor.fetchmany(batch_size): - records = [transform_row(columns, row) for row in rows] - async with connection.transaction(): - await connection.copy_records_to_table(stage, records=records, columns=columns) - selected = ", ".join(columns) - source_expressions = ", ".join( - f"{column}::jsonb" if column in JSON_FIELDS else column - for column in columns - ) - await connection.execute( - f"INSERT INTO {table} ({selected}) SELECT {source_expressions} FROM {stage} " - f"ON CONFLICT ({conflict_columns}) DO NOTHING" - ) - await connection.execute(f"TRUNCATE {stage}") - copied += len(records) - print(f"{table}: streamed {copied:,}/{source_count:,}", flush=True) - after = int(await connection.fetchval(f"SELECT COUNT(*) FROM {table}")) - inserted = after - before - if copied != source_count: - raise RuntimeError(f"{table}: source changed while reading ({source_count} -> {copied})") - print(f"{table}: source={source_count:,}, inserted={inserted:,}, conflicts={source_count - inserted:,}") - return source_count, inserted - -async def migrate(source_path: Path, dsn: str, batch_size: int) -> None: - source = open_source(source_path) - try: - quick_check = source.execute("PRAGMA quick_check").fetchone()[0] - if quick_check != "ok": - raise RuntimeError(f"SQLite quick_check failed: {quick_check}") - - backend = PostgresqlGameplayBackend(dsn=dsn) - await backend.initialize() - pool = await backend._get_pool() - started = perf_counter() - results = {} - try: - for table, columns, conflicts in TABLES: - results[table] = await import_table( - pool, source, table, columns, conflicts, max(100, batch_size), - ) - async with pool.acquire() as connection: - invalid = int(await connection.fetchval(""" - SELECT - (SELECT COUNT(*) FROM turns t LEFT JOIN games g USING (game_id) WHERE g.game_id IS NULL) + - (SELECT COUNT(*) FROM game_snakes s LEFT JOIN games g USING (game_id) WHERE g.game_id IS NULL) + - (SELECT COUNT(*) FROM snake_turns s LEFT JOIN games g USING (game_id) WHERE g.game_id IS NULL) - """)) - if invalid: - raise RuntimeError(f"PostgreSQL foreign-key verification found {invalid} orphan rows") - counts = { - table: int(await connection.fetchval(f"SELECT COUNT(*) FROM {table}")) - for table, _, _ in TABLES - } - print(f"verified PostgreSQL counts: {counts}") - print(f"migration elapsed: {perf_counter() - started:.1f}s") - finally: - await backend.close() - finally: - source.close() - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--source", required=True, type=Path) - parser.add_argument("--dsn", required=True) - parser.add_argument("--batch-size", type=int, default=10_000) - args = parser.parse_args() - asyncio.run(migrate(args.source.expanduser().resolve(), args.dsn, args.batch_size)) - -if __name__ == "__main__": - main() diff --git a/server/database/backend/PostgresqlGameplayBackend.py b/server/database/backend/PostgresqlGameplayBackend.py index 087a42e..bf7fb25 100644 --- a/server/database/backend/PostgresqlGameplayBackend.py +++ b/server/database/backend/PostgresqlGameplayBackend.py @@ -1,10 +1,8 @@ """PostgreSQL gameplay backend using asyncpg. -JSON columns use the JSONB type so PostgreSQL stores them in a binary, -decomposed format and automatically compresses large values via TOAST -(Oversized-Attribute Storage Technique). No application-level -serialisation/deserialisation round-trip is needed for reads — asyncpg -decodes JSONB rows directly into Python dicts/lists. +Running games are stored as compact rows (``game_turns``/``snake_states``). +When a game ends it is packed into one zstd blob in ``game_replays`` and its +rows are removed. See ``server.database.compact`` for the encodings. Connection: pass a DSN via the `dsn` constructor argument, e.g. postgresql://user:password@host:5432/dbname @@ -12,13 +10,22 @@ Connection: pass a DSN via the `dsn` constructor argument, e.g. or set GAMEPLAY_DB_PG_DSN in the environment. """ -import asyncio, logging, sqlite3, json, sys +import json, logging, sys from datetime import datetime, timezone from pathlib import Path from urllib.parse import urlparse, urlunparse from .Template import GameplayBackendTemplate -from server.database.normalized_turn import compact_turn_json +from .postgresql_schema import DROP_REDUNDANT_INDEXES, TABLES_DDL +from server.database.compact.cache import RunningGame, RunningGameCache +from server.database.compact.codec import ( + decode_body, decode_move, decode_points, encode_body, encode_move, encode_points, +) +from server.database.compact.games import GAME_COLUMNS, normalize_game +from server.database.compact.records import ( + GameBundle, SnakeIdentity, SnakeState, TurnRecord, merge_records, records_from_game_state, +) +from server.database.compact.replay_blob import REPLAY_FORMAT, pack_replay, unpack_replay logger = logging.getLogger(__name__) if not logger.handlers: @@ -27,106 +34,46 @@ if not logger.handlers: logger.addHandler(_handler) logger.propagate = False -# DDL --------------------------------------------------------------------- # +_SCHEMA_LOCK_ID = 7_315_002 -_DDL = """ -CREATE TABLE IF NOT EXISTS games ( - game_id TEXT PRIMARY KEY, - started_at TIMESTAMPTZ NOT NULL, - ended_at TIMESTAMPTZ, - width INTEGER, - height INTEGER, - source TEXT, - map_name TEXT, - ruleset_name TEXT, - ruleset_version TEXT, - your_snake_id TEXT, - your_snake_name TEXT, - your_snake_type TEXT, - your_snake_version TEXT, - game_type TEXT, - winner_name TEXT, - winner_you BOOLEAN NOT NULL DEFAULT FALSE, - final_turn INTEGER NOT NULL DEFAULT 0, - status TEXT NOT NULL DEFAULT 'running', - has_replay BOOLEAN NOT NULL DEFAULT TRUE, - quality_status TEXT NOT NULL DEFAULT 'retained', - quality_score INTEGER, - quality_tier TEXT, - quality_reasons JSONB -); - -CREATE TABLE IF NOT EXISTS turns ( - id BIGSERIAL PRIMARY KEY, - game_id TEXT NOT NULL REFERENCES games(game_id) ON DELETE CASCADE, - turn INTEGER NOT NULL, - observed_at TIMESTAMPTZ NOT NULL, - my_move TEXT, - my_thinking JSONB, - board_state JSONB NOT NULL, - snakes JSONB NOT NULL, - you JSONB NOT NULL, - food JSONB NOT NULL, - hazards JSONB NOT NULL, - UNIQUE (game_id, turn) -); - -CREATE TABLE IF NOT EXISTS game_snakes ( - game_id TEXT NOT NULL REFERENCES games(game_id) ON DELETE CASCADE, - snake_id TEXT NOT NULL, - snake_name TEXT, - is_you BOOLEAN NOT NULL DEFAULT FALSE, - customizations JSONB NOT NULL DEFAULT '{}'::jsonb, - PRIMARY KEY (game_id, snake_id) -); - -CREATE TABLE IF NOT EXISTS snake_turns ( - id BIGSERIAL PRIMARY KEY, - game_id TEXT NOT NULL REFERENCES games(game_id) ON DELETE CASCADE, - turn INTEGER NOT NULL, - snake_id TEXT NOT NULL, - snake_name TEXT, - health INTEGER, - length INTEGER, - head_x INTEGER, - head_y INTEGER, - body JSONB NOT NULL, - is_you BOOLEAN NOT NULL DEFAULT FALSE, - inferred_move TEXT, - latency TEXT, - UNIQUE (game_id, turn, snake_id) -); - -CREATE INDEX IF NOT EXISTS idx_turns_game_turn ON turns(game_id, turn); -CREATE INDEX IF NOT EXISTS idx_games_status ON games(status); -CREATE INDEX IF NOT EXISTS idx_snake_turns_game_turn ON snake_turns(game_id, turn); +_GAME_SELECT = """ + SELECT gid, game_id, started_at, ended_at, width, height, source, map_name, + ruleset_name, ruleset_version, game_type, your_snake_id, your_snake_name, + your_snake_type, your_snake_version, winner_name, winner_you, final_turn, + status, has_replay, quality_status, quality_score, quality_tier, quality_reasons + FROM games """ -# Schema evolution: add new columns to existing tables (idempotent). -_ALTER_DDL = """ -ALTER TABLE games ADD COLUMN IF NOT EXISTS game_type TEXT; -ALTER TABLE games ADD COLUMN IF NOT EXISTS your_snake_type TEXT; -ALTER TABLE games ADD COLUMN IF NOT EXISTS your_snake_version TEXT; -ALTER TABLE games ADD COLUMN IF NOT EXISTS winner_name TEXT; -ALTER TABLE games ADD COLUMN IF NOT EXISTS has_replay BOOLEAN NOT NULL DEFAULT TRUE; -ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_status TEXT NOT NULL DEFAULT 'retained'; -ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_score INTEGER; -ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_tier TEXT; -ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_reasons JSONB; -ALTER TABLE turns ADD COLUMN IF NOT EXISTS my_thinking JSONB; -ALTER TABLE snake_turns ADD COLUMN IF NOT EXISTS latency TEXT; -ALTER TABLE game_snakes ADD COLUMN IF NOT EXISTS customizations JSONB NOT NULL DEFAULT '{}'::jsonb; +_UPSERT_SNAKES = """ + INSERT INTO game_snakes (game_id, snake_id, snake_name, is_you, customizations) + SELECT $1, s.snake_id, s.snake_name, s.is_you, s.customizations::jsonb + FROM unnest($2::text[], $3::text[], $4::bool[], $5::text[]) + AS s(snake_id, snake_name, is_you, customizations) + ON CONFLICT (game_id, snake_id) DO UPDATE SET + snake_name = EXCLUDED.snake_name, + is_you = EXCLUDED.is_you, + customizations = EXCLUDED.customizations + RETURNING snake_id, gsid """ -# Force TOAST compression on the large JSONB columns so that even -# moderately-sized payloads get compressed on-disk. -_TOAST_DDL = """ -ALTER TABLE turns ALTER COLUMN board_state SET STORAGE EXTENDED; -ALTER TABLE turns ALTER COLUMN snakes SET STORAGE EXTENDED; -ALTER TABLE turns ALTER COLUMN you SET STORAGE EXTENDED; -ALTER TABLE turns ALTER COLUMN food SET STORAGE EXTENDED; -ALTER TABLE turns ALTER COLUMN hazards SET STORAGE EXTENDED; -ALTER TABLE snake_turns ALTER COLUMN body SET STORAGE EXTENDED; +_UPSERT_TURN = """ + INSERT INTO game_turns (gid, turn, observed_at, my_move, thinking, food, hazards) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (gid, turn) DO UPDATE SET + observed_at = EXCLUDED.observed_at, + my_move = EXCLUDED.my_move, + thinking = EXCLUDED.thinking, + food = EXCLUDED.food, + hazards = EXCLUDED.hazards +""" + +_UPSERT_STATES = """ + INSERT INTO snake_states (gsid, turn, health, latency, body) + SELECT * FROM unnest($1::int[], $2::int[], $3::smallint[], $4::smallint[], $5::bytea[]) + ON CONFLICT (gsid, turn) DO UPDATE SET + health = EXCLUDED.health, + latency = EXCLUDED.latency, + body = EXCLUDED.body """ class PostgresqlGameplayBackend(GameplayBackendTemplate): @@ -142,6 +89,7 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate): self._max_size = max_size self._sqlite_migration_path = sqlite_migration_path self._pool = None # asyncpg.Pool, typed at runtime + self._running = RunningGameCache() # ── DSN normalisation ────────────────────────────────────────────────────── @@ -149,25 +97,16 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate): @classmethod def _ensure_db_name(cls, dsn:str) -> str: - """Return *dsn* with a database name appended when none is present. - - A DSN has no database name when its path component is empty or ``/``. - In that case ``battlesnake`` is appended so asyncpg gets a complete - connection string without the caller having to remember to add one. - """ + """Return *dsn* with a database name appended when none is present.""" parsed = urlparse(dsn) - db = parsed.path.lstrip("/") - if db: + if parsed.path.lstrip("/"): return dsn - - new_path = f"/{cls._DEFAULT_DB_NAME}" - return urlunparse(parsed._replace(path=new_path)) + return urlunparse(parsed._replace(path=f"/{cls._DEFAULT_DB_NAME}")) # ── pool / schema ────────────────────────────────────────────────────────── async def initialize(self) -> None: - """Eagerly create the connection pool on startup so schema init and - SQLite migration run immediately rather than on the first game request.""" + """Eagerly create the pool so schema init and legacy SQLite import run on startup.""" await self._get_pool() async def _get_pool(self): @@ -194,27 +133,22 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate): init=_init_conn, ) await self._initialize_schema() - await self._maybe_migrate_from_sqlite() + await self._maybe_import_legacy_sqlite() return self._pool async def _ensure_database_exists(self, asyncpg, target_dsn:str) -> None: - """Connect to the postgres maintenance DB and CREATE the target database - if it does not already exist. Uses a plain connection (not a pool) so - the CREATE DATABASE statement can run outside any transaction.""" + """CREATE the target database through the maintenance DB when missing.""" parsed = urlparse(target_dsn) db_name = parsed.path.lstrip("/") maintenance_dsn = urlunparse(parsed._replace(path="/postgres")) try: conn = await asyncpg.connect(dsn=maintenance_dsn) except Exception: - # Fall back to connecting without specifying a database — some setups - # (e.g. Cloud SQL, managed PG) disallow direct access to 'postgres'. + # Some managed setups disallow direct access to 'postgres'. maintenance_dsn = urlunparse(parsed._replace(path="")) conn = await asyncpg.connect(dsn=maintenance_dsn) try: - exists = await conn.fetchval( - "SELECT 1 FROM pg_database WHERE datname = $1", db_name - ) + exists = await conn.fetchval("SELECT 1 FROM pg_database WHERE datname = $1", db_name) if not exists: await conn.execute(f'CREATE DATABASE "{db_name}"') logger.info(f"PostgreSQL: created database '{db_name}'") @@ -224,199 +158,143 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate): async def _initialize_schema(self) -> None: assert self._pool is not None async with self._pool.acquire() as conn: - await conn.execute(_DDL) - await conn.execute(_ALTER_DDL) - # TOAST storage hints are idempotent; ignore errors on repeated runs. + # Several workers may start at once; serialize DDL. + await conn.execute("SELECT pg_advisory_lock($1)", _SCHEMA_LOCK_ID) try: - await conn.execute(_TOAST_DDL) - except Exception as exc: - logger.debug(f"TOAST DDL skipped (likely already set): {exc}") + await conn.execute(TABLES_DDL) + finally: + await conn.execute("SELECT pg_advisory_unlock($1)", _SCHEMA_LOCK_ID) + for statement in DROP_REDUNDANT_INDEXES: + await conn.execute(statement) - # ── sqlite migration ─────────────────────────────────────────────────────── - - async def _maybe_migrate_from_sqlite(self) -> None: + async def _maybe_import_legacy_sqlite(self) -> None: if not self._sqlite_migration_path: return - - src = Path(self._sqlite_migration_path) - if not src.exists(): + source = Path(self._sqlite_migration_path) + if not source.exists(): return - logger.info(f"SQLite migration: found {src}, starting migration to PostgreSQL …") + from server.database.legacy.migrate import migrate_legacy # noqa: PLC0415 + from server.database.legacy.reader_sqlite import LegacySqliteReader # noqa: PLC0415 + + logger.info(f"Legacy SQLite import: found {source}, importing into PostgreSQL …") try: - games, turns, snake_turns = await asyncio.to_thread(self._read_sqlite_data_sync, str(src)) - await self._insert_migrated_data(games, turns, snake_turns) - - done_path = src.with_suffix(".migrated") - src.rename(done_path) - - logger.info( - f"SQLite migration complete: {len(games)} games, {len(turns)} turns, " - f"{len(snake_turns)} snake_turns migrated. " - f"Source file renamed to {done_path.name}" - ) + with LegacySqliteReader(source) as reader: + stats = await migrate_legacy(reader, self) + if stats.failed: + logger.warning(f"Legacy SQLite import incomplete ({stats.failed} games failed); source kept") + return + done_path = source.with_suffix(".migrated") + source.rename(done_path) + logger.info(f"Legacy SQLite import complete: {stats.games} games. Source renamed to {done_path.name}") except Exception: - logger.exception("SQLite migration failed — PostgreSQL data is untouched, original SQLite file kept") - - def _read_sqlite_data_sync(self, db_path:str) -> tuple[list[sqlite3.Row], list[sqlite3.Row], list[sqlite3.Row]]: - conn = sqlite3.connect(db_path, timeout=30, isolation_level=None) - conn.row_factory = sqlite3.Row - - try: - game_columns = { - row["name"] for row in conn.execute("PRAGMA table_info(games)").fetchall() - } - if "winner_name" in game_columns: - winner_name_expression = "winner_name" - elif "winner_names_json" in game_columns: - winner_name_expression = "winner_names_json" - else: - winner_name_expression = "NULL" - games = conn.execute(f""" - SELECT game_id, started_at, ended_at, width, height, source, map_name, - ruleset_name, ruleset_version, your_snake_id, your_snake_name, - your_snake_type, your_snake_version, game_type, - {winner_name_expression} AS winner_name, winner_you, final_turn, status - FROM games - ORDER BY started_at ASC - """).fetchall() - turns = conn.execute(""" - SELECT game_id, turn, observed_at, my_move, my_thinking_json, - board_state_json, snakes_json, you_json, food_json, hazards_json - FROM turns - ORDER BY game_id ASC, turn ASC - """).fetchall() - snake_turns = conn.execute(""" - SELECT game_id, turn, snake_id, snake_name, health, length, - head_x, head_y, body_json, is_you, inferred_move, latency - FROM snake_turns - ORDER BY game_id ASC, turn ASC, snake_id ASC - """).fetchall() - finally: - conn.close() - - return games, turns, snake_turns - - def _parse_ts(self, value:str|None) -> datetime|None: - """Parse an ISO-8601 TEXT timestamp from SQLite into a timezone-aware datetime.""" - ts = self._parse_utc_timestamp(value) - return ts # already UTC-aware from base class helper - - def _parse_json(self, value: str|None) -> object: - if not value: - return None - try: - return json.loads(value) - except (json.JSONDecodeError, TypeError): - return None - - def _migrated_winner_name(self, value:str|None) -> str|None: - """Accept both the current winner_name and legacy winner_names_json value.""" - if not value: - return None - parsed = self._parse_json(value) - if isinstance(parsed, list): - return next((str(name) for name in parsed if name), None) - return value - - async def _insert_migrated_data(self, games:list, turns:list, snake_turns:list) -> None: - assert self._pool is not None - async with self._pool.acquire() as conn: - async with conn.transaction(): - # games ───────────────────────────────────────────────────────────── - # winner_name is TEXT — no cast needed. - await conn.executemany(""" - INSERT INTO games ( - game_id, started_at, ended_at, width, height, source, map_name, - ruleset_name, ruleset_version, your_snake_id, your_snake_name, - your_snake_type, your_snake_version, game_type, - winner_name, winner_you, final_turn, status - ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18) - ON CONFLICT (game_id) DO NOTHING - """, - [ - ( - row["game_id"], - self._parse_ts(row["started_at"]), - self._parse_ts(row["ended_at"]), - row["width"], - row["height"], - row["source"], - row["map_name"], - row["ruleset_name"], - row["ruleset_version"], - row["your_snake_id"], - row["your_snake_name"], - row["your_snake_type"], - row["your_snake_version"], - row["game_type"], - self._migrated_winner_name(row["winner_name"]), - bool(row["winner_you"]), - row["final_turn"], - row["status"], - ) - for row in games - ], - ) - - await conn.executemany(""" - INSERT INTO turns ( - game_id, turn, observed_at, my_move, my_thinking, - board_state, snakes, you, food, hazards - ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) - ON CONFLICT (game_id, turn) DO NOTHING - """, - [ - ( - row["game_id"], - row["turn"], - self._parse_ts(row["observed_at"]), - row["my_move"], - self._parse_json(row["my_thinking_json"]), - self._parse_json(row["board_state_json"]), - self._parse_json(row["snakes_json"]), - self._parse_json(row["you_json"]), - self._parse_json(row["food_json"]), - self._parse_json(row["hazards_json"]), - ) - for row in turns - ], - ) - - # snake_turns - await conn.executemany(""" - INSERT INTO snake_turns ( - game_id, turn, snake_id, snake_name, health, length, - head_x, head_y, body, is_you, inferred_move, latency - ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12) - ON CONFLICT (game_id, turn, snake_id) DO NOTHING - """, - [ - ( - row["game_id"], - row["turn"], - row["snake_id"], - row["snake_name"], - row["health"], - row["length"], - row["head_x"], - row["head_y"], - self._parse_json(row["body_json"]), - bool(row["is_you"]), - row["inferred_move"], - row["latency"], - ) - for row in snake_turns - ], - ) + logger.exception("Legacy SQLite import failed — original SQLite file kept") # ── helpers ──────────────────────────────────────────────────────────────── def _utc_now_ts(self) -> datetime: return datetime.now(timezone.utc) - # The pool init callback registers JSON/JSONB codecs so asyncpg automatically - # encodes Python dicts/lists on write and decodes them on read. + async def _running_game(self, conn, game_id:str) -> RunningGame: + game = self._running.get(game_id) + if game is None: + gid = await conn.fetchval("SELECT gid FROM games WHERE game_id = $1", game_id) + if gid is None: + raise LookupError(f"Turn for unknown game {game_id!r}") + game = RunningGame(gid=gid) + return game + + async def _upsert_identities(self, conn, game_id:str, identities:list[SnakeIdentity]) -> dict[str, int]: + if not identities: + return {} + rows = await conn.fetch( + _UPSERT_SNAKES, game_id, + [identity.snake_id for identity in identities], + [identity.name for identity in identities], + [identity.is_you for identity in identities], + [json.dumps(identity.customizations or {}) for identity in identities], + ) + return {row["snake_id"]: row["gsid"] for row in rows} + + async def _write_rows(self, conn, gid:int, gsids:dict[str, int], turns:list[TurnRecord], states:list[SnakeState]) -> None: + if turns: + await conn.executemany(_UPSERT_TURN, [ + ( + gid, record.turn, record.observed_at or self._utc_now_ts(), encode_move(record.move), + record.thinking, encode_points(record.food), encode_points(record.hazards), + ) + for record in turns + ]) + states = [state for state in states if state.snake_id in gsids] + if states: + await conn.execute( + _UPSERT_STATES, + [gsids[state.snake_id] for state in states], + [state.turn for state in states], + [state.health for state in states], + [state.latency for state in states], + [encode_body(state.body) for state in states], + ) + + async def _load_records(self, conn, game_id:str, gid:int) -> tuple[list[SnakeIdentity], list[TurnRecord], list[SnakeState]]: + identity_rows = await conn.fetch(""" + SELECT snake_id, snake_name, is_you, customizations + FROM game_snakes WHERE game_id = $1 + """, game_id) + identities = [ + SnakeIdentity(row["snake_id"], row["snake_name"], bool(row["is_you"]), row["customizations"] or {}) + for row in identity_rows + ] + blob = await conn.fetchrow("SELECT format, data FROM game_replays WHERE gid = $1", gid) + turn_rows = await conn.fetch(""" + SELECT turn, observed_at, my_move, thinking, food, hazards + FROM game_turns WHERE gid = $1 ORDER BY turn + """, gid) + state_rows = await conn.fetch(""" + SELECT gs.snake_id, s.turn, s.health, s.latency, s.body + FROM game_snakes AS gs + JOIN snake_states AS s ON s.gsid = gs.gsid + WHERE gs.game_id = $1 + """, game_id) + turns = [ + TurnRecord( + turn=row["turn"], observed_at=row["observed_at"], move=decode_move(row["my_move"]), + thinking=row["thinking"], food=decode_points(row["food"]), hazards=decode_points(row["hazards"]), + ) + for row in turn_rows + ] + states = [ + SnakeState(row["snake_id"], row["turn"], row["health"], row["latency"], decode_body(row["body"])) + for row in state_rows + ] + if blob is not None: + packed_turns, packed_states = unpack_replay(blob["data"], blob["format"]) + turns, states = merge_records(packed_turns, packed_states, turns, states) + return identities, turns, states + + async def _store_replay(self, conn, game_id:str, gid:int, turns:list[TurnRecord], states:list[SnakeState]) -> None: + await conn.execute(""" + INSERT INTO game_replays (gid, format, turn_count, data) VALUES ($1, $2, $3, $4) + ON CONFLICT (gid) DO UPDATE SET + format = EXCLUDED.format, turn_count = EXCLUDED.turn_count, data = EXCLUDED.data + """, gid, REPLAY_FORMAT, len(turns), pack_replay(turns, states)) + await conn.execute("DELETE FROM game_turns WHERE gid = $1", gid) + await conn.execute(""" + DELETE FROM snake_states + WHERE gsid IN (SELECT gsid FROM game_snakes WHERE game_id = $1) + """, game_id) + + async def _pack_game(self, conn, game_id:str) -> None: + """Move a game's rows into its replay blob (idempotent).""" + async with conn.transaction(): + gid = await conn.fetchval("SELECT gid FROM games WHERE game_id = $1 FOR UPDATE", game_id) + if gid is None: + return + has_rows = await conn.fetchval("SELECT EXISTS (SELECT 1 FROM game_turns WHERE gid = $1)", gid) + if not has_rows: + return + _, turns, states = await self._load_records(conn, game_id, gid) + await self._store_replay(conn, game_id, gid, turns, states) # ── write methods ────────────────────────────────────────────────────────── @@ -465,111 +343,24 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate): ) async def record_turn(self, game_state:dict, my_move:str|None, my_thinking:dict|None=None) -> None: - game = game_state.get("game", {}) - board = game_state.get("board", {}) - snakes = self._extract_snakes(game_state) - you = self._extract_you(game_state) - game_id = game.get("id") - turn = int(game_state.get("turn", 0)) - - board_json, snakes_json, you_json, food_json, hazards_json = compact_turn_json(board) - + game_id = game_state.get("game", {}).get("id") pool = await self._get_pool() async with pool.acquire() as conn: async with conn.transaction(): - await conn.executemany(""" - INSERT INTO game_snakes ( - game_id, snake_id, snake_name, is_you, customizations - ) VALUES ($1,$2,$3,$4,$5) - ON CONFLICT (game_id, snake_id) DO UPDATE SET - snake_name = EXCLUDED.snake_name, - is_you = EXCLUDED.is_you, - customizations = EXCLUDED.customizations - """, - [ - ( - game_id, - snake.get("id"), - snake.get("name"), - snake.get("id") == you.get("id"), - snake.get("customizations") or {}, - ) - for snake in snakes if snake.get("id") is not None - ], + running = await self._running_game(conn, game_id) + record, states, identities, points = records_from_game_state( + game_state, my_move, my_thinking, running.points, ) - await conn.execute(""" - INSERT INTO turns ( - game_id, turn, observed_at, my_move, my_thinking, - board_state, snakes, you, food, hazards - ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) - ON CONFLICT (game_id, turn) DO UPDATE SET - observed_at = EXCLUDED.observed_at, - my_move = EXCLUDED.my_move, - my_thinking = EXCLUDED.my_thinking, - board_state = EXCLUDED.board_state, - snakes = EXCLUDED.snakes, - you = EXCLUDED.you, - food = EXCLUDED.food, - hazards = EXCLUDED.hazards - """, - game_id, - turn, - self._utc_now_ts(), - my_move, - my_thinking, - board_json, - snakes_json, - you_json, - food_json, - hazards_json, - ) - - previous_positions:dict[str, tuple[int, int]] = {} - if turn > 0: - prev_rows = await conn.fetch(""" - SELECT snake_id, head_x, head_y - FROM snake_turns - WHERE game_id = $1 AND turn = $2 - """, - game_id, turn - 1, - ) - previous_positions = { - row["snake_id"]: (int(row["head_x"]), int(row["head_y"])) - for row in prev_rows - if row["head_x"] is not None and row["head_y"] is not None - } - - you_id = you.get("id") - for params in self._build_snake_turn_params(snakes, you_id, game_id, turn, previous_positions): - (p_game_id, p_turn, p_snake_id, p_name, p_health, p_length, - p_head_x, p_head_y, p_body, p_is_you, p_inferred, p_latency) = params - await conn.execute(""" - INSERT INTO snake_turns ( - game_id, turn, snake_id, snake_name, health, length, - head_x, head_y, body, is_you, inferred_move, latency - ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12) - ON CONFLICT (game_id, turn, snake_id) DO UPDATE SET - snake_name = EXCLUDED.snake_name, - health = EXCLUDED.health, - length = EXCLUDED.length, - head_x = EXCLUDED.head_x, - head_y = EXCLUDED.head_y, - body = EXCLUDED.body, - is_you = EXCLUDED.is_you, - inferred_move = EXCLUDED.inferred_move, - latency = EXCLUDED.latency - """, - p_game_id, p_turn, p_snake_id, None, p_health, p_length, - p_head_x, p_head_y, p_body, False, p_inferred, p_latency, - ) - - await conn.execute(""" - UPDATE games - SET final_turn = GREATEST(final_turn, $1) - WHERE game_id = $2 - """, - turn, game_id, + missing = [identity for identity in identities if identity.snake_id not in running.snake_ids] + if missing: + running.snake_ids.update(await self._upsert_identities(conn, game_id, identities)) + await self._write_rows(conn, running.gid, running.snake_ids, [record], states) + await conn.execute( + "UPDATE games SET final_turn = GREATEST(final_turn, $1) WHERE gid = $2", + record.turn, running.gid, ) + running.points = points + self._running.put(game_id, running) async def record_game_end(self, game_state:dict) -> None: game_id, winner_name, winner_you, turn = self._extract_game_end_params(game_state) @@ -585,12 +376,10 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate): status = 'finished' WHERE game_id = $5 """, - self._utc_now_ts(), - winner_name, - winner_you, - turn, - game_id, + self._utc_now_ts(), winner_name, winner_you, turn, game_id, ) + await self._pack_game(conn, game_id) + self._running.drop(game_id) # ── stale game finalization ──────────────────────────────────────────────── @@ -602,7 +391,7 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate): pool = await self._get_pool() async with pool.acquire() as conn: rows = await conn.fetch(""" - SELECT game_id, started_at, final_turn, your_snake_id + SELECT gid, game_id, started_at, final_turn, your_snake_id FROM games WHERE status = 'running' ORDER BY started_at ASC @@ -618,40 +407,9 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate): continue game_id = row["game_id"] - your_snake_id = row["your_snake_id"] - final_turn = int(row["final_turn"] or 0) - - snake_rows = await conn.fetch(""" - SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name) AS snake_name - FROM snake_turns AS st - LEFT JOIN game_snakes AS gs - ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id - WHERE st.game_id = $1 AND st.turn = $2 - ORDER BY COALESCE(gs.is_you, st.is_you) DESC, snake_name ASC - """, - game_id, final_turn, - ) - - if len(snake_rows) == 0: - latest_row = await conn.fetchrow( - "SELECT MAX(turn) AS latest_turn FROM snake_turns WHERE game_id = $1", - game_id, - ) - - if latest_row is not None and latest_row["latest_turn"] is not None: - final_turn = int(latest_row["latest_turn"]) - snake_rows = await conn.fetch(""" - SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name) AS snake_name - FROM snake_turns AS st - LEFT JOIN game_snakes AS gs - ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id - WHERE st.game_id = $1 AND st.turn = $2 - ORDER BY COALESCE(gs.is_you, st.is_you) DESC, snake_name ASC - """, - game_id, final_turn, - ) - - winner_you, survivor_name = self._calculate_survivor(snake_rows, your_snake_id) + identities, _, states = await self._load_records(conn, game_id, row["gid"]) + final_turn, survivors = self._survivors(identities, states, int(row["final_turn"] or 0)) + winner_you, survivor_name = self._calculate_survivor(survivors, row["your_snake_id"]) tag = await conn.execute(""" UPDATE games @@ -662,14 +420,21 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate): status = 'finished' WHERE game_id = $5 AND status = 'running' """, - self._utc_now_ts(), - survivor_name, - winner_you, - final_turn, - game_id, + self._utc_now_ts(), survivor_name, winner_you, final_turn, game_id, ) if tag and tag.endswith("1"): finalized += 1 + await self._pack_game(conn, game_id) + self._running.drop(game_id) + + # Turns that arrived after a game was packed are folded into its blob. + late = await conn.fetch(""" + SELECT DISTINCT g.game_id + FROM game_turns AS t JOIN games AS g ON g.gid = t.gid + WHERE g.status = 'finished' + """) + for late_row in late: + await self._pack_game(conn, late_row["game_id"]) return finalized @@ -734,51 +499,51 @@ class PostgresqlGameplayBackend(GameplayBackendTemplate): async def get_game_replay(self, game_id:str) -> dict|None: pool = await self._get_pool() async with pool.acquire() as conn: - game_row = await conn.fetchrow(""" - SELECT game_id, started_at, ended_at, width, height, source, map_name, - ruleset_name, ruleset_version, game_type, your_snake_id, your_snake_name, - your_snake_type, your_snake_version, - winner_name, winner_you, final_turn, status - FROM games - WHERE game_id = $1 AND has_replay - """, - game_id, - ) - + game_row = await conn.fetchrow(_GAME_SELECT + " WHERE game_id = $1 AND has_replay", game_id) if game_row is None: return None + identities, turns, states = await self._load_records(conn, game_id, game_row["gid"]) - turn_rows = await conn.fetch(""" - SELECT turn, observed_at, my_move, - my_thinking AS my_thinking_json, - board_state AS board_state_json, - food AS food_json, - hazards AS hazards_json, - you AS you_json - FROM turns - WHERE game_id = $1 - ORDER BY turn ASC - """, - game_id, - ) + return self._build_game_replay_output(game_row, identities, turns, states) - snake_rows = await conn.fetch(""" - SELECT st.turn, st.snake_id, - COALESCE(gs.snake_name, st.snake_name) AS snake_name, - st.health, st.length, st.head_x, st.head_y, st.body AS body_json, - COALESCE(gs.is_you, st.is_you) AS is_you, - COALESCE(gs.customizations, '{}'::jsonb) AS customizations_json, - st.inferred_move, st.latency - FROM snake_turns AS st - LEFT JOIN game_snakes AS gs - ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id - WHERE st.game_id = $1 - ORDER BY st.turn ASC, is_you DESC, snake_name ASC - """, - game_id, - ) + # ── bundles (migration / merge) ──────────────────────────────────────────── - return self._build_game_replay_output(game_row, turn_rows, snake_rows, decode_json=lambda x: x) + async def list_game_ids(self) -> list[str]: + pool = await self._get_pool() + async with pool.acquire() as conn: + rows = await conn.fetch("SELECT game_id FROM games ORDER BY started_at, game_id") + return [row["game_id"] for row in rows] + + async def read_game_bundle(self, game_id:str) -> GameBundle|None: + pool = await self._get_pool() + async with pool.acquire() as conn: + game_row = await conn.fetchrow(_GAME_SELECT + " WHERE game_id = $1", game_id) + if game_row is None: + return None + identities, turns, states = await self._load_records(conn, game_id, game_row["gid"]) + return GameBundle(normalize_game(dict(game_row)), identities, turns, states) + + async def import_game_bundle(self, bundle:GameBundle) -> None: + game = normalize_game(bundle.game) + columns = ", ".join(GAME_COLUMNS) + placeholders = ", ".join(f"${index}" for index in range(1, len(GAME_COLUMNS) + 1)) + updates = ", ".join(f"{column} = EXCLUDED.{column}" for column in GAME_COLUMNS[1:]) + pool = await self._get_pool() + async with pool.acquire() as conn: + async with conn.transaction(): + gid = await conn.fetchval( + f"INSERT INTO games ({columns}) VALUES ({placeholders}) " + f"ON CONFLICT (game_id) DO UPDATE SET {updates} RETURNING gid", + *[game[column] for column in GAME_COLUMNS], + ) + gsids = await self._upsert_identities(conn, game["game_id"], bundle.snakes) + if not bundle.turns or not game["has_replay"]: + return + if game["status"] == "finished": + await self._store_replay(conn, game["game_id"], gid, bundle.turns, bundle.states) + else: + await self._write_rows(conn, gid, gsids, bundle.turns, bundle.states) + self._running.drop(game["game_id"]) # ── lifecycle ────────────────────────────────────────────────────────────── diff --git a/server/database/backend/SqliteGameplayBackend.py b/server/database/backend/SqliteGameplayBackend.py index 8d5f8cc..49fb530 100644 --- a/server/database/backend/SqliteGameplayBackend.py +++ b/server/database/backend/SqliteGameplayBackend.py @@ -1,11 +1,24 @@ -from quart_common.web.env import env_bool +"""SQLite gameplay backend with the same compact layout as PostgreSQL. -import asyncio, sqlite3, json, os, logging, sys +Running games use ``game_turns``/``snake_states``; finished games are packed +into one zstd blob per game in ``game_replays``. +""" + +import asyncio, json, sqlite3, logging, sys from datetime import datetime, timezone from pathlib import Path from server.database.backend.Template import GameplayBackendTemplate -from server.database.normalized_turn import compact_turn_json +from server.database.backend.sqlite_schema import create_schema +from server.database.compact.cache import RunningGame, RunningGameCache +from server.database.compact.codec import ( + decode_body, decode_move, decode_points, encode_body, encode_move, encode_points, +) +from server.database.compact.games import GAME_COLUMNS, normalize_game +from server.database.compact.records import ( + GameBundle, SnakeIdentity, SnakeState, TurnRecord, merge_records, records_from_game_state, +) +from server.database.compact.replay_blob import REPLAY_FORMAT, pack_replay, unpack_replay logger = logging.getLogger(__name__) if not logger.handlers: @@ -14,37 +27,45 @@ if not logger.handlers: logger.addHandler(_handler) logger.propagate = False -_ZSTD_EXT = Path(os.environ.get("SQLITE_ZSTD_EXT", "/usr/local/lib/libsqlite_zstd.so")).expanduser().resolve() +_GAME_SELECT = """ + SELECT gid, game_id, started_at, ended_at, width, height, source, map_name, + ruleset_name, ruleset_version, game_type, your_snake_id, your_snake_name, + your_snake_type, your_snake_version, winner_name, winner_you, final_turn, + status, has_replay, quality_status, quality_score, quality_tier, quality_reasons_json + FROM games +""" + +def _game_from_row(row:sqlite3.Row) -> dict: + data = dict(row) + reasons = data.pop("quality_reasons_json", None) + data["quality_reasons"] = json.loads(reasons) if reasons else None + return normalize_game(data) class SqliteGameplayBackend(GameplayBackendTemplate): - def __init__(self, db_path:str, busy_timeout_ms:int=5000, initialize_indexes:bool=True, journal_mode:str="WAL"): + def __init__(self, db_path:str, busy_timeout_ms:int=5000, journal_mode:str="WAL", read_only:bool=False): self.db_path = db_path self.busy_timeout_ms = max(1000, int(busy_timeout_ms)) - self._zstd_available = False - self._initialize_indexes = initialize_indexes self._journal_mode = journal_mode - self._initialize_database() + self._read_only = read_only + self._running = RunningGameCache() + if not read_only: + self._initialize_database() # ── connection ───────────────────────────────────────────────────────────── def _connect(self) -> sqlite3.Connection: + if self._read_only: + connection = sqlite3.connect(f"file:{self.db_path}?mode=ro", uri=True, isolation_level=None) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA query_only = ON") + return connection + connection = sqlite3.connect( self.db_path, timeout=max(1, self.busy_timeout_ms // 1000), isolation_level=None, ) connection.row_factory = sqlite3.Row - - if _ZSTD_EXT.exists() and not env_bool('DISABLE_GAMEPLAY_DB_COMPRESSION', True): - try: - connection.enable_load_extension(True) - connection.load_extension(str(_ZSTD_EXT)) - self._zstd_available = True - except sqlite3.OperationalError as e: - logger.warning(f"sqlite-zstd extension skipped: {e}") - finally: - connection.enable_load_extension(False) - connection.execute("PRAGMA foreign_keys = ON") connection.execute(f"PRAGMA journal_mode = {self._journal_mode}") connection.execute("PRAGMA synchronous = NORMAL") @@ -53,159 +74,142 @@ class SqliteGameplayBackend(GameplayBackendTemplate): connection.execute(f"PRAGMA busy_timeout = {self.busy_timeout_ms}") return connection - def _ensure_auto_vacuum_full(self, connection:sqlite3.Connection) -> None: - current = connection.execute("PRAGMA auto_vacuum").fetchone()[0] - if current != 1: - connection.execute("PRAGMA auto_vacuum = FULL") - connection.execute("VACUUM") - - # ── schema setup ─────────────────────────────────────────────────────────── - def _initialize_database(self) -> None: Path(self.db_path).parent.mkdir(parents=True, exist_ok=True) - with self._connect() as connection: - self._ensure_auto_vacuum_full(connection) - connection.executescript(""" - CREATE TABLE IF NOT EXISTS games ( - game_id TEXT PRIMARY KEY, - started_at TEXT NOT NULL, - ended_at TEXT, - width INTEGER, - height INTEGER, - source TEXT, - map_name TEXT, - ruleset_name TEXT, - ruleset_version TEXT, - your_snake_id TEXT, - your_snake_name TEXT, - your_snake_type TEXT, - your_snake_version TEXT, - winner_name TEXT, - winner_you INTEGER NOT NULL DEFAULT 0, - final_turn INTEGER NOT NULL DEFAULT 0, - status TEXT NOT NULL DEFAULT 'running', - has_replay INTEGER NOT NULL DEFAULT 1, - quality_status TEXT NOT NULL DEFAULT 'retained', - quality_score INTEGER, - quality_tier TEXT, - quality_reasons_json TEXT - ); - - CREATE TABLE IF NOT EXISTS turns ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - game_id TEXT NOT NULL, - turn INTEGER NOT NULL, - observed_at TEXT NOT NULL, - my_move TEXT, - my_thinking_json TEXT, - board_state_json TEXT NOT NULL, - snakes_json TEXT NOT NULL, - you_json TEXT NOT NULL, - food_json TEXT NOT NULL, - hazards_json TEXT NOT NULL, - UNIQUE (game_id, turn), - FOREIGN KEY (game_id) REFERENCES games(game_id) ON DELETE CASCADE - ); - - CREATE TABLE IF NOT EXISTS game_snakes ( - game_id TEXT NOT NULL, - snake_id TEXT NOT NULL, - snake_name TEXT, - is_you INTEGER NOT NULL DEFAULT 0, - customizations_json TEXT NOT NULL DEFAULT '{}', - PRIMARY KEY (game_id, snake_id), - FOREIGN KEY (game_id) REFERENCES games(game_id) ON DELETE CASCADE - ); - - CREATE TABLE IF NOT EXISTS snake_turns ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - game_id TEXT NOT NULL, - turn INTEGER NOT NULL, - snake_id TEXT NOT NULL, - snake_name TEXT, - health INTEGER, - length INTEGER, - head_x INTEGER, - head_y INTEGER, - body_json TEXT NOT NULL, - is_you INTEGER NOT NULL DEFAULT 0, - inferred_move TEXT, - UNIQUE (game_id, turn, snake_id), - FOREIGN KEY (game_id) REFERENCES games(game_id) ON DELETE CASCADE - ); - """) - if self._initialize_indexes: - self._create_indexes_if_tables(connection) - self._ensure_column_exists(connection, "turns", "my_thinking_json", "TEXT") - self._ensure_column_exists(connection, "games", "your_snake_type", "TEXT") - self._ensure_column_exists(connection, "games", "your_snake_version", "TEXT") - self._ensure_column_exists(connection, "games", "game_type", "TEXT") - self._ensure_column_exists(connection, "snake_turns", "latency", "TEXT") - self._ensure_column_exists(connection, "game_snakes", "customizations_json", "TEXT NOT NULL DEFAULT '{}'") - self._ensure_column_exists(connection, "games", "winner_name", "TEXT") - self._ensure_column_exists(connection, "games", "has_replay", "INTEGER NOT NULL DEFAULT 1") - self._ensure_column_exists(connection, "games", "quality_status", "TEXT NOT NULL DEFAULT 'retained'") - self._ensure_column_exists(connection, "games", "quality_score", "INTEGER") - self._ensure_column_exists(connection, "games", "quality_tier", "TEXT") - self._ensure_column_exists(connection, "games", "quality_reasons_json", "TEXT") - if self._zstd_available: - self._enable_zstd_compression(connection) + connection = self._connect() + try: + if connection.execute("PRAGMA auto_vacuum").fetchone()[0] != 1: + connection.execute("PRAGMA auto_vacuum = FULL") + connection.execute("VACUUM") + create_schema(connection) connection.execute("PRAGMA optimize") + finally: + connection.close() - def _create_indexes_if_tables(self, connection:sqlite3.Connection) -> None: - real_tables = { - row[0] for row in connection.execute( - "SELECT name FROM sqlite_master WHERE type='table'" - ).fetchall() - } - indexes = [ - ("idx_turns_game_turn", "turns", "game_id, turn"), - ("idx_games_status", "games", "status"), - ("idx_snake_turns_game_turn", "snake_turns", "game_id, turn"), - ] - for idx_name, table, cols in indexes: - if table in real_tables: - connection.execute(f"CREATE INDEX IF NOT EXISTS {idx_name} ON {table}({cols})") + # ── record helpers ───────────────────────────────────────────────────────── - def _ensure_column_exists(self, connection:sqlite3.Connection, table_name:str, column_name:str, column_type:str) -> None: - obj = connection.execute( - "SELECT type FROM sqlite_master WHERE name = ?", (table_name,) - ).fetchone() + def _running_game(self, connection:sqlite3.Connection, game_id:str) -> RunningGame: + game = self._running.get(game_id) + if game is None: + row = connection.execute("SELECT gid FROM games WHERE game_id = ?", (game_id,)).fetchone() + if row is None: + raise LookupError(f"Turn for unknown game {game_id!r}") + game = RunningGame(gid=row["gid"]) + return game - if obj and obj["type"] == "view": - underlying = f"_{table_name}_zstd" - exists = connection.execute( - "SELECT 1 FROM sqlite_master WHERE name = ? AND type = 'table'", (underlying,) + def _upsert_identities(self, connection:sqlite3.Connection, game_id:str, identities:list[SnakeIdentity]) -> dict[str, int]: + gsids = {} + for identity in identities: + row = connection.execute(""" + INSERT INTO game_snakes (game_id, snake_id, snake_name, is_you, customizations_json, gsid) + VALUES (?, ?, ?, ?, ?, (SELECT COALESCE(MAX(gsid), 0) + 1 FROM game_snakes)) + ON CONFLICT(game_id, snake_id) DO UPDATE SET + snake_name = excluded.snake_name, + is_you = excluded.is_you, + customizations_json = excluded.customizations_json + RETURNING gsid + """, + ( + game_id, identity.snake_id, identity.name, 1 if identity.is_you else 0, + self._to_json(identity.customizations or {}), + ), ).fetchone() - if not exists: - return - actual_table = underlying - else: - actual_table = table_name + gsids[identity.snake_id] = row["gsid"] + return gsids - existing = connection.execute(f"PRAGMA table_info({actual_table})").fetchall() - if any(row["name"] == column_name for row in existing): - return - connection.execute(f"ALTER TABLE {actual_table} ADD COLUMN {column_name} {column_type}") - - def _enable_zstd_compression(self, connection:sqlite3.Connection) -> None: - compressed_columns = [ - ("turns", "board_state_json"), - ("turns", "snakes_json"), - ("turns", "you_json"), - ("turns", "food_json"), - ("turns", "hazards_json"), - ("snake_turns", "body_json"), - ] - for table, column in compressed_columns: - try: - connection.execute( - "SELECT zstd_enable_transparent(?)", - [json.dumps({"table": table, "column": column, "compression_level": 6, "dict_chooser": "'a'"})], + def _write_rows(self, connection:sqlite3.Connection, gid:int, gsids:dict[str, int], turns:list[TurnRecord], states:list[SnakeState]) -> None: + connection.executemany(""" + INSERT INTO game_turns (gid, turn, observed_at, my_move, thinking_json, food, hazards) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(gid, turn) DO UPDATE SET + observed_at = excluded.observed_at, + my_move = excluded.my_move, + thinking_json = excluded.thinking_json, + food = excluded.food, + hazards = excluded.hazards + """, + [ + ( + gid, record.turn, self._ts_to_str(record.observed_at) or self._utc_now(), + encode_move(record.move), + self._to_json(record.thinking) if record.thinking is not None else None, + encode_points(record.food), encode_points(record.hazards), ) - except sqlite3.OperationalError: - pass - connection.execute("SELECT zstd_incremental_maintenance(null, 1)") + for record in turns + ], + ) + connection.executemany(""" + INSERT INTO snake_states (gsid, turn, health, latency, body) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(gsid, turn) DO UPDATE SET + health = excluded.health, + latency = excluded.latency, + body = excluded.body + """, + [ + (gsids[state.snake_id], state.turn, state.health, state.latency, encode_body(state.body)) + for state in states if state.snake_id in gsids + ], + ) + + def _load_records(self, connection:sqlite3.Connection, game_id:str, gid:int) -> tuple[list[SnakeIdentity], list[TurnRecord], list[SnakeState]]: + identities = [ + SnakeIdentity(row["snake_id"], row["snake_name"], bool(row["is_you"]), self._from_json(row["customizations_json"]) or {}) + for row in connection.execute(""" + SELECT snake_id, snake_name, is_you, customizations_json + FROM game_snakes WHERE game_id = ? + """, (game_id,)) + ] + turns = [ + TurnRecord( + turn=row["turn"], observed_at=row["observed_at"], move=decode_move(row["my_move"]), + thinking=self._from_json(row["thinking_json"]), + food=decode_points(row["food"]), hazards=decode_points(row["hazards"]), + ) + for row in connection.execute(""" + SELECT turn, observed_at, my_move, thinking_json, food, hazards + FROM game_turns WHERE gid = ? ORDER BY turn + """, (gid,)) + ] + states = [ + SnakeState(row["snake_id"], row["turn"], row["health"], row["latency"], decode_body(row["body"])) + for row in connection.execute(""" + SELECT gs.snake_id, s.turn, s.health, s.latency, s.body + FROM game_snakes AS gs + JOIN snake_states AS s ON s.gsid = gs.gsid + WHERE gs.game_id = ? + """, (game_id,)) + ] + blob = connection.execute("SELECT format, data FROM game_replays WHERE gid = ?", (gid,)).fetchone() + if blob is not None: + packed_turns, packed_states = unpack_replay(blob["data"], blob["format"]) + turns, states = merge_records(packed_turns, packed_states, turns, states) + return identities, turns, states + + def _store_replay(self, connection:sqlite3.Connection, game_id:str, gid:int, turns:list[TurnRecord], states:list[SnakeState]) -> None: + connection.execute(""" + INSERT INTO game_replays (gid, format, turn_count, data) VALUES (?, ?, ?, ?) + ON CONFLICT(gid) DO UPDATE SET + format = excluded.format, turn_count = excluded.turn_count, data = excluded.data + """, (gid, REPLAY_FORMAT, len(turns), pack_replay(turns, states))) + connection.execute("DELETE FROM game_turns WHERE gid = ?", (gid,)) + connection.execute( + "DELETE FROM snake_states WHERE gsid IN (SELECT gsid FROM game_snakes WHERE game_id = ?)", + (game_id,), + ) + + def _pack_game(self, connection:sqlite3.Connection, game_id:str) -> None: + connection.execute("BEGIN IMMEDIATE") + try: + row = connection.execute("SELECT gid FROM games WHERE game_id = ?", (game_id,)).fetchone() + if row is not None and connection.execute( + "SELECT 1 FROM game_turns WHERE gid = ? LIMIT 1", (row["gid"],), + ).fetchone(): + _, turns, states = self._load_records(connection, game_id, row["gid"]) + self._store_replay(connection, game_id, row["gid"], turns, states) + connection.execute("COMMIT") + except Exception: + connection.execute("ROLLBACK") + raise # ── sync write methods ───────────────────────────────────────────────────── @@ -216,13 +220,15 @@ class SqliteGameplayBackend(GameplayBackendTemplate): ruleset = game.get("ruleset", {}) game_type = self._derive_game_type(board, ruleset) - with self._connect() as connection: + connection = self._connect() + try: connection.execute(""" INSERT INTO games ( game_id, started_at, width, height, source, map_name, ruleset_name, ruleset_version, your_snake_id, your_snake_name, - your_snake_type, your_snake_version, game_type, status - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'running') + your_snake_type, your_snake_version, game_type, status, gid + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'running', + (SELECT COALESCE(MAX(gid), 0) + 1 FROM games)) ON CONFLICT(game_id) DO UPDATE SET width = excluded.width, height = excluded.height, @@ -238,166 +244,71 @@ class SqliteGameplayBackend(GameplayBackendTemplate): status = 'running' """, ( - game.get("id"), - self._utc_now(), - board.get("width"), - board.get("height"), - game.get("source"), - game.get("map"), - ruleset.get("name"), - ruleset.get("version"), - you.get("id"), - you.get("name"), - snake_type, - snake_version, - game_type, + game.get("id"), self._utc_now(), board.get("width"), board.get("height"), + game.get("source"), game.get("map"), ruleset.get("name"), ruleset.get("version"), + you.get("id"), you.get("name"), snake_type, snake_version, game_type, ), ) connection.execute("PRAGMA wal_checkpoint(PASSIVE)") - connection.execute("PRAGMA optimize") + finally: + connection.close() def _record_turn_sync(self, game_state:dict, my_move:str|None, my_thinking:dict|None) -> None: - game = game_state.get("game", {}) - board = game_state.get("board", {}) - snakes = self._extract_snakes(game_state) - you = self._extract_you(game_state) - game_id = game.get("id") - turn = int(game_state.get("turn", 0)) - - board_json, snakes_json, you_json, food_json, hazards_json = compact_turn_json(board) - - with self._connect() as connection: - connection.executemany(""" - INSERT INTO game_snakes ( - game_id, snake_id, snake_name, is_you, customizations_json - ) VALUES (?, ?, ?, ?, ?) - ON CONFLICT(game_id, snake_id) DO UPDATE SET - snake_name = excluded.snake_name, - is_you = excluded.is_you, - customizations_json = excluded.customizations_json - """, - [ - ( - game_id, - snake.get("id"), - snake.get("name"), - 1 if snake.get("id") == you.get("id") else 0, - self._to_json(snake.get("customizations") or {}), - ) - for snake in snakes if snake.get("id") is not None - ], - ) - connection.execute(""" - INSERT INTO turns ( - game_id, turn, observed_at, my_move, my_thinking_json, - board_state_json, snakes_json, you_json, food_json, hazards_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(game_id, turn) DO UPDATE SET - observed_at = excluded.observed_at, - my_move = excluded.my_move, - my_thinking_json = excluded.my_thinking_json, - board_state_json = excluded.board_state_json, - snakes_json = excluded.snakes_json, - you_json = excluded.you_json, - food_json = excluded.food_json, - hazards_json = excluded.hazards_json - """, - ( - game_id, - turn, - self._utc_now(), - my_move, - self._to_json(my_thinking) if my_thinking is not None else None, - self._to_json(board_json), - self._to_json(snakes_json), - self._to_json(you_json), - self._to_json(food_json), - self._to_json(hazards_json), - ), - ) - - previous_positions: dict[str, tuple[int, int]] = {} - if turn > 0: - previous_rows = connection.execute(""" - SELECT snake_id, head_x, head_y - FROM snake_turns - WHERE game_id = ? AND turn = ? - """, - (game_id, turn - 1), - ).fetchall() - previous_positions = { - row["snake_id"]: (int(row["head_x"]), int(row["head_y"])) - for row in previous_rows - if row["head_x"] is not None and row["head_y"] is not None - } - - you_id = you.get("id") - for params in self._build_snake_turn_params(snakes, you_id, game_id, turn, previous_positions): - (p_game_id, p_turn, p_snake_id, p_name, p_health, p_length, - p_head_x, p_head_y, p_body, p_is_you, p_inferred, p_latency) = params - connection.execute(""" - INSERT INTO snake_turns ( - game_id, turn, snake_id, snake_name, health, length, - head_x, head_y, body_json, is_you, inferred_move, latency - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(game_id, turn, snake_id) DO UPDATE SET - snake_name = excluded.snake_name, - health = excluded.health, - length = excluded.length, - head_x = excluded.head_x, - head_y = excluded.head_y, - body_json = excluded.body_json, - is_you = excluded.is_you, - inferred_move = excluded.inferred_move, - latency = excluded.latency - """, - ( - p_game_id, p_turn, p_snake_id, None, p_health, p_length, - p_head_x, p_head_y, self._to_json(p_body), - 0, - p_inferred, p_latency, - ), + game_id = game_state.get("game", {}).get("id") + connection = self._connect() + try: + connection.execute("BEGIN IMMEDIATE") + try: + running = self._running_game(connection, game_id) + record, states, identities, points = records_from_game_state( + game_state, my_move, my_thinking, running.points, ) - - connection.execute(""" - UPDATE games - SET final_turn = CASE WHEN ? > final_turn THEN ? ELSE final_turn END - WHERE game_id = ? - """, - (turn, turn, game_id), - ) + if any(identity.snake_id not in running.snake_ids for identity in identities): + running.snake_ids.update(self._upsert_identities(connection, game_id, identities)) + self._write_rows(connection, running.gid, running.snake_ids, [record], states) + connection.execute( + "UPDATE games SET final_turn = MAX(final_turn, ?) WHERE gid = ?", + (record.turn, running.gid), + ) + connection.execute("COMMIT") + except Exception: + connection.execute("ROLLBACK") + raise + finally: + connection.close() + running.points = points + self._running.put(game_id, running) def _record_game_end_sync(self, game_state:dict) -> None: game_id, winner_name, winner_you, turn = self._extract_game_end_params(game_state) - with self._connect() as connection: + connection = self._connect() + try: connection.execute(""" UPDATE games SET ended_at = ?, winner_name = ?, winner_you = ?, - final_turn = CASE WHEN ? > final_turn THEN ? ELSE final_turn END, + final_turn = MAX(final_turn, ?), status = 'finished' WHERE game_id = ? """, - ( - self._utc_now(), - winner_name, - 1 if winner_you else 0, - turn, - turn, - game_id, - ), + (self._utc_now(), winner_name, 1 if winner_you else 0, turn, game_id), ) + self._pack_game(connection, game_id) + finally: + connection.close() + self._running.drop(game_id) def _finalize_stale_running_games_sync(self, stale_after_seconds:int=600) -> int: threshold = max(60, int(stale_after_seconds)) now_utc = datetime.now(timezone.utc) finalized = 0 - with self._connect() as connection: + connection = self._connect() + try: rows = connection.execute(""" - SELECT game_id, started_at, final_turn, your_snake_id + SELECT gid, game_id, started_at, final_turn, your_snake_id FROM games WHERE status = 'running' ORDER BY started_at ASC @@ -405,73 +316,47 @@ class SqliteGameplayBackend(GameplayBackendTemplate): for row in rows: started_at = self._parse_utc_timestamp(row["started_at"]) - if started_at is None: - continue - if (now_utc - started_at).total_seconds() < threshold: + if started_at is None or (now_utc - started_at).total_seconds() < threshold: continue game_id = row["game_id"] - your_snake_id = row["your_snake_id"] - final_turn = int(row["final_turn"] or 0) - - snake_rows = connection.execute(""" - SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name) AS snake_name - FROM snake_turns AS st - LEFT JOIN game_snakes AS gs - ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id - WHERE st.game_id = ? AND st.turn = ? - ORDER BY COALESCE(gs.is_you, st.is_you) DESC, snake_name ASC - """, - (game_id, final_turn), - ).fetchall() - - if len(snake_rows) == 0: - latest_row = connection.execute( - "SELECT MAX(turn) AS latest_turn FROM snake_turns WHERE game_id = ?", - (game_id,), - ).fetchone() - if latest_row is not None and latest_row["latest_turn"] is not None: - final_turn = int(latest_row["latest_turn"]) - snake_rows = connection.execute(""" - SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name) AS snake_name - FROM snake_turns AS st - LEFT JOIN game_snakes AS gs - ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id - WHERE st.game_id = ? AND st.turn = ? - ORDER BY COALESCE(gs.is_you, st.is_you) DESC, snake_name ASC - """, - (game_id, final_turn), - ).fetchall() - - winner_you, survivor_name = self._calculate_survivor(snake_rows, your_snake_id) + identities, _, states = self._load_records(connection, game_id, row["gid"]) + final_turn, survivors = self._survivors(identities, states, int(row["final_turn"] or 0)) + winner_you, survivor_name = self._calculate_survivor(survivors, row["your_snake_id"]) result = connection.execute(""" UPDATE games SET ended_at = ?, winner_name = ?, winner_you = ?, - final_turn = CASE WHEN ? > final_turn THEN ? ELSE final_turn END, + final_turn = MAX(final_turn, ?), status = 'finished' WHERE game_id = ? AND status = 'running' """, - ( - self._utc_now(), - survivor_name, - 1 if winner_you else 0, - final_turn, - final_turn, - game_id, - ), + (self._utc_now(), survivor_name, 1 if winner_you else 0, final_turn, game_id), ) if result.rowcount > 0: finalized += 1 + self._pack_game(connection, game_id) + self._running.drop(game_id) + + late = connection.execute(""" + SELECT DISTINCT g.game_id + FROM game_turns AS t JOIN games AS g ON g.gid = t.gid + WHERE g.status = 'finished' + """).fetchall() + for late_row in late: + self._pack_game(connection, late_row["game_id"]) + finally: + connection.close() return finalized # ── sync read methods ────────────────────────────────────────────────────── def _get_summary_sync(self, recent_limit:int=15) -> dict: - with self._connect() as connection: + connection = self._connect() + try: totals = connection.execute(""" SELECT COUNT(*) AS total_games, @@ -505,11 +390,14 @@ class SqliteGameplayBackend(GameplayBackendTemplate): """, (max(1, int(recent_limit)),), ).fetchall() + finally: + connection.close() return self._build_summary_output(totals, by_type, recent) def _list_games_sync(self, limit:int=50) -> list[dict]: - with self._connect() as connection: + connection = self._connect() + try: rows = connection.execute(""" SELECT game_id, started_at, ended_at, map_name, source, ruleset_name, game_type, your_snake_name, your_snake_type, your_snake_version, @@ -521,52 +409,75 @@ class SqliteGameplayBackend(GameplayBackendTemplate): """, (max(1, int(limit)),), ).fetchall() + finally: + connection.close() return self._build_game_list_output(rows) def _get_game_replay_sync(self, game_id:str) -> dict|None: - with self._connect() as connection: - game_row = connection.execute(""" - SELECT game_id, started_at, ended_at, width, height, source, map_name, - ruleset_name, ruleset_version, game_type, your_snake_id, your_snake_name, - your_snake_type, your_snake_version, - winner_name, winner_you, final_turn, status - FROM games - WHERE game_id = ? AND has_replay = 1 - """, - (game_id,), - ).fetchone() - + connection = self._connect() + try: + game_row = connection.execute(_GAME_SELECT + " WHERE game_id = ? AND has_replay = 1", (game_id,)).fetchone() if game_row is None: return None + identities, turns, states = self._load_records(connection, game_id, game_row["gid"]) + finally: + connection.close() - turn_rows = connection.execute(""" - SELECT turn, observed_at, my_move, my_thinking_json, - board_state_json, food_json, hazards_json, you_json - FROM turns - WHERE game_id = ? - ORDER BY turn ASC - """, - (game_id,), - ).fetchall() + return self._build_game_replay_output(game_row, identities, turns, states) - snake_rows = connection.execute(""" - SELECT st.turn, st.snake_id, - COALESCE(gs.snake_name, st.snake_name) AS snake_name, - st.health, st.length, st.head_x, st.head_y, st.body_json, - COALESCE(gs.is_you, st.is_you) AS is_you, - COALESCE(gs.customizations_json, '{}') AS customizations_json, - st.inferred_move, st.latency - FROM snake_turns AS st - LEFT JOIN game_snakes AS gs - ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id - WHERE st.game_id = ? - ORDER BY st.turn ASC, is_you DESC, snake_name ASC - """, - (game_id,), - ).fetchall() + # ── bundles (migration / merge) ──────────────────────────────────────────── - return self._build_game_replay_output(game_row, turn_rows, snake_rows, decode_json=self._from_json) + def list_game_ids_sync(self) -> list[str]: + connection = self._connect() + try: + return [row[0] for row in connection.execute("SELECT game_id FROM games ORDER BY started_at, game_id")] + finally: + connection.close() + + def read_game_bundle_sync(self, game_id:str) -> GameBundle|None: + connection = self._connect() + try: + row = connection.execute(_GAME_SELECT + " WHERE game_id = ?", (game_id,)).fetchone() + if row is None: + return None + identities, turns, states = self._load_records(connection, game_id, row["gid"]) + finally: + connection.close() + return GameBundle(_game_from_row(row), identities, turns, states) + + def import_game_bundle_sync(self, bundle:GameBundle) -> None: + game = normalize_game(bundle.game) + columns = [column for column in GAME_COLUMNS if column != "quality_reasons"] + ["quality_reasons_json"] + values = [game[column] for column in GAME_COLUMNS if column != "quality_reasons"] + values.append(self._to_json(game["quality_reasons"]) if game["quality_reasons"] is not None else None) + values = [self._ts_to_str(value) if isinstance(value, datetime) else value for value in values] + updates = ", ".join(f"{column} = excluded.{column}" for column in columns[1:]) + + connection = self._connect() + try: + connection.execute("BEGIN IMMEDIATE") + try: + connection.execute( + f"INSERT INTO games ({', '.join(columns)}, gid) " + f"VALUES ({', '.join('?' for _ in columns)}, (SELECT COALESCE(MAX(gid), 0) + 1 FROM games)) " + f"ON CONFLICT(game_id) DO UPDATE SET {updates}", + values, + ) + gid = connection.execute("SELECT gid FROM games WHERE game_id = ?", (game["game_id"],)).fetchone()["gid"] + gsids = self._upsert_identities(connection, game["game_id"], bundle.snakes) + if bundle.turns and game["has_replay"]: + if game["status"] == "finished": + self._store_replay(connection, game["game_id"], gid, bundle.turns, bundle.states) + else: + self._write_rows(connection, gid, gsids, bundle.turns, bundle.states) + connection.execute("COMMIT") + except Exception: + connection.execute("ROLLBACK") + raise + finally: + connection.close() + self._running.drop(game["game_id"]) # ── public async interface ───────────────────────────────────────────────── @@ -591,5 +502,14 @@ class SqliteGameplayBackend(GameplayBackendTemplate): async def get_game_replay(self, game_id:str) -> dict|None: return await asyncio.to_thread(self._get_game_replay_sync, game_id) + async def list_game_ids(self) -> list[str]: + return await asyncio.to_thread(self.list_game_ids_sync) + + async def read_game_bundle(self, game_id:str) -> GameBundle|None: + return await asyncio.to_thread(self.read_game_bundle_sync, game_id) + + async def import_game_bundle(self, bundle:GameBundle) -> None: + await asyncio.to_thread(self.import_game_bundle_sync, bundle) + async def close(self) -> None: return None diff --git a/server/database/backend/Template.py b/server/database/backend/Template.py index 40d9cff..2f560d2 100644 --- a/server/database/backend/Template.py +++ b/server/database/backend/Template.py @@ -2,7 +2,8 @@ import json from datetime import datetime, timezone from typing import Any -from server.database.normalized_turn import hydrate_replay_turns +from server.database.compact.hydrate import hydrate_replay_turns +from server.database.compact.records import GameBundle, SnakeIdentity, SnakeState, TurnRecord class GameplayBackendTemplate: """Abstract base for gameplay database backends. @@ -40,6 +41,17 @@ class GameplayBackendTemplate: async def get_game_replay(self, game_id:str) -> dict|None: raise NotImplementedError + async def read_game_bundle(self, game_id:str) -> GameBundle|None: + """Return all stored data of one game (used by migrations and merges).""" + raise NotImplementedError + + async def import_game_bundle(self, bundle:GameBundle) -> None: + """Store a complete game; finished games are packed into a replay blob.""" + raise NotImplementedError + + async def list_game_ids(self) -> list[str]: + raise NotImplementedError + async def close(self) -> None: return None @@ -79,22 +91,6 @@ class GameplayBackendTemplate: def _extract_you(self, game_state:dict) -> dict: return dict(game_state.get("you", {})) - def _infer_direction(self, old_head:tuple[int, int]|None, new_head:tuple[int, int]|None) -> str|None: - if old_head is None or new_head is None: - return None - - dx = new_head[0] - old_head[0] - dy = new_head[1] - old_head[1] - if dx == 1 and dy == 0: - return "right" - if dx == -1 and dy == 0: - return "left" - if dx == 0 and dy == 1: - return "up" - if dx == 0 and dy == -1: - return "down" - return None - def _derive_game_type(self, board:dict, ruleset:dict) -> str: if len(board.get("snakes", [])) == 2: return "duel" @@ -159,14 +155,21 @@ class GameplayBackendTemplate: "status": row["status"], } for row in rows] - def _build_game_replay_output(self, game_row, turn_rows, snake_rows, decode_json) -> dict: - """Build the full replay dict. - - decode_json: callable applied to raw column values that may be JSON strings - (SQLite) or already-decoded objects (PostgreSQL). Pass self._from_json for - SQLite; pass (lambda x: x) for PostgreSQL. - """ - hydrated_turns = hydrate_replay_turns(game_row, turn_rows, snake_rows, decode_json) + def _build_game_replay_output( + self, + game_row, + identities:list[SnakeIdentity], + turns:list[TurnRecord], + states:list[SnakeState], + ) -> dict: + game = { + "width": game_row["width"], + "height": game_row["height"], + "your_snake_id": game_row["your_snake_id"], + } + hydrated_turns = hydrate_replay_turns( + game, {identity.snake_id: identity for identity in identities}, turns, states, + ) return { "game": { @@ -197,45 +200,6 @@ class GameplayBackendTemplate: # ── shared write-path helpers ────────────────────────────────────────────── - def _build_snake_turn_params( - self, - snakes:list[dict], - you_id:str|None, - game_id:str|None, - turn:int, - previous_positions:dict, - ) -> list[tuple]: - """Return one parameter tuple per snake for a snake_turns INSERT.""" - result = [] - for snake in snakes: - snake_id = snake.get("id") - if snake_id is None: - continue - head = snake.get("head", {}) - head_x = head.get("x") - head_y = head.get("y") - new_head = ( - (int(head_x), int(head_y)) - if head_x is not None and head_y is not None - else None - ) - inferred = self._infer_direction(previous_positions.get(snake_id), new_head) - result.append(( - game_id, - turn, - snake_id, - snake.get("name"), - snake.get("health"), - snake.get("length"), - head_x, - head_y, - snake.get("body", []), - snake_id == you_id, - inferred, - snake.get("latency"), - )) - return result - def _extract_game_end_params(self, game_state:dict) -> tuple: """Return (game_id, winner_name, winner_you, turn) from a game_end payload.""" game = game_state.get("game", {}) @@ -249,6 +213,25 @@ class GameplayBackendTemplate: turn = int(game_state.get("turn", 0)) return game_id, winner_name, winner_you, turn + def _survivors(self, identities:list[SnakeIdentity], states:list[SnakeState], final_turn:int) -> tuple[int, list[dict]]: + """Snakes alive at the final (or latest stored) turn, you first, then by name.""" + turns = {state.turn for state in states} + if final_turn not in turns and turns: + final_turn = max(turns) + by_id = {identity.snake_id: identity for identity in identities} + alive = [] + for state in states: + if state.turn != final_turn: + continue + identity = by_id.get(state.snake_id) + alive.append({ + "snake_id": state.snake_id, + "snake_name": identity.name if identity else None, + "is_you": bool(identity and identity.is_you), + }) + alive.sort(key=lambda row: (not row["is_you"], row["snake_name"] is None, row["snake_name"] or "")) + return final_turn, alive + def _calculate_survivor(self, snake_rows, your_snake_id:str|None) -> tuple[bool, str|None]: """Return (winner_you, survivor_name) from a list of snake_turns rows.""" survivor_ids = [s["snake_id"] for s in snake_rows if s["snake_id"]] diff --git a/server/database/backend/postgresql_schema.py b/server/database/backend/postgresql_schema.py new file mode 100644 index 0000000..fb40442 --- /dev/null +++ b/server/database/backend/postgresql_schema.py @@ -0,0 +1,106 @@ +"""PostgreSQL schema for compact gameplay storage. + +Finished games live in ``game_replays`` as one zstd blob per game. Running +games use ``game_turns`` and ``snake_states`` keyed by small integer ids +(``games.gid`` / ``game_snakes.gsid``) instead of repeating text ids per row. +""" + +TABLES_DDL = """ +CREATE TABLE IF NOT EXISTS games ( + game_id TEXT PRIMARY KEY, + started_at TIMESTAMPTZ NOT NULL, + ended_at TIMESTAMPTZ, + width INTEGER, + height INTEGER, + source TEXT, + map_name TEXT, + ruleset_name TEXT, + ruleset_version TEXT, + your_snake_id TEXT, + your_snake_name TEXT, + your_snake_type TEXT, + your_snake_version TEXT, + game_type TEXT, + winner_name TEXT, + winner_you BOOLEAN NOT NULL DEFAULT FALSE, + final_turn INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'running', + has_replay BOOLEAN NOT NULL DEFAULT TRUE, + quality_status TEXT NOT NULL DEFAULT 'retained', + quality_score INTEGER, + quality_tier TEXT, + quality_reasons JSONB +); + +CREATE TABLE IF NOT EXISTS game_snakes ( + game_id TEXT NOT NULL REFERENCES games(game_id) ON DELETE CASCADE, + snake_id TEXT NOT NULL, + snake_name TEXT, + is_you BOOLEAN NOT NULL DEFAULT FALSE, + customizations JSONB NOT NULL DEFAULT '{}'::jsonb, + PRIMARY KEY (game_id, snake_id) +); + +ALTER TABLE games ADD COLUMN IF NOT EXISTS game_type TEXT; +ALTER TABLE games ADD COLUMN IF NOT EXISTS your_snake_type TEXT; +ALTER TABLE games ADD COLUMN IF NOT EXISTS your_snake_version TEXT; +ALTER TABLE games ADD COLUMN IF NOT EXISTS winner_name TEXT; +ALTER TABLE games ADD COLUMN IF NOT EXISTS has_replay BOOLEAN NOT NULL DEFAULT TRUE; +ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_status TEXT NOT NULL DEFAULT 'retained'; +ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_score INTEGER; +ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_tier TEXT; +ALTER TABLE games ADD COLUMN IF NOT EXISTS quality_reasons JSONB; +ALTER TABLE game_snakes ADD COLUMN IF NOT EXISTS customizations JSONB NOT NULL DEFAULT '{}'::jsonb; + +ALTER TABLE games ADD COLUMN IF NOT EXISTS gid INTEGER GENERATED BY DEFAULT AS IDENTITY; +ALTER TABLE game_snakes ADD COLUMN IF NOT EXISTS gsid INTEGER GENERATED BY DEFAULT AS IDENTITY; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'games_gid_key') THEN + ALTER TABLE games ADD CONSTRAINT games_gid_key UNIQUE (gid); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'game_snakes_gsid_key') THEN + ALTER TABLE game_snakes ADD CONSTRAINT game_snakes_gsid_key UNIQUE (gsid); + END IF; +END $$; + +CREATE TABLE IF NOT EXISTS game_turns ( + gid INTEGER NOT NULL REFERENCES games(gid) ON DELETE CASCADE, + turn INTEGER NOT NULL, + observed_at TIMESTAMPTZ NOT NULL, + my_move SMALLINT, + thinking JSONB, + food BYTEA, + hazards BYTEA, + PRIMARY KEY (gid, turn) +); + +CREATE TABLE IF NOT EXISTS snake_states ( + gsid INTEGER NOT NULL REFERENCES game_snakes(gsid) ON DELETE CASCADE, + turn INTEGER NOT NULL, + health SMALLINT, + latency SMALLINT, + body BYTEA NOT NULL, + PRIMARY KEY (gsid, turn) +); + +CREATE TABLE IF NOT EXISTS game_replays ( + gid INTEGER PRIMARY KEY REFERENCES games(gid) ON DELETE CASCADE, + format SMALLINT NOT NULL, + turn_count INTEGER NOT NULL, + data BYTEA NOT NULL +); + +-- Replay blobs are already zstd-compressed; skip TOAST compression attempts. +ALTER TABLE game_replays ALTER COLUMN data SET STORAGE EXTERNAL; + +CREATE INDEX IF NOT EXISTS idx_games_status ON games(status); +""" + +# Superseded by the UNIQUE constraints of the legacy turns/snake_turns tables. +# CONCURRENTLY avoids blocking writes; each must run outside a transaction. +DROP_REDUNDANT_INDEXES = ( + "DROP INDEX CONCURRENTLY IF EXISTS idx_turns_game_turn", + "DROP INDEX CONCURRENTLY IF EXISTS idx_snake_turns_game_turn", +) diff --git a/server/database/backend/sqlite_schema.py b/server/database/backend/sqlite_schema.py new file mode 100644 index 0000000..47294ce --- /dev/null +++ b/server/database/backend/sqlite_schema.py @@ -0,0 +1,114 @@ +"""SQLite schema for compact gameplay storage (mirrors postgresql_schema).""" + +import sqlite3 + +BASE_TABLES = """ +CREATE TABLE IF NOT EXISTS games ( + game_id TEXT PRIMARY KEY, + started_at TEXT NOT NULL, + ended_at TEXT, + width INTEGER, + height INTEGER, + source TEXT, + map_name TEXT, + ruleset_name TEXT, + ruleset_version TEXT, + your_snake_id TEXT, + your_snake_name TEXT, + your_snake_type TEXT, + your_snake_version TEXT, + game_type TEXT, + winner_name TEXT, + winner_you INTEGER NOT NULL DEFAULT 0, + final_turn INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'running', + has_replay INTEGER NOT NULL DEFAULT 1, + quality_status TEXT NOT NULL DEFAULT 'retained', + quality_score INTEGER, + quality_tier TEXT, + quality_reasons_json TEXT, + gid INTEGER +); + +CREATE TABLE IF NOT EXISTS game_snakes ( + game_id TEXT NOT NULL, + snake_id TEXT NOT NULL, + snake_name TEXT, + is_you INTEGER NOT NULL DEFAULT 0, + customizations_json TEXT NOT NULL DEFAULT '{}', + gsid INTEGER, + PRIMARY KEY (game_id, snake_id), + FOREIGN KEY (game_id) REFERENCES games(game_id) ON DELETE CASCADE +); +""" + +# Columns added over time; older databases get them via ALTER TABLE. +LEGACY_COLUMNS = ( + ("games", "your_snake_type", "TEXT"), + ("games", "your_snake_version", "TEXT"), + ("games", "game_type", "TEXT"), + ("games", "winner_name", "TEXT"), + ("games", "has_replay", "INTEGER NOT NULL DEFAULT 1"), + ("games", "quality_status", "TEXT NOT NULL DEFAULT 'retained'"), + ("games", "quality_score", "INTEGER"), + ("games", "quality_tier", "TEXT"), + ("games", "quality_reasons_json", "TEXT"), + ("games", "gid", "INTEGER"), + ("game_snakes", "customizations_json", "TEXT NOT NULL DEFAULT '{}'"), + ("game_snakes", "gsid", "INTEGER"), +) + +COMPACT_TABLES = """ +CREATE UNIQUE INDEX IF NOT EXISTS games_gid_key ON games(gid); +CREATE UNIQUE INDEX IF NOT EXISTS game_snakes_gsid_key ON game_snakes(gsid); +CREATE INDEX IF NOT EXISTS idx_games_status ON games(status); + +UPDATE games +SET gid = rowid + (SELECT COALESCE(MAX(gid), 0) FROM games) +WHERE gid IS NULL; + +UPDATE game_snakes +SET gsid = rowid + (SELECT COALESCE(MAX(gsid), 0) FROM game_snakes) +WHERE gsid IS NULL; + +CREATE TABLE IF NOT EXISTS game_turns ( + gid INTEGER NOT NULL REFERENCES games(gid) ON DELETE CASCADE, + turn INTEGER NOT NULL, + observed_at TEXT NOT NULL, + my_move INTEGER, + thinking_json TEXT, + food BLOB, + hazards BLOB, + PRIMARY KEY (gid, turn) +) WITHOUT ROWID; + +CREATE TABLE IF NOT EXISTS snake_states ( + gsid INTEGER NOT NULL REFERENCES game_snakes(gsid) ON DELETE CASCADE, + turn INTEGER NOT NULL, + health INTEGER, + latency INTEGER, + body BLOB NOT NULL, + PRIMARY KEY (gsid, turn) +) WITHOUT ROWID; + +CREATE TABLE IF NOT EXISTS game_replays ( + gid INTEGER PRIMARY KEY REFERENCES games(gid) ON DELETE CASCADE, + format INTEGER NOT NULL, + turn_count INTEGER NOT NULL, + data BLOB NOT NULL +); + +DROP INDEX IF EXISTS idx_turns_game_turn; +DROP INDEX IF EXISTS idx_snake_turns_game_turn; +""" + +def ensure_column(connection:sqlite3.Connection, table:str, column:str, column_type:str) -> None: + existing = {row[1] for row in connection.execute(f"PRAGMA table_info({table})")} + if column not in existing: + connection.execute(f"ALTER TABLE {table} ADD COLUMN {column} {column_type}") + +def create_schema(connection:sqlite3.Connection) -> None: + connection.executescript(BASE_TABLES) + for table, column, column_type in LEGACY_COLUMNS: + ensure_column(connection, table, column, column_type) + connection.executescript(COMPACT_TABLES) diff --git a/server/database/benchmark_states.py b/server/database/benchmark_states.py index c4b2989..e734fda 100644 --- a/server/database/benchmark_states.py +++ b/server/database/benchmark_states.py @@ -1,108 +1,57 @@ -"""Load sampled gameplay positions from SQLite or PostgreSQL.""" +"""Load sampled gameplay positions from a SQLite or PostgreSQL gameplay DB.""" import asyncio -import json from urllib.parse import urlparse -def is_postgresql_source(source: str) -> bool: +def is_postgresql_source(source:str) -> bool: return urlparse(source).scheme.lower() in {"postgres", "postgresql"} -def _decode_json(value, default): - if value is None: - return default - if isinstance(value, str): - return json.loads(value) - return value - -def _build_state(row, snake_rows) -> tuple[dict, dict] | None: - board = _decode_json(row[1], {}) - you = _decode_json(row[2], {}) - if not board.get("snakes"): - snakes = [] - for snake_row in snake_rows: - snake_id = snake_row[0] - snake_name = snake_row[1] or (row[6] if snake_id == row[5] else snake_id) - snakes.append({ - "id": snake_id, - "name": snake_name, - "health": snake_row[2], - "length": snake_row[3], - "head": {"x": snake_row[4], "y": snake_row[5]}, - "body": _decode_json(snake_row[6], []), - "customizations": _decode_json(snake_row[7], {}), - }) - board = { - "width": row[7], - "height": row[8], - "food": _decode_json(row[3], []), - "hazards": _decode_json(row[4], []), - "snakes": snakes, - } - if not you: - you = next( - (snake for snake in board.get("snakes", []) if snake.get("id") == row[5]), - {}, - ) +def replay_turn_to_state(game:dict, turn:dict) -> tuple[dict, dict]|None: + board = turn.get("board") or {} + you = turn.get("you") or {} if not you or not board.get("snakes"): return None return board, { "you": you, - "game_id": row[9], - "source": row[10] or "custom", - "map": row[11] or "standard", + "game_id": game["game_id"], + "source": game.get("source") or "custom", + "map": game.get("map") or "standard", "ruleset": { - "name": row[12] or "standard", - "version": row[13] or "v1.0.0", + "name": game.get("ruleset_name") or "standard", + "version": game.get("ruleset_version") or "v1.0.0", "settings": {}, }, - "turn": int(row[14]), + "turn": int(turn["turn"]), } -async def _load_postgresql_states(dsn: str, samples: int, stride: int) -> list[tuple[dict, dict]]: - try: - import asyncpg - except ImportError as exc: - raise RuntimeError("asyncpg is required for PostgreSQL benchmark sources") from exc +async def sample_states(backend, samples:int, stride:int) -> list[tuple[dict, dict]]: + """Every ``stride``-th stored turn, newest games first.""" + states = [] + seen = 0 + for game_id in reversed(await backend.list_game_ids()): + replay = await backend.get_game_replay(game_id) + if replay is None: + continue + for turn in reversed(replay["turns"]): + if seen % stride == 0: + state = replay_turn_to_state(replay["game"], turn) + if state is not None: + states.append(state) + if len(states) >= samples: + return states + seen += 1 + return states - connection = await asyncpg.connect(dsn=dsn) +async def _load_states(source:str, samples:int, stride:int) -> list[tuple[dict, dict]]: + if is_postgresql_source(source): + from server.database.backend.PostgresqlGameplayBackend import PostgresqlGameplayBackend # noqa: PLC0415 + backend = PostgresqlGameplayBackend(source) + else: + from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend # noqa: PLC0415 + backend = SqliteGameplayBackend(source, read_only=True) try: - max_id = int(await connection.fetchval("SELECT max(id) FROM turns") or 0) - if max_id == 0: - return [] - query = """ - SELECT t.id, t.board_state, t.you, t.food, t.hazards, - g.your_snake_id, g.your_snake_name, g.width, g.height, - g.game_id, g.source, g.map_name, - g.ruleset_name, g.ruleset_version, t.turn - FROM turns AS t - JOIN games AS g ON g.game_id = t.game_id - WHERE t.id >= $1 - ORDER BY t.id - LIMIT 1 - """ - snake_query = """ - SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name), - st.health, st.length, st.head_x, st.head_y, st.body, - COALESCE(gs.customizations, '{}'::jsonb) - FROM snake_turns AS st - LEFT JOIN game_snakes AS gs - ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id - WHERE st.game_id = $1 AND st.turn = $2 - ORDER BY st.id - """ - states = [] - next_id = max(1, max_id - (samples - 1) * stride) - while len(states) < samples and next_id <= max_id: - row = await connection.fetchrow(query, next_id) - if row is None: - break - snake_rows = await connection.fetch(snake_query, row[9], row[14]) - state = _build_state(row, snake_rows) - if state is not None: - states.append(state) - next_id = int(row[0]) + stride - return states + return await sample_states(backend, samples, stride) finally: - await connection.close() + await backend.close() -def load_postgresql_states(dsn: str, samples: int, stride: int) -> list[tuple[dict, dict]]: - return asyncio.run(_load_postgresql_states(dsn, samples, stride)) +def load_states(source:str, samples:int, stride:int) -> list[tuple[dict, dict]]: + return asyncio.run(_load_states(source, max(1, samples), max(1, stride))) diff --git a/server/database/compact/__init__.py b/server/database/compact/__init__.py new file mode 100644 index 0000000..86759e1 --- /dev/null +++ b/server/database/compact/__init__.py @@ -0,0 +1 @@ +"""Compact gameplay storage: codecs, records, replay blobs and hydration.""" diff --git a/server/database/compact/cache.py b/server/database/compact/cache.py new file mode 100644 index 0000000..9067d9d --- /dev/null +++ b/server/database/compact/cache.py @@ -0,0 +1,33 @@ +"""Per-process cache of running games to avoid repeated lookups per turn.""" + +from collections import OrderedDict +from dataclasses import dataclass, field + +from .records import PointsState + +@dataclass +class RunningGame: + gid:int + snake_ids:dict[str, int] = field(default_factory=dict) + points:PointsState|None = None + +class RunningGameCache: + def __init__(self, max_games:int=512): + self._games:OrderedDict[str, RunningGame] = OrderedDict() + self._max_games = max(1, max_games) + + def get(self, game_id:str) -> RunningGame|None: + game = self._games.get(game_id) + if game is not None: + self._games.move_to_end(game_id) + return game + + def put(self, game_id:str, game:RunningGame) -> RunningGame: + self._games[game_id] = game + self._games.move_to_end(game_id) + while len(self._games) > self._max_games: + self._games.popitem(last=False) + return game + + def drop(self, game_id:str) -> None: + self._games.pop(game_id, None) diff --git a/server/database/compact/codec.py b/server/database/compact/codec.py new file mode 100644 index 0000000..81b86b1 --- /dev/null +++ b/server/database/compact/codec.py @@ -0,0 +1,146 @@ +"""Byte-level encodings for compact gameplay storage. + +All encodings are lossless. Values that do not fit the compact form (for +example coordinates outside 0..255) fall back to a JSON payload, marked by the +leading format byte, so unusual maps can never corrupt stored data. +""" + +from datetime import datetime, timezone +import json + +MOVES = ("up", "down", "left", "right") +_MOVE_CODES = {move: index + 1 for index, move in enumerate(MOVES)} + +_FORMAT_JSON = 0 +_FORMAT_COMPACT = 1 + +# Body step codes relative to the previous segment. +_SAME, _UP, _DOWN, _LEFT, _RIGHT, _ABSOLUTE = range(6) +_STEPS = {(0, 0): _SAME, (0, 1): _UP, (0, -1): _DOWN, (-1, 0): _LEFT, (1, 0): _RIGHT} +_STEP_DELTAS = {code: delta for delta, code in _STEPS.items()} + +def encode_move(move:str|None) -> int|None: + return _MOVE_CODES.get(move) if move is not None else None + +def decode_move(code:int|None) -> str|None: + if not code: + return None + return MOVES[int(code) - 1] + +def encode_latency(latency:object) -> int|None: + if latency is None or latency == "": + return None + try: + value = int(str(latency)) + except ValueError: + return None + return value if -32768 <= value <= 32767 else None + +def decode_latency(value:int|None) -> str|None: + return None if value is None else str(value) + +def _point(point:object) -> tuple[int, int]|None: + if not isinstance(point, dict): + return None + x, y = point.get("x"), point.get("y") + if type(x) is not int or type(y) is not int: + return None + return x, y + +def _fits_byte(*values:int) -> bool: + return all(0 <= value <= 255 for value in values) + +def _json_fallback(value:object) -> bytes: + return bytes([_FORMAT_JSON]) + json.dumps(value, separators=(",", ":")).encode() + +def encode_body(body:list) -> bytes: + """Head as absolute byte pair, then one step code per following segment.""" + points = [_point(segment) for segment in body] + if any(point is None for point in points) or not all(_fits_byte(*point) for point in points): + return _json_fallback(body) + + output = bytearray([_FORMAT_COMPACT]) + previous = None + for x, y in points: + if previous is None: + output += bytes((x, y)) + else: + step = _STEPS.get((x - previous[0], y - previous[1])) + output += bytes((step,)) if step is not None else bytes((_ABSOLUTE, x, y)) + previous = (x, y) + return bytes(output) + +def decode_body(data:bytes|memoryview|None) -> list[dict]: + if not data: + return [] + data = bytes(data) + if data[0] == _FORMAT_JSON: + return json.loads(data[1:]) + + body = [] + index = 1 + if len(data) >= 3: + x, y = data[1], data[2] + body.append({"x": x, "y": y}) + index = 3 + while index < len(data): + code = data[index] + if code == _ABSOLUTE: + x, y = data[index + 1], data[index + 2] + index += 3 + else: + dx, dy = _STEP_DELTAS[code] + x, y = x + dx, y + dy + index += 1 + body.append({"x": x, "y": y}) + return body + +def encode_points(points:list|None) -> bytes|None: + """Food/hazard lists as byte pairs. ``None`` means "unchanged".""" + if points is None: + return None + pairs = [_point(point) for point in points] + if any(pair is None for pair in pairs) or not all(_fits_byte(*pair) for pair in pairs): + return _json_fallback(points) + output = bytearray([_FORMAT_COMPACT]) + for x, y in pairs: + output += bytes((x, y)) + return bytes(output) + +def decode_points(data:bytes|memoryview|None) -> list[dict]|None: + if data is None: + return None + data = bytes(data) + if not data: + return [] + if data[0] == _FORMAT_JSON: + return json.loads(data[1:]) + return [{"x": data[i], "y": data[i + 1]} for i in range(1, len(data) - 1, 2)] + +def flatten_points(points:list[dict]) -> list: + """Compact JSON form used inside replay blobs: [x0, y0, x1, y1, ...].""" + pairs = [_point(point) for point in points] + if any(pair is None for pair in pairs): + return [points] + return [value for pair in pairs for value in pair] + +def unflatten_points(values:list) -> list[dict]: + if len(values) == 1 and isinstance(values[0], list): + return values[0] + return [{"x": values[i], "y": values[i + 1]} for i in range(0, len(values), 2)] + +def to_epoch_us(value:datetime|str|None) -> int|None: + if value is None: + return None + if isinstance(value, str): + value = datetime.fromisoformat(value.replace("Z", "+00:00")) + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + delta = value - datetime(1970, 1, 1, tzinfo=timezone.utc) + return (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds + +def from_epoch_us(value:int|None) -> datetime|None: + if value is None: + return None + seconds, micros = divmod(int(value), 1_000_000) + return datetime.fromtimestamp(seconds, tz=timezone.utc).replace(microsecond=micros) diff --git a/server/database/compact/games.py b/server/database/compact/games.py new file mode 100644 index 0000000..407dc2f --- /dev/null +++ b/server/database/compact/games.py @@ -0,0 +1,28 @@ +"""Column list of the ``games`` table shared by backends and migrations.""" + +GAME_COLUMNS = ( + "game_id", "started_at", "ended_at", "width", "height", "source", "map_name", + "ruleset_name", "ruleset_version", "your_snake_id", "your_snake_name", + "your_snake_type", "your_snake_version", "game_type", "winner_name", + "winner_you", "final_turn", "status", "has_replay", "quality_status", + "quality_score", "quality_tier", "quality_reasons", +) + +GAME_DEFAULTS = { + "winner_you": False, + "final_turn": 0, + "status": "running", + "has_replay": True, + "quality_status": "retained", +} + +def normalize_game(row:dict) -> dict: + """Return a dict with every games column, filling legacy gaps with defaults.""" + game = {column: row.get(column, GAME_DEFAULTS.get(column)) for column in GAME_COLUMNS} + for column, default in GAME_DEFAULTS.items(): + if game[column] is None: + game[column] = default + game["winner_you"] = bool(game["winner_you"]) + game["has_replay"] = bool(game["has_replay"]) + game["final_turn"] = int(game["final_turn"] or 0) + return game diff --git a/server/database/compact/hydrate.py b/server/database/compact/hydrate.py new file mode 100644 index 0000000..8d0cf62 --- /dev/null +++ b/server/database/compact/hydrate.py @@ -0,0 +1,100 @@ +"""Rebuild the dashboard replay payload from compact records.""" + +from .codec import decode_latency +from .records import SnakeIdentity, SnakeState, TurnRecord, you_view +from .thinking import derived_values, restore_thinking + +_DIRECTIONS = {(1, 0): "right", (-1, 0): "left", (0, 1): "up", (0, -1): "down"} + +def _head(state:SnakeState) -> dict|None: + return state.body[0] if state.body else None + +def _inferred_move(previous:SnakeState|None, current:SnakeState) -> str|None: + if previous is None: + return None + old, new = _head(previous), _head(current) + if old is None or new is None: + return None + return _DIRECTIONS.get((new["x"] - old["x"], new["y"] - old["y"])) + +def _sort_key(identity:SnakeIdentity|None): + name = identity.name if identity else None + is_you = bool(identity and identity.is_you) + return (not is_you, name is None, name or "") + +def hydrate_replay_turns( + game:dict, + identities:dict[str, SnakeIdentity], + turns:list[TurnRecord], + states:list[SnakeState], +) -> list[dict]: + states_by_turn:dict[int, dict[str, SnakeState]] = {} + for state in states: + states_by_turn.setdefault(state.turn, {})[state.snake_id] = state + + width, height = game.get("width"), game.get("height") + you_id = game.get("your_snake_id") + food:list = [] + hazards:list = [] + output = [] + for record in sorted(turns, key=lambda item: item.turn): + food = food if record.food is None else record.food + hazards = hazards if record.hazards is None else record.hazards + current = states_by_turn.get(record.turn, {}) + previous = states_by_turn.get(record.turn - 1, {}) + ordered = sorted(current.values(), key=lambda s: _sort_key(identities.get(s.snake_id))) + + snakes = [] + api_snakes = [] + for state in ordered: + identity = identities.get(state.snake_id) + name = identity.name if identity else None + customizations = identity.customizations if identity else {} + latency = decode_latency(state.latency) + api_snake = { + "id": state.snake_id, + "name": name, + "health": state.health, + "length": len(state.body), + "head": _head(state) or {"x": None, "y": None}, + "body": state.body, + "customizations": customizations, + } + if latency is not None: + api_snake["latency"] = latency + api_snakes.append(api_snake) + snakes.append({ + "snake_id": state.snake_id, + "snake_name": name, + "health": state.health, + "length": len(state.body), + "head": api_snake["head"], + "body": state.body, + "customizations": customizations, + "is_you": bool(identity and identity.is_you), + "inferred_move": _inferred_move(previous.get(state.snake_id), state), + "latency": latency, + }) + + derived = derived_values( + record.turn, record.move, you_view(list(current.values()), you_id), + food, width, height, len(current), + ) + output.append({ + "turn": record.turn, + "observed_at": record.observed_at, + "my_move": record.move, + "my_thinking": restore_thinking(record.thinking, derived), + "board": { + "width": width, + "height": height, + "food": food, + "hazards": hazards, + "snakes": api_snakes, + }, + "food": food, + "hazards": hazards, + "you": next((snake for snake in api_snakes if snake["id"] == you_id), {}), + "snakes": snakes, + }) + return output diff --git a/server/database/compact/quality.py b/server/database/compact/quality.py new file mode 100644 index 0000000..85c0558 --- /dev/null +++ b/server/database/compact/quality.py @@ -0,0 +1,23 @@ +"""Rate a game's replay quality from its records.""" + +from server.database.game_quality import GameQuality, GameQualityInput, rate_game_quality + +from .codec import MOVES +from .records import GameBundle + +def rate_bundle(bundle:GameBundle) -> GameQuality: + turns = bundle.turns + numbers = [record.turn for record in turns] + moves = [record.move for record in turns] + return rate_game_quality(GameQualityInput( + status=bundle.game.get("status") or "running", + final_turn=int(bundle.game.get("final_turn") or 0), + turn_rows=len(turns), + min_turn=min(numbers) if numbers else None, + max_turn=max(numbers) if numbers else None, + valid_moves=sum(1 for move in moves if move in MOVES), + thinking_rows=sum(1 for record in turns if record.thinking not in (None, {})), + distinct_moves=len({move for move in moves if move is not None}), + snake_turn_rows=len(bundle.states), + winner_name=bundle.game.get("winner_name"), + )) diff --git a/server/database/compact/records.py b/server/database/compact/records.py new file mode 100644 index 0000000..9522978 --- /dev/null +++ b/server/database/compact/records.py @@ -0,0 +1,136 @@ +"""In-memory records shared by both backends, the replay blob and migrations. + +A ``TurnRecord`` is already in compact form: ``thinking`` has derivable fields +stripped, and ``food``/``hazards`` are ``None`` when unchanged since the +previous stored turn. +""" + +from dataclasses import dataclass, field +from datetime import datetime, timezone + +from .codec import encode_latency +from .thinking import derived_values, slim_thinking + +@dataclass(frozen=True) +class SnakeIdentity: + snake_id:str + name:str|None + is_you:bool + customizations:dict = field(default_factory=dict) + +@dataclass(frozen=True) +class SnakeState: + snake_id:str + turn:int + health:int|None + latency:int|None + body:list + +@dataclass(frozen=True) +class TurnRecord: + turn:int + observed_at:datetime|None + move:str|None + thinking:object + food:list|None + hazards:list|None + +@dataclass(frozen=True) +class PointsState: + """Last stored food/hazards, used to store only changes.""" + turn:int + food:list + hazards:list + +@dataclass +class GameBundle: + game:dict + snakes:list[SnakeIdentity] + turns:list[TurnRecord] + states:list[SnakeState] + +def you_view(states:list[SnakeState], you_id:str|None) -> dict|None: + state = next((s for s in states if s.snake_id == you_id), None) + if state is None: + return None + return { + "head": state.body[0] if state.body else None, + "health": state.health, + "length": len(state.body), + } + +def compact_turn( + turn:int, + observed_at:datetime|None, + move:str|None, + thinking:object, + food:list, + hazards:list, + states:list[SnakeState], + you_id:str|None, + width:int|None, + height:int|None, + previous:PointsState|None, +) -> tuple[TurnRecord, PointsState]: + derived = derived_values(turn, move, you_view(states, you_id), food, width, height, len(states)) + unchanged = previous is not None and previous.turn == turn - 1 + record = TurnRecord( + turn=turn, + observed_at=observed_at, + move=move, + thinking=slim_thinking(thinking, derived), + food=None if unchanged and previous.food == food else food, + hazards=None if unchanged and previous.hazards == hazards else hazards, + ) + return record, PointsState(turn, food, hazards) + +def records_from_game_state( + game_state:dict, + my_move:str|None, + thinking:object, + previous:PointsState|None, +) -> tuple[TurnRecord, list[SnakeState], list[SnakeIdentity], PointsState]: + board = game_state.get("board", {}) + you_id = game_state.get("you", {}).get("id") + turn = int(game_state.get("turn", 0)) + identities = [] + states = [] + for snake in board.get("snakes", []): + snake_id = snake.get("id") + if snake_id is None: + continue + identities.append(SnakeIdentity( + snake_id=snake_id, + name=snake.get("name"), + is_you=snake_id == you_id, + customizations=snake.get("customizations") or {}, + )) + states.append(SnakeState( + snake_id=snake_id, + turn=turn, + health=snake.get("health"), + latency=encode_latency(snake.get("latency")), + body=list(snake.get("body", [])), + )) + record, points = compact_turn( + turn, datetime.now(timezone.utc), my_move, thinking, + list(board.get("food", [])), list(board.get("hazards", [])), + states, you_id, board.get("width"), board.get("height"), previous, + ) + return record, states, identities, points + +def merge_records( + base_turns:list[TurnRecord], + base_states:list[SnakeState], + turns:list[TurnRecord], + states:list[SnakeState], +) -> tuple[list[TurnRecord], list[SnakeState]]: + """Combine a packed replay with rows written after it was packed.""" + merged_turns = {record.turn: record for record in base_turns} + merged_turns.update({record.turn: record for record in turns}) + merged_states = {(state.snake_id, state.turn): state for state in base_states} + merged_states.update({(state.snake_id, state.turn): state for state in states}) + return ( + sorted(merged_turns.values(), key=lambda record: record.turn), + sorted(merged_states.values(), key=lambda state: (state.turn, state.snake_id)), + ) diff --git a/server/database/compact/replay_blob.py b/server/database/compact/replay_blob.py new file mode 100644 index 0000000..ec3a069 --- /dev/null +++ b/server/database/compact/replay_blob.py @@ -0,0 +1,63 @@ +"""Pack a finished game's turns into one zstd-compressed blob. + +Consecutive turns are nearly identical, so compressing a whole game at once is +far smaller than storing one row per turn and snake. +""" + +from compression import zstd +import json + +from .codec import ( + decode_move, encode_move, flatten_points, from_epoch_us, to_epoch_us, + unflatten_points, +) +from .records import SnakeState, TurnRecord + +REPLAY_FORMAT = 1 +_LEVEL = 10 + +def _optional_points(points:list|None) -> list|None: + return None if points is None else flatten_points(points) + +def _optional_unflatten(values:list|None) -> list|None: + return None if values is None else unflatten_points(values) + +def pack_replay(turns:list[TurnRecord], states:list[SnakeState]) -> bytes: + by_snake:dict[str, list] = {} + for state in sorted(states, key=lambda s: (s.snake_id, s.turn)): + by_snake.setdefault(state.snake_id, []).append( + [state.turn, state.health, state.latency, flatten_points(state.body)] + ) + payload = { + "t": [ + [ + record.turn, to_epoch_us(record.observed_at), encode_move(record.move), + record.thinking, _optional_points(record.food), _optional_points(record.hazards), + ] + for record in sorted(turns, key=lambda item: item.turn) + ], + "s": by_snake, + } + raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode() + return zstd.compress(raw, level=_LEVEL) + +def unpack_replay(data:bytes|memoryview, format_version:int=REPLAY_FORMAT) -> tuple[list[TurnRecord], list[SnakeState]]: + if format_version != REPLAY_FORMAT: + raise ValueError(f"Unsupported replay format: {format_version}") + payload = json.loads(zstd.decompress(bytes(data))) + turns = [ + TurnRecord( + turn=turn, observed_at=from_epoch_us(observed), move=decode_move(move), + thinking=thinking, food=_optional_unflatten(food), hazards=_optional_unflatten(hazards), + ) + for turn, observed, move, thinking, food, hazards in payload["t"] + ] + states = [ + SnakeState( + snake_id=snake_id, turn=turn, health=health, latency=latency, + body=unflatten_points(body), + ) + for snake_id, rows in payload["s"].items() + for turn, health, latency, body in rows + ] + return turns, states diff --git a/server/database/compact/thinking.py b/server/database/compact/thinking.py new file mode 100644 index 0000000..77de426 --- /dev/null +++ b/server/database/compact/thinking.py @@ -0,0 +1,57 @@ +"""Strip move-reasoning fields that can be rebuilt from the stored turn. + +Snakes put values like ``turn``, ``health`` or ``head`` into their reasoning +payload although the same values are stored in the turn and snake rows. Such a +field is only removed when it equals the derived value, and a bitmask records +which fields were removed, so restoring is exact. +""" + +DERIVED_KEYS = ("turn", "move", "head", "health", "length", "food_count", "board_size", "snakes") +MASK_KEY = "_derived" +_RAW_KEY = "_raw" + +def derived_values(turn:int, move:str|None, you:dict|None, food:list, width:int|None, height:int|None, snake_count:int) -> dict: + you = you or {} + return { + "turn": turn, + "move": move, + "head": you.get("head"), + "health": you.get("health"), + "length": you.get("length"), + "food_count": len(food), + "board_size": f"{width}x{height}", + "snakes": snake_count, + } + +def _same(stored:object, derived:object) -> bool: + # bool is an int subclass; never treat True == 1 as derivable. + return type(stored) is type(derived) and stored == derived + +def slim_thinking(thinking:object, derived:dict) -> object: + if not isinstance(thinking, dict): + return thinking + if MASK_KEY in thinking: + # A payload that already uses the marker key is stored verbatim. + return {MASK_KEY: 0, _RAW_KEY: thinking} + mask = 0 + slim = {} + for key, value in thinking.items(): + if key in DERIVED_KEYS and _same(value, derived[key]): + mask |= 1 << DERIVED_KEYS.index(key) + else: + slim[key] = value + if mask: + slim[MASK_KEY] = mask + return slim + +def restore_thinking(thinking:object, derived:dict) -> object: + if not isinstance(thinking, dict) or MASK_KEY not in thinking: + return thinking + mask = int(thinking[MASK_KEY]) + if mask == 0 and _RAW_KEY in thinking: + return thinking[_RAW_KEY] + restored = {key: value for key, value in thinking.items() if key != MASK_KEY} + for index, key in enumerate(DERIVED_KEYS): + if mask & (1 << index): + restored[key] = derived[key] + return restored diff --git a/server/database/legacy/__init__.py b/server/database/legacy/__init__.py new file mode 100644 index 0000000..1e6b1d6 --- /dev/null +++ b/server/database/legacy/__init__.py @@ -0,0 +1 @@ +"""Read the legacy turns/snake_turns layout and migrate it to compact storage.""" diff --git a/server/database/legacy/convert.py b/server/database/legacy/convert.py new file mode 100644 index 0000000..60e1894 --- /dev/null +++ b/server/database/legacy/convert.py @@ -0,0 +1,98 @@ +"""Convert rows of the legacy layout (turns/snake_turns) into a compact bundle.""" + +import json + +from server.database.compact.codec import encode_latency +from server.database.compact.games import normalize_game +from server.database.compact.records import ( + GameBundle, PointsState, SnakeIdentity, SnakeState, compact_turn, +) + +class LegacyDataError(ValueError): + """Legacy rows that cannot be converted without losing data.""" + +def parse_json(value:object, default:object=None) -> object: + if value is None or value == "": + return default + if not isinstance(value, str): + return value + try: + return json.loads(value) + except (json.JSONDecodeError, TypeError) as exc: + raise LegacyDataError(f"Invalid JSON value: {value[:120]}") from exc + +def _state_from_row(row:dict) -> SnakeState: + body = parse_json(row.get("body"), []) or [] + head_x, head_y, length = row.get("head_x"), row.get("head_y"), row.get("length") + if length is not None and int(length) != len(body): + raise LegacyDataError(f"snake {row['snake_id']} turn {row['turn']}: length {length} != body {len(body)}") + if body and head_x is not None and (head_x, head_y) != (body[0].get("x"), body[0].get("y")): + raise LegacyDataError(f"snake {row['snake_id']} turn {row['turn']}: head differs from body[0]") + return SnakeState( + snake_id=row["snake_id"], turn=int(row["turn"]), health=row.get("health"), + latency=encode_latency(row.get("latency")), body=body, + ) + +def _states_from_board(board:dict, turn:int) -> list[SnakeState]: + return [ + SnakeState( + snake_id=snake["id"], turn=turn, health=snake.get("health"), + latency=encode_latency(snake.get("latency")), body=list(snake.get("body", [])), + ) + for snake in board.get("snakes", []) if snake.get("id") is not None + ] + +def _identities(game:dict, identity_rows:list[dict], snake_rows:list[dict], board_snakes:list[dict]) -> list[SnakeIdentity]: + identities:dict[str, SnakeIdentity] = {} + for row in identity_rows: + identities[row["snake_id"]] = SnakeIdentity( + row["snake_id"], row.get("snake_name"), bool(row.get("is_you")), + parse_json(row.get("customizations"), {}) or {}, + ) + # Older databases have an empty or partial game_snakes table. + for row in snake_rows + board_snakes: + snake_id = row.get("snake_id") or row.get("id") + if snake_id is None or snake_id in identities: + continue + name = row.get("snake_name") or row.get("name") + is_you = bool(row.get("is_you")) or snake_id == game.get("your_snake_id") + identities[snake_id] = SnakeIdentity(snake_id, name, is_you, row.get("customizations") or {}) + for snake_id, identity in list(identities.items()): + if identity.name is None: + name = next((row.get("snake_name") for row in snake_rows if row.get("snake_id") == snake_id and row.get("snake_name")), None) + if name: + identities[snake_id] = SnakeIdentity(snake_id, name, identity.is_you, identity.customizations) + return list(identities.values()) + +def build_legacy_bundle(game_row:dict, identity_rows:list[dict], turn_rows:list[dict], snake_rows:list[dict]) -> GameBundle: + game = normalize_game(game_row) + states_by_turn:dict[int, list[SnakeState]] = {} + for row in snake_rows: + state = _state_from_row(row) + states_by_turn.setdefault(state.turn, []).append(state) + + turns = [] + states = [] + board_snakes = [] + previous:PointsState|None = None + for row in sorted(turn_rows, key=lambda item: int(item["turn"])): + turn = int(row["turn"]) + board = parse_json(row.get("board_state"), {}) or {} + food = board["food"] if "food" in board else parse_json(row.get("food"), []) or [] + hazards = board["hazards"] if "hazards" in board else parse_json(row.get("hazards"), []) or [] + turn_states = states_by_turn.pop(turn, None) + if turn_states is None: + turn_states = _states_from_board(board, turn) + board_snakes.extend(board.get("snakes", [])) + states.extend(turn_states) + record, previous = compact_turn( + turn, row.get("observed_at"), row.get("my_move"), parse_json(row.get("my_thinking")), + food, hazards, turn_states, game.get("your_snake_id"), game.get("width"), game.get("height"), + previous, + ) + turns.append(record) + # Snake rows without a turn row are kept as well. + for orphan in states_by_turn.values(): + states.extend(orphan) + + return GameBundle(game, _identities(game, identity_rows, snake_rows, board_snakes), turns, states) diff --git a/server/database/legacy/migrate.py b/server/database/legacy/migrate.py new file mode 100644 index 0000000..2258903 --- /dev/null +++ b/server/database/legacy/migrate.py @@ -0,0 +1,84 @@ +"""Copy legacy-layout games into the compact layout, verifying each game.""" + +from dataclasses import dataclass, field +import inspect +import json + +from server.database.compact.quality import rate_bundle +from server.database.compact.records import GameBundle +from server.database.game_quality import quality_meets_minimum + +from .convert import LegacyDataError + +@dataclass +class MigrationStats: + games:int = 0 + replays:int = 0 + turns:int = 0 + states:int = 0 + dropped_replays:int = 0 + failed:int = 0 + failures:list[str] = field(default_factory=list) + +async def _resolve(value): + return await value if inspect.isawaitable(value) else value + +def _signature(bundle:GameBundle) -> tuple: + turns = sorted( + (record.turn, record.move, json.dumps(record.thinking, sort_keys=True), json.dumps(record.food), json.dumps(record.hazards)) + for record in bundle.turns + ) + states = sorted( + (state.snake_id, state.turn, state.health, state.latency, json.dumps(state.body)) + for state in bundle.states + ) + return turns, states + +def apply_quality(bundle:GameBundle, minimum_quality:str) -> bool: + """Rate the replay; drop its turns below the minimum. Returns whether it is kept.""" + quality = rate_bundle(bundle) + keep = quality_meets_minimum(quality.tier, minimum_quality) + bundle.game.update({ + "has_replay": keep, + "quality_status": "retained" if keep else "low_quality", + "quality_score": quality.score, + "quality_tier": quality.tier, + "quality_reasons": list(quality.reasons), + }) + if not keep: + bundle.turns = [] + bundle.states = [] + return keep + +async def migrate_legacy( + reader, + backend, + since:str|None=None, + minimum_quality:str|None=None, + verify:bool=True, + progress=None, +) -> MigrationStats: + stats = MigrationStats() + game_ids = await _resolve(reader.game_ids(since)) + for index, game_id in enumerate(game_ids, start=1): + try: + bundle = await _resolve(reader.read_bundle(game_id)) + if bundle is None: + continue + if minimum_quality and not apply_quality(bundle, minimum_quality): + stats.dropped_replays += 1 + await backend.import_game_bundle(bundle) + if verify and bundle.turns and bundle.game["has_replay"]: + stored = await backend.read_game_bundle(game_id) + if stored is None or _signature(stored) != _signature(bundle): + raise LegacyDataError("stored replay differs from source") + stats.games += 1 + stats.replays += int(bool(bundle.turns)) + stats.turns += len(bundle.turns) + stats.states += len(bundle.states) + except LegacyDataError as error: + stats.failed += 1 + stats.failures.append(f"{game_id}: {error}") + if progress is not None and (index % 500 == 0 or index == len(game_ids)): + progress(index, len(game_ids), stats) + return stats diff --git a/server/database/legacy/reader_postgresql.py b/server/database/legacy/reader_postgresql.py new file mode 100644 index 0000000..6c2d066 --- /dev/null +++ b/server/database/legacy/reader_postgresql.py @@ -0,0 +1,69 @@ +"""Read legacy-layout games (turns/snake_turns) from PostgreSQL.""" + +import json + +from server.database.compact.games import GAME_COLUMNS +from server.database.compact.records import GameBundle + +from .convert import build_legacy_bundle + +class LegacyPostgresqlReader: + def __init__(self, pool): + self._pool = pool + + @classmethod + async def connect(cls, dsn:str) -> "LegacyPostgresqlReader": + import asyncpg # noqa: PLC0415 + + async def _init(conn) -> None: + await conn.set_type_codec("jsonb", encoder=json.dumps, decoder=json.loads, schema="pg_catalog") + await conn.set_type_codec("json", encoder=json.dumps, decoder=json.loads, schema="pg_catalog") + + return cls(await asyncpg.create_pool(dsn=dsn, min_size=1, max_size=4, init=_init)) + + async def close(self) -> None: + await self._pool.close() + + async def has_legacy_tables(self) -> bool: + async with self._pool.acquire() as conn: + return bool(await conn.fetchval( + "SELECT to_regclass('public.turns') IS NOT NULL AND to_regclass('public.snake_turns') IS NOT NULL" + )) + + async def game_ids(self, since:str|None=None) -> list[str]: + async with self._pool.acquire() as conn: + if since: + rows = await conn.fetch(""" + SELECT game_id FROM games WHERE started_at >= $1::timestamptz + ORDER BY started_at, game_id + """, since) + else: + rows = await conn.fetch("SELECT game_id FROM games ORDER BY started_at, game_id") + return [row["game_id"] for row in rows] + + async def read_bundle(self, game_id:str) -> GameBundle|None: + async with self._pool.acquire() as conn: + game = await conn.fetchrow(f"SELECT {', '.join(GAME_COLUMNS)} FROM games WHERE game_id = $1", game_id) + if game is None: + return None + identities = await conn.fetch(""" + SELECT snake_id, snake_name, is_you, customizations + FROM game_snakes WHERE game_id = $1 + """, game_id) + turns = await conn.fetch(""" + SELECT turn, observed_at, my_move, my_thinking, board_state, food, hazards + FROM turns WHERE game_id = $1 + """, game_id) + snakes = await conn.fetch(""" + SELECT snake_id, turn, snake_name, health, length, head_x, head_y, body, is_you, latency + FROM snake_turns WHERE game_id = $1 + """, game_id) + return build_legacy_bundle( + dict(game), [dict(row) for row in identities], + [dict(row) for row in turns], [dict(row) for row in snakes], + ) + + async def turn_counts(self) -> dict[str, int]: + async with self._pool.acquire() as conn: + rows = await conn.fetch("SELECT game_id, COUNT(*) AS turns FROM turns GROUP BY game_id") + return {row["game_id"]: int(row["turns"]) for row in rows} diff --git a/server/database/legacy/reader_sqlite.py b/server/database/legacy/reader_sqlite.py new file mode 100644 index 0000000..2863b21 --- /dev/null +++ b/server/database/legacy/reader_sqlite.py @@ -0,0 +1,100 @@ +"""Read legacy-layout games from a SQLite file (opened read-only).""" + +from pathlib import Path +import sqlite3 + +from server.database.compact.games import GAME_COLUMNS +from server.database.compact.records import GameBundle + +from .convert import build_legacy_bundle, parse_json + +_TURN_COLUMNS = { + "turn": "turn", "observed_at": "observed_at", "my_move": "my_move", + "my_thinking": "my_thinking_json", "board_state": "board_state_json", + "food": "food_json", "hazards": "hazards_json", +} +_SNAKE_COLUMNS = { + "snake_id": "snake_id", "turn": "turn", "snake_name": "snake_name", + "health": "health", "length": "length", "head_x": "head_x", "head_y": "head_y", + "body": "body_json", "is_you": "is_you", "latency": "latency", +} + +class LegacySqliteReader: + def __init__(self, path:str|Path): + self.path = Path(path) + self._connection = sqlite3.connect(f"file:{self.path}?mode=ro", uri=True, timeout=60) + self._connection.row_factory = sqlite3.Row + self._connection.execute("PRAGMA query_only = ON") + + def __enter__(self) -> "LegacySqliteReader": + return self + + def __exit__(self, *exc) -> None: + self.close() + + def close(self) -> None: + self._connection.close() + + def _columns(self, table:str) -> set[str]: + return {row[1] for row in self._connection.execute(f"PRAGMA table_info({table})")} + + def _select(self, table:str, mapping:dict[str, str]) -> str: + available = self._columns(table) + return ", ".join( + f"{source} AS {name}" if source in available else f"NULL AS {name}" + for name, source in mapping.items() + ) + + def has_legacy_tables(self) -> bool: + return bool(self._columns("turns")) and bool(self._columns("snake_turns")) + + def game_ids(self, since:str|None=None) -> list[str]: + if since: + rows = self._connection.execute( + "SELECT game_id FROM games WHERE started_at >= ? ORDER BY started_at, game_id", (since,), + ) + else: + rows = self._connection.execute("SELECT game_id FROM games ORDER BY started_at, game_id") + return [row[0] for row in rows] + + def read_bundle(self, game_id:str) -> GameBundle|None: + available = self._columns("games") + selected = ", ".join( + column if column in available + else "quality_reasons_json AS quality_reasons" if column == "quality_reasons" and "quality_reasons_json" in available + else f"NULL AS {column}" + for column in GAME_COLUMNS + ) + game = self._connection.execute(f"SELECT {selected} FROM games WHERE game_id = ?", (game_id,)).fetchone() + if game is None: + return None + game = dict(game) + game["quality_reasons"] = parse_json(game.get("quality_reasons")) + + identities = [] + snake_columns = self._columns("game_snakes") + if snake_columns: + customizations = "customizations_json" if "customizations_json" in snake_columns else "NULL" + identities = [ + dict(row) for row in self._connection.execute(f""" + SELECT snake_id, snake_name, is_you, {customizations} AS customizations + FROM game_snakes WHERE game_id = ? + """, (game_id,)) + ] + turns = [ + dict(row) for row in self._connection.execute( + f"SELECT {self._select('turns', _TURN_COLUMNS)} FROM turns WHERE game_id = ?", (game_id,), + ) + ] + snakes = [ + dict(row) for row in self._connection.execute( + f"SELECT {self._select('snake_turns', _SNAKE_COLUMNS)} FROM snake_turns WHERE game_id = ?", (game_id,), + ) + ] + return build_legacy_bundle(game, identities, turns, snakes) + + def turn_counts(self) -> dict[str, int]: + return { + row[0]: int(row[1]) + for row in self._connection.execute("SELECT game_id, COUNT(*) FROM turns GROUP BY game_id") + } diff --git a/server/database/normalized_turn.py b/server/database/normalized_turn.py deleted file mode 100644 index 460a705..0000000 --- a/server/database/normalized_turn.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Normalized gameplay turn storage and replay hydration. - -A turn is split into one board row plus one snake row per participating snake. -Static snake identity belongs to ``game_snakes``. This avoids storing complete -snake payloads in the board, snakes, you, and snake-turn columns simultaneously. -""" - -from typing import Callable - -def compact_turn_json(board:dict) -> tuple[dict, list, dict, list, list]: - """Return compatibility JSON plus canonical food and hazard values.""" - return {}, [], {}, board.get("food", []), board.get("hazards", []) - -def hydrate_replay_turns(game_row, turn_rows, snake_rows, decode_json:Callable) -> list[dict]: - rows_by_turn:dict[int, list] = {} - for row in snake_rows: - rows_by_turn.setdefault(int(row["turn"]), []).append(row) - - turns = [] - for row in turn_rows: - turn = int(row["turn"]) - stored_board = decode_json(row["board_state_json"]) or {} - food = decode_json(row["food_json"]) - hazards = decode_json(row["hazards_json"]) - - snakes = [] - api_snakes = [] - for snake_row in rows_by_turn.get(turn, []): - body = decode_json(snake_row["body_json"]) or [] - customizations = decode_json(snake_row["customizations_json"]) or {} - api_snake = { - "id": snake_row["snake_id"], - "name": snake_row["snake_name"], - "health": snake_row["health"], - "length": snake_row["length"], - "head": {"x": snake_row["head_x"], "y": snake_row["head_y"]}, - "body": body, - "customizations": customizations, - } - if snake_row["latency"] is not None: - api_snake["latency"] = snake_row["latency"] - api_snakes.append(api_snake) - snakes.append({ - "snake_id": snake_row["snake_id"], - "snake_name": snake_row["snake_name"], - "health": snake_row["health"], - "length": snake_row["length"], - "head": api_snake["head"], - "body": body, - "customizations": customizations, - "is_you": bool(snake_row["is_you"]), - "inferred_move": snake_row["inferred_move"], - "latency": snake_row["latency"], - }) - - board = stored_board or { - "width": game_row["width"], - "height": game_row["height"], - "food": food or [], - "hazards": hazards or [], - "snakes": api_snakes, - } - if food is None: - food = board.get("food", []) - if hazards is None: - hazards = board.get("hazards", []) - - you = decode_json(row["you_json"]) or {} - if not you: - you = next( - (snake for snake in api_snakes if snake["id"] == game_row["your_snake_id"]), - {}, - ) - - turns.append({ - "turn": turn, - "observed_at": row["observed_at"], - "my_move": row["my_move"], - "my_thinking": decode_json(row["my_thinking_json"]), - "board": board, - "food": food or [], - "hazards": hazards or [], - "you": you, - "snakes": snakes, - }) - - return turns diff --git a/tests/test_BenchmarkStates.py b/tests/test_BenchmarkStates.py index e412c3f..8509676 100644 --- a/tests/test_BenchmarkStates.py +++ b/tests/test_BenchmarkStates.py @@ -1,6 +1,21 @@ +import asyncio import unittest +from unittest.mock import AsyncMock -from server.database.benchmark_states import _build_state, is_postgresql_source +from server.database.benchmark_states import is_postgresql_source, replay_turn_to_state, sample_states + +GAME = { + "game_id": "game-id", "source": "league", "map": "standard", + "ruleset_name": "standard", "ruleset_version": "v1", +} + +def _turn(number:int) -> dict: + you = {"id": "you-id", "head": {"x": 1, "y": 2}, "body": [{"x": 1, "y": 2}]} + return { + "turn": number, + "board": {"width": 11, "height": 11, "food": [{"x": 5, "y": 5}], "hazards": [], "snakes": [you]}, + "you": you, + } class TestBenchmarkStates(unittest.TestCase): def test_detects_postgresql_dsn(self): @@ -8,23 +23,27 @@ class TestBenchmarkStates(unittest.TestCase): self.assertTrue(is_postgresql_source("postgres://user:pass@example.com/db")) self.assertFalse(is_postgresql_source("/tmp/gameplay.sqlite3")) - def test_builds_normalized_state_from_postgresql_json_values(self): - row = ( - 1, {}, {}, [{"x": 5, "y": 5}], [], "you-id", "You", 11, 11, - "game-id", "league", "standard", "standard", "v1", 7, - ) - snake_rows = [ - ("you-id", "You", 90, 3, 1, 2, [{"x": 1, "y": 2}], {"head": "default"}), - ] - - state = _build_state(row, snake_rows) - - self.assertIsNotNone(state) - board, metadata = state + def test_builds_state_from_replay_turn(self): + board, metadata = replay_turn_to_state(GAME, _turn(7)) self.assertEqual(board["food"], [{"x": 5, "y": 5}]) - self.assertEqual(board["snakes"][0]["id"], "you-id") self.assertEqual(metadata["you"]["id"], "you-id") self.assertEqual(metadata["turn"], 7) + self.assertEqual(metadata["ruleset"]["version"], "v1") + + def test_turn_without_snakes_is_skipped(self): + self.assertIsNone(replay_turn_to_state(GAME, {"turn": 0, "board": {"snakes": []}, "you": {}})) + + def test_samples_every_stride_turn_from_newest_games(self): + backend = AsyncMock() + backend.list_game_ids.return_value = ["old", "new"] + backend.get_game_replay.side_effect = lambda game_id: { + "game": {**GAME, "game_id": game_id}, + "turns": [_turn(number) for number in range(4)], + } + + states = asyncio.run(sample_states(backend, samples=3, stride=3)) + + self.assertEqual([(m["game_id"], m["turn"]) for _, m in states], [("new", 3), ("new", 0), ("old", 1)]) if __name__ == "__main__": unittest.main() diff --git a/tests/test_CompactStorage.py b/tests/test_CompactStorage.py new file mode 100644 index 0000000..0df7368 --- /dev/null +++ b/tests/test_CompactStorage.py @@ -0,0 +1,124 @@ +import unittest +from datetime import datetime, timezone + +from server.database.compact.codec import ( + decode_body, decode_latency, decode_move, decode_points, encode_body, + encode_latency, encode_move, encode_points, from_epoch_us, to_epoch_us, +) +from server.database.compact.records import PointsState, SnakeState, TurnRecord, compact_turn +from server.database.compact.replay_blob import pack_replay, unpack_replay +from server.database.compact.thinking import derived_values, restore_thinking, slim_thinking + +def _body(*points): + return [{"x": x, "y": y} for x, y in points] + +class TestCodec(unittest.TestCase): + def test_body_roundtrip_including_stacked_tail(self): + body = _body((5, 5), (5, 6), (4, 6), (4, 6)) + encoded = encode_body(body) + self.assertEqual(decode_body(encoded), body) + # 1 format byte + head pair + 3 step codes + self.assertEqual(len(encoded), 6) + + def test_body_roundtrip_with_wrapped_jump(self): + body = _body((0, 5), (10, 5), (9, 5)) + self.assertEqual(decode_body(encode_body(body)), body) + + def test_body_out_of_byte_range_falls_back_to_json(self): + body = _body((-1, 3), (300, 3)) + self.assertEqual(decode_body(encode_body(body)), body) + + def test_empty_body(self): + self.assertEqual(decode_body(encode_body([])), []) + + def test_points_keep_none_as_unchanged(self): + self.assertIsNone(encode_points(None)) + self.assertIsNone(decode_points(None)) + self.assertEqual(decode_points(encode_points([])), []) + food = _body((1, 2), (10, 0)) + self.assertEqual(decode_points(encode_points(food)), food) + + def test_moves_and_latency(self): + for move in ("up", "down", "left", "right", None): + self.assertEqual(decode_move(encode_move(move)), move) + self.assertEqual(decode_latency(encode_latency("77")), "77") + self.assertIsNone(encode_latency("")) + self.assertIsNone(encode_latency("timeout")) + + def test_epoch_microseconds_roundtrip(self): + value = datetime(2026, 11, 4, 21, 41, 48, 123456, tzinfo=timezone.utc) + self.assertEqual(from_epoch_us(to_epoch_us(value)), value) + self.assertEqual(from_epoch_us(to_epoch_us(value.isoformat())), value) + +class TestThinking(unittest.TestCase): + def setUp(self): + self.derived = derived_values( + turn=12, move="up", you={"head": {"x": 1, "y": 2}, "health": 90, "length": 4}, + food=[{"x": 0, "y": 0}], width=11, height=11, snake_count=2, + ) + + def test_strips_and_restores_derived_fields(self): + thinking = { + "turn": 12, "move": "up", "head": {"x": 1, "y": 2}, "health": 90, + "length": 4, "food_count": 1, "board_size": "11x11", "snakes": 2, + "scores": {"up": 1.5}, "mode": "duel", + } + slim = slim_thinking(thinking, self.derived) + self.assertEqual(set(slim), {"scores", "mode", "_derived"}) + self.assertEqual(restore_thinking(slim, self.derived), thinking) + + def test_keeps_values_that_differ_from_derived(self): + thinking = {"turn": 11, "health": 89, "snakes": True} + slim = slim_thinking(thinking, self.derived) + self.assertEqual(slim, thinking) + self.assertEqual(restore_thinking(slim, self.derived), thinking) + + def test_payload_using_marker_key_is_stored_verbatim(self): + thinking = {"_derived": "mine", "turn": 12} + self.assertEqual(restore_thinking(slim_thinking(thinking, self.derived), self.derived), thinking) + + def test_non_dict_payload_passes_through(self): + self.assertIsNone(slim_thinking(None, self.derived)) + self.assertEqual(slim_thinking(["a"], self.derived), ["a"]) + +class TestCompactTurn(unittest.TestCase): + def _compact(self, turn, food, previous): + states = [SnakeState("me", turn, 90, 5, _body((1, 1)))] + return compact_turn(turn, None, "up", None, food, [], states, "me", 11, 11, previous) + + def test_unchanged_food_is_stored_as_none_only_after_consecutive_turn(self): + food = _body((3, 3)) + first, points = self._compact(1, food, None) + second, points = self._compact(2, list(food), points) + after_gap, _ = self._compact(4, list(food), points) + self.assertEqual(first.food, food) + self.assertIsNone(second.food) + self.assertEqual(second.hazards, None) + self.assertEqual(after_gap.food, food) + + def test_previous_state_from_other_turn_is_not_used(self): + record, _ = self._compact(5, [], PointsState(5, [], [])) + self.assertEqual(record.food, []) + +class TestReplayBlob(unittest.TestCase): + def test_pack_roundtrip(self): + observed = datetime(2026, 11, 4, 12, 0, 0, 5, tzinfo=timezone.utc) + turns = [ + TurnRecord(0, observed, None, None, _body((1, 1)), []), + TurnRecord(1, observed, "left", {"scores": {"left": 1.0}, "_derived": 3}, None, None), + ] + states = [ + SnakeState("me", 0, 100, None, _body((5, 5), (5, 5), (5, 5))), + SnakeState("me", 1, 99, 42, _body((4, 5), (5, 5), (5, 5))), + SnakeState("enemy", 1, 99, 7, _body((-1, 2))), + ] + unpacked_turns, unpacked_states = unpack_replay(pack_replay(turns, states)) + self.assertEqual(unpacked_turns, turns) + self.assertEqual(sorted(unpacked_states, key=lambda s: (s.snake_id, s.turn)), sorted(states, key=lambda s: (s.snake_id, s.turn))) + + def test_unknown_format_is_rejected(self): + with self.assertRaises(ValueError): + unpack_replay(b"", 99) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_GameplayDatabase.py b/tests/test_GameplayDatabase.py index 72f32bc..e22b398 100644 --- a/tests/test_GameplayDatabase.py +++ b/tests/test_GameplayDatabase.py @@ -2,6 +2,7 @@ import unittest from pathlib import Path import tempfile, sqlite3 +from contextlib import closing from server.database import GameplayDatabase, GameplayBackendBuilder @@ -59,93 +60,127 @@ class TestGameplayDatabase(unittest.IsolatedAsyncioTestCase): "you": snakes[0], } - async def test_records_gameplay_with_wal_and_inferred_moves(self): + async def _play_game(self, database) -> None: + await database.record_game_start(self._build_state(turn=0, me_head=(1, 1), enemy_head=(5, 5))) + await database.record_turn( + self._build_state(turn=1, me_head=(2, 1), enemy_head=(5, 4)), + my_move="right", + my_thinking={ + "turn": 1, + "reason": "safe_space", + "health": 90, + "length": 3, + "scores": {"right": 1.0}, + }, + ) + await database.record_turn( + self._build_state(turn=2, me_head=(2, 2), enemy_head=(4, 4)), + my_move="up", + my_thinking={"turn": 2, "reason": "food", "scores": {"up": 1.4}}, + ) + + async def test_running_game_uses_compact_rows(self): with tempfile.TemporaryDirectory() as temp_dir: db_path = Path(temp_dir) / "gameplay.sqlite3" database = GameplayDatabase(GameplayBackendBuilder.build(db_path=str(db_path), busy_timeout_ms=4000)) + await self._play_game(database) - await database.record_game_start(self._build_state(turn=0, me_head=(1, 1), enemy_head=(5, 5))) - await database.record_turn( - self._build_state(turn=1, me_head=(2, 1), enemy_head=(5, 4)), - my_move="right", - my_thinking={ - "turn": 1, - "reason": "safe_space", - "scores": {"right": 1.0}, - }, - ) - await database.record_turn( - self._build_state(turn=2, me_head=(2, 2), enemy_head=(4, 4)), - my_move="up", - my_thinking={"turn": 2, "reason": "food", "scores": {"up": 1.4}}, - ) + with closing(sqlite3.connect(str(db_path))) as connection: + self.assertEqual(str(connection.execute("PRAGMA journal_mode").fetchone()[0]).lower(), "wal") + rows = connection.execute(""" + SELECT t.turn, t.my_move, t.thinking_json, t.food + FROM game_turns AS t JOIN games AS g ON g.gid = t.gid + WHERE g.game_id = ? ORDER BY t.turn + """, ("game-abc",)).fetchall() + self.assertEqual([row[0] for row in rows], [1, 2]) + self.assertEqual([row[1] for row in rows], [4, 1]) # right, up + # Derivable thinking fields are stripped; unchanged food is NULL. + self.assertNotIn('"turn"', rows[0][2]) + self.assertNotIn('"health"', rows[0][2]) + self.assertIsNotNone(rows[0][3]) + self.assertIsNone(rows[1][3]) + states = connection.execute("SELECT COUNT(*) FROM snake_states").fetchone()[0] + self.assertEqual(states, 4) + self.assertEqual(connection.execute("SELECT COUNT(*) FROM game_replays").fetchone()[0], 0) + + replay = await database.get_game_replay("game-abc") + self.assertEqual(replay["game"]["status"], "running") + self.assertEqual(replay["turns"][1]["food"], [{"x": 2, "y": 2}]) + + async def test_finished_game_is_packed_and_replays_identically(self): + with tempfile.TemporaryDirectory() as temp_dir: + db_path = Path(temp_dir) / "gameplay.sqlite3" + database = GameplayDatabase(GameplayBackendBuilder.build(db_path=str(db_path), busy_timeout_ms=4000)) + await self._play_game(database) + running_replay = await database.get_game_replay("game-abc") await database.record_game_end(self._build_state(turn=2, me_head=(2, 2), enemy_head=(4, 4), include_enemy=False)) - connection = sqlite3.connect(str(db_path)) - journal_mode = connection.execute("PRAGMA journal_mode").fetchone()[0] - self.assertEqual(str(journal_mode).lower(), "wal") - - games = connection.execute("SELECT status, winner_you, final_turn FROM games WHERE game_id = ?", ("game-abc",)).fetchone() - self.assertEqual(games[0], "finished") - self.assertEqual(games[1], 1) - self.assertEqual(games[2], 2) - - turns_count = connection.execute("SELECT COUNT(*) FROM turns WHERE game_id = ?", ("game-abc",)).fetchone()[0] - self.assertEqual(turns_count, 2) - - compact_turn = connection.execute(""" - SELECT snakes_json, you_json, food_json, hazards_json - FROM turns WHERE game_id = ? AND turn = ? - """, ("game-abc", 2)).fetchone() - self.assertEqual(compact_turn, ("[]", "{}", '[{"x":2,"y":2}]', "[]")) - stored_body = connection.execute(""" - SELECT body_json FROM snake_turns - WHERE game_id = ? AND turn = ? AND snake_id = ? - """, ("game-abc", 2, "me")).fetchone()[0] - self.assertNotEqual(stored_body, "[]") - identities = connection.execute(""" - SELECT snake_id, snake_name, is_you, customizations_json FROM game_snakes - WHERE game_id = ? ORDER BY snake_id - """, ("game-abc",)).fetchall() - self.assertEqual(identities, [ - ("enemy", "Enemy", 0, "{}"), - ("me", "Me", 1, '{"color":"#00ff00","head":"ferret","tail":"swirl"}'), - ]) - repeated_identity = connection.execute(""" - SELECT snake_name, is_you FROM snake_turns - WHERE game_id = ? AND turn = ? AND snake_id = ? - """, ("game-abc", 2, "me")).fetchone() - self.assertEqual(repeated_identity, (None, 0)) - - me_inferred = connection.execute("SELECT inferred_move FROM snake_turns WHERE game_id = ? AND turn = ? AND snake_id = ?", ("game-abc", 2, "me")).fetchone()[0] - enemy_inferred = connection.execute("SELECT inferred_move FROM snake_turns WHERE game_id = ? AND turn = ? AND snake_id = ?", ("game-abc", 2, "enemy")).fetchone()[0] - self.assertEqual(me_inferred, "up") - self.assertEqual(enemy_inferred, "left") + with closing(sqlite3.connect(str(db_path))) as connection: + games = connection.execute("SELECT status, winner_you, final_turn FROM games WHERE game_id = ?", ("game-abc",)).fetchone() + self.assertEqual(games, ("finished", 1, 2)) + self.assertEqual(connection.execute("SELECT COUNT(*) FROM game_turns").fetchone()[0], 0) + self.assertEqual(connection.execute("SELECT COUNT(*) FROM snake_states").fetchone()[0], 0) + self.assertEqual(connection.execute("SELECT turn_count FROM game_replays").fetchone()[0], 2) + identities = connection.execute(""" + SELECT snake_id, snake_name, is_you, customizations_json FROM game_snakes + WHERE game_id = ? ORDER BY snake_id + """, ("game-abc",)).fetchall() + self.assertEqual(identities, [ + ("enemy", "Enemy", 0, "{}"), + ("me", "Me", 1, '{"color":"#00ff00","head":"ferret","tail":"swirl"}'), + ]) summary = await database.get_summary() self.assertEqual(summary["finished_games"], 1) self.assertEqual(summary["wins"], 1) replay = await database.get_game_replay("game-abc") - self.assertIsNotNone(replay) - replay = replay or {} + self.assertEqual(replay["turns"], running_replay["turns"]) self.assertEqual(replay["game"]["final_turn"], 2) - self.assertEqual(len(replay["turns"]), 2) - self.assertEqual(replay["turns"][1]["my_move"], "up") - self.assertEqual(replay["turns"][1]["my_thinking"]["reason"], "food") - self.assertEqual(replay["turns"][1]["food"], [{"x": 2, "y": 2}]) - self.assertEqual(replay["turns"][1]["you"]["id"], "me") - me = next(snake for snake in replay["turns"][1]["snakes"] if snake["snake_id"] == "me") - self.assertEqual(me["customizations"], { - "color": "#00ff00", - "head": "ferret", - "tail": "swirl", - }) + turn = replay["turns"][1] + self.assertEqual(turn["my_move"], "up") + self.assertEqual(turn["my_thinking"], {"turn": 2, "reason": "food", "scores": {"up": 1.4}}) + self.assertEqual(replay["turns"][0]["my_thinking"]["health"], 90) + self.assertEqual(turn["food"], [{"x": 2, "y": 2}]) + self.assertEqual(turn["you"]["id"], "me") + me = next(snake for snake in turn["snakes"] if snake["snake_id"] == "me") + enemy = next(snake for snake in turn["snakes"] if snake["snake_id"] == "enemy") + self.assertEqual(me["inferred_move"], "up") + self.assertEqual(enemy["inferred_move"], "left") + self.assertEqual(me["customizations"], {"color": "#00ff00", "head": "ferret", "tail": "swirl"}) self.assertEqual(len(me["body"]), 3) - api_me = next(snake for snake in replay["turns"][1]["board"]["snakes"] if snake["id"] == "me") + self.assertEqual(turn["snakes"][0]["snake_id"], "me") + api_me = next(snake for snake in turn["board"]["snakes"] if snake["id"] == "me") self.assertEqual(api_me["customizations"], me["customizations"]) - connection.close() + async def test_turn_after_restart_stores_full_food(self): + with tempfile.TemporaryDirectory() as temp_dir: + db_path = Path(temp_dir) / "gameplay.sqlite3" + first = GameplayDatabase(GameplayBackendBuilder.build(db_path=str(db_path))) + await first.record_game_start(self._build_state(turn=0, me_head=(1, 1), enemy_head=(5, 5))) + await first.record_turn(self._build_state(turn=1, me_head=(2, 1), enemy_head=(5, 4)), my_move="right") + second = GameplayDatabase(GameplayBackendBuilder.build(db_path=str(db_path))) + await second.record_turn(self._build_state(turn=2, me_head=(2, 2), enemy_head=(4, 4)), my_move="up") + + replay = await second.get_game_replay("game-abc") + self.assertEqual([turn["food"] for turn in replay["turns"]], [[{"x": 2, "y": 2}]] * 2) + + async def test_stale_running_game_is_finalized_and_packed(self): + with tempfile.TemporaryDirectory() as temp_dir: + db_path = Path(temp_dir) / "gameplay.sqlite3" + database = GameplayDatabase(GameplayBackendBuilder.build(db_path=str(db_path))) + await self._play_game(database) + with closing(sqlite3.connect(str(db_path))) as connection, connection: + connection.execute("UPDATE games SET started_at = '2020-01-01T00:00:00+00:00'") + + self.assertEqual(await database.finalize_stale_running_games(600), 1) + + replay = await database.get_game_replay("game-abc") + self.assertEqual(replay["game"]["status"], "finished") + self.assertFalse(replay["game"]["winner_you"]) + self.assertEqual(len(replay["turns"]), 2) + with closing(sqlite3.connect(str(db_path))) as connection: + self.assertEqual(connection.execute("SELECT COUNT(*) FROM game_turns").fetchone()[0], 0) if __name__ == "__main__": unittest.main() diff --git a/tests/test_GameplaySchema.py b/tests/test_GameplaySchema.py new file mode 100644 index 0000000..942efd4 --- /dev/null +++ b/tests/test_GameplaySchema.py @@ -0,0 +1,120 @@ +import sqlite3 +import tempfile +import unittest +from contextlib import closing +from pathlib import Path + +from server.database.backend import postgresql_schema +from server.database.backend.PostgresqlGameplayBackend import PostgresqlGameplayBackend +from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend + +REDUNDANT = {"idx_turns_game_turn", "idx_snake_turns_game_turn"} + +class _RecordingConnection: + def __init__(self) -> None: + self.statements:list[str] = [] + + async def execute(self, statement:str, *args) -> str: + self.statements.append(statement) + return "OK" + +class _AcquireContext: + def __init__(self, connection) -> None: + self._connection = connection + + async def __aenter__(self): + return self._connection + + async def __aexit__(self, *exc) -> None: + return None + +class _Pool: + def __init__(self, connection) -> None: + self._connection = connection + + def acquire(self): + return _AcquireContext(self._connection) + +class TestPostgresqlSchema(unittest.IsolatedAsyncioTestCase): + def test_ddl_defines_compact_tables_without_legacy_indexes(self): + ddl = postgresql_schema.TABLES_DDL + for table in ("game_turns", "snake_states", "game_replays"): + self.assertIn(f"CREATE TABLE IF NOT EXISTS {table}", ddl) + for name in REDUNDANT: + self.assertNotIn(f"CREATE INDEX IF NOT EXISTS {name}", ddl) + self.assertIn("SET STORAGE EXTERNAL", ddl) + + async def test_schema_init_is_locked_and_drops_redundant_indexes_separately(self): + connection = _RecordingConnection() + backend = PostgresqlGameplayBackend("postgresql://example") + backend._pool = _Pool(connection) + + await backend._initialize_schema() + + self.assertIn("pg_advisory_lock", connection.statements[0]) + drops = [s for s in connection.statements if "DROP INDEX" in s] + self.assertEqual({s.split()[-1] for s in drops}, REDUNDANT) + for statement in drops: + # CONCURRENTLY only works as a standalone statement outside a transaction. + self.assertIn("CONCURRENTLY IF EXISTS", statement) + self.assertNotIn(";", statement) + +class TestSqliteSchema(unittest.TestCase): + def _objects(self, db_path:Path, kind:str) -> set[str]: + with closing(sqlite3.connect(db_path)) as connection: + return {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type = ?", (kind,))} + + def test_new_database_has_compact_tables(self): + with tempfile.TemporaryDirectory() as temp_dir: + db_path = Path(temp_dir) / "gameplay.sqlite3" + SqliteGameplayBackend(str(db_path)) + tables = self._objects(db_path, "table") + self.assertTrue({"games", "game_snakes", "game_turns", "snake_states", "game_replays"} <= tables) + self.assertFalse({"turns", "snake_turns"} & tables) + self.assertFalse(self._objects(db_path, "index") & REDUNDANT) + + def test_legacy_database_gets_ids_and_keeps_legacy_tables(self): + with tempfile.TemporaryDirectory() as temp_dir: + db_path = Path(temp_dir) / "gameplay.sqlite3" + with closing(sqlite3.connect(db_path)) as connection, connection: + connection.executescript(""" + CREATE TABLE games (game_id TEXT PRIMARY KEY, started_at TEXT NOT NULL, + winner_you INTEGER NOT NULL DEFAULT 0, final_turn INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'running', width INTEGER, height INTEGER, + source TEXT, map_name TEXT, ruleset_name TEXT, ruleset_version TEXT, + your_snake_id TEXT, your_snake_name TEXT); + CREATE TABLE game_snakes (game_id TEXT NOT NULL, snake_id TEXT NOT NULL, + snake_name TEXT, is_you INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (game_id, snake_id)); + CREATE TABLE turns (game_id TEXT, turn INTEGER); + CREATE TABLE snake_turns (game_id TEXT, turn INTEGER, snake_id TEXT); + CREATE INDEX idx_turns_game_turn ON turns(game_id, turn); + INSERT INTO games (game_id, started_at) VALUES ('a', '2026-01-01'), ('b', '2026-01-02'); + INSERT INTO game_snakes (game_id, snake_id) VALUES ('a', 's1'), ('a', 's2'); + """) + + SqliteGameplayBackend(str(db_path)) + + with closing(sqlite3.connect(db_path)) as connection: + gids = [row[0] for row in connection.execute("SELECT gid FROM games ORDER BY game_id")] + gsids = [row[0] for row in connection.execute("SELECT gsid FROM game_snakes ORDER BY snake_id")] + self.assertEqual(len(set(gids)), 2) + self.assertNotIn(None, gids) + self.assertEqual(len(set(gsids)), 2) + self.assertNotIn(None, gsids) + self.assertTrue({"turns", "snake_turns"} <= self._objects(db_path, "table")) + self.assertFalse(self._objects(db_path, "index") & REDUNDANT) + + def test_running_game_lookups_use_primary_keys(self): + with tempfile.TemporaryDirectory() as temp_dir: + db_path = Path(temp_dir) / "gameplay.sqlite3" + SqliteGameplayBackend(str(db_path)) + with closing(sqlite3.connect(db_path)) as connection: + for query in ( + "SELECT * FROM game_turns WHERE gid = 1 ORDER BY turn", + "SELECT * FROM snake_states WHERE gsid = 1 AND turn = 3", + ): + plan = " ".join(str(row[-1]) for row in connection.execute(f"EXPLAIN QUERY PLAN {query}")) + self.assertIn("PRIMARY KEY", plan, plan) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_LegacyMigration.py b/tests/test_LegacyMigration.py new file mode 100644 index 0000000..2d755b6 --- /dev/null +++ b/tests/test_LegacyMigration.py @@ -0,0 +1,176 @@ +import asyncio +import json +import sqlite3 +import subprocess +import sys +import tempfile +import unittest +from contextlib import closing +from pathlib import Path + +from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend +from server.database.legacy.convert import LegacyDataError, build_legacy_bundle +from server.database.legacy.migrate import migrate_legacy +from server.database.legacy.reader_sqlite import LegacySqliteReader + +REPO_ROOT = Path(__file__).resolve().parents[1] + +LEGACY_SCHEMA = """ +CREATE TABLE games ( + game_id TEXT PRIMARY KEY, started_at TEXT NOT NULL, ended_at TEXT, + width INTEGER, height INTEGER, source TEXT, map_name TEXT, + ruleset_name TEXT, ruleset_version TEXT, your_snake_id TEXT, + your_snake_name TEXT, your_snake_type TEXT, your_snake_version TEXT, + game_type TEXT, winner_name TEXT, winner_you INTEGER NOT NULL DEFAULT 0, + final_turn INTEGER NOT NULL DEFAULT 0, status TEXT NOT NULL DEFAULT 'running' +); +CREATE TABLE game_snakes ( + game_id TEXT NOT NULL, snake_id TEXT NOT NULL, snake_name TEXT, + is_you INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (game_id, snake_id) +); +CREATE TABLE turns ( + id INTEGER PRIMARY KEY AUTOINCREMENT, game_id TEXT NOT NULL, turn INTEGER NOT NULL, + observed_at TEXT NOT NULL, my_move TEXT, my_thinking_json TEXT, + board_state_json TEXT NOT NULL, snakes_json TEXT NOT NULL, you_json TEXT NOT NULL, + food_json TEXT NOT NULL, hazards_json TEXT NOT NULL, UNIQUE (game_id, turn) +); +CREATE TABLE snake_turns ( + id INTEGER PRIMARY KEY AUTOINCREMENT, game_id TEXT NOT NULL, turn INTEGER NOT NULL, + snake_id TEXT NOT NULL, snake_name TEXT, health INTEGER, length INTEGER, + head_x INTEGER, head_y INTEGER, body_json TEXT NOT NULL, is_you INTEGER NOT NULL DEFAULT 0, + inferred_move TEXT, latency TEXT, UNIQUE (game_id, turn, snake_id) +); +""" + +def _body(*points): + return [{"x": x, "y": y} for x, y in points] + +THINKING = {"turn": 1, "health": 99, "length": 3, "food_count": 1, "board_size": "11x11", "scores": {"up": 2.5}} + +def create_legacy_database(path:Path, game_snakes:bool=True) -> None: + with closing(sqlite3.connect(path)) as connection, connection: + connection.executescript(LEGACY_SCHEMA) + connection.execute(""" + INSERT INTO games (game_id, started_at, ended_at, width, height, source, map_name, + ruleset_name, ruleset_version, your_snake_id, your_snake_name, game_type, + winner_name, winner_you, final_turn, status) + VALUES ('g1', '2026-08-01T10:00:00+00:00', '2026-08-01T10:01:00+00:00', 11, 11, + 'league', 'standard', 'standard', 'v1', 'me', 'Me', 'duel', 'Me', 1, 2, 'finished') + """) + if game_snakes: + connection.executemany( + "INSERT INTO game_snakes VALUES (?, ?, ?, ?)", + [("g1", "me", "Me", 1), ("g1", "enemy", "Enemy", 0)], + ) + food = json.dumps(_body((5, 5))) + for turn, move, thinking in ((0, None, None), (1, "up", THINKING), (2, "left", {"scores": {"left": 1}})): + connection.execute(""" + INSERT INTO turns (game_id, turn, observed_at, my_move, my_thinking_json, + board_state_json, snakes_json, you_json, food_json, hazards_json) + VALUES ('g1', ?, ?, ?, ?, '{}', '[]', '{}', ?, '[]') + """, (turn, f"2026-08-01T10:00:0{turn}.123456+00:00", move, json.dumps(thinking) if thinking else None, food)) + me = [_body((1, 1), (1, 1), (1, 1)), _body((1, 2), (1, 1), (1, 1)), _body((0, 2), (1, 2), (1, 1))] + enemy = [_body((9, 9), (9, 9), (9, 9)), _body((9, 8), (9, 9), (9, 9)), _body((9, 7), (9, 8), (9, 9))] + for turn in range(3): + for snake_id, bodies, latency in (("me", me, "12"), ("enemy", enemy, "")): + body = bodies[turn] + connection.execute(""" + INSERT INTO snake_turns (game_id, turn, snake_id, snake_name, health, length, + head_x, head_y, body_json, is_you, latency) + VALUES ('g1', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, ( + turn, snake_id, None if game_snakes else snake_id.title(), 100 - turn, len(body), + body[0]["x"], body[0]["y"], json.dumps(body), int(snake_id == "me" and not game_snakes), latency, + )) + +class TestLegacyConvert(unittest.TestCase): + def test_rejects_inconsistent_rows_instead_of_losing_data(self): + with self.assertRaises(LegacyDataError): + build_legacy_bundle( + {"game_id": "g", "started_at": "2026-01-01"}, [], [], + [{"snake_id": "s", "turn": 0, "length": 5, "body": json.dumps(_body((1, 1)))}], + ) + + def test_invalid_json_aborts(self): + with self.assertRaises(LegacyDataError): + build_legacy_bundle( + {"game_id": "g", "started_at": "2026-01-01"}, [], + [{"turn": 0, "observed_at": "2026-01-01", "my_thinking": "{invalid"}], [], + ) + +class TestLegacyMigration(unittest.TestCase): + def _migrate(self, source:Path, destination:Path, **options): + async def run(): + backend = SqliteGameplayBackend(str(destination)) + with LegacySqliteReader(source) as reader: + stats = await migrate_legacy(reader, backend, **options) + return stats, await backend.get_game_replay("g1") + return asyncio.run(run()) + + def test_migrates_finished_game_into_identical_replay(self): + with tempfile.TemporaryDirectory() as temp_dir: + source, destination = Path(temp_dir) / "legacy.sqlite3", Path(temp_dir) / "compact.sqlite3" + create_legacy_database(source) + + stats, replay = self._migrate(source, destination) + + self.assertEqual((stats.games, stats.turns, stats.states, stats.failed), (1, 3, 6, 0)) + turns = replay["turns"] + self.assertEqual([turn["my_move"] for turn in turns], [None, "up", "left"]) + self.assertEqual(turns[1]["my_thinking"], THINKING) + self.assertEqual(turns[2]["food"], _body((5, 5))) + self.assertEqual(turns[1]["observed_at"], "2026-08-01T10:00:01.123456+00:00") + me = turns[2]["snakes"][0] + self.assertEqual((me["snake_id"], me["snake_name"], me["is_you"]), ("me", "Me", True)) + self.assertEqual((me["inferred_move"], me["latency"], me["health"], me["length"]), ("left", "12", 98, 3)) + enemy = turns[2]["snakes"][1] + self.assertEqual((enemy["inferred_move"], enemy["latency"]), ("down", None)) + with closing(sqlite3.connect(destination)) as connection: + self.assertEqual(connection.execute("SELECT turn_count FROM game_replays").fetchone()[0], 3) + + def test_synthesizes_missing_game_snakes_from_snake_rows(self): + with tempfile.TemporaryDirectory() as temp_dir: + source, destination = Path(temp_dir) / "legacy.sqlite3", Path(temp_dir) / "compact.sqlite3" + create_legacy_database(source, game_snakes=False) + + _, replay = self._migrate(source, destination) + + snakes = replay["turns"][0]["snakes"] + self.assertEqual([(s["snake_id"], s["snake_name"], s["is_you"]) for s in snakes], [ + ("me", "Me", True), ("enemy", "Enemy", False), + ]) + self.assertEqual(snakes[0]["customizations"], {}) + + def test_quality_filter_keeps_result_but_drops_replay(self): + with tempfile.TemporaryDirectory() as temp_dir: + source, destination = Path(temp_dir) / "legacy.sqlite3", Path(temp_dir) / "compact.sqlite3" + create_legacy_database(source) + + stats, replay = self._migrate(source, destination, minimum_quality="high") + + self.assertEqual(stats.dropped_replays, 1) + self.assertIsNone(replay) + with closing(sqlite3.connect(destination)) as connection: + row = connection.execute("SELECT has_replay, quality_status, winner_you FROM games").fetchone() + self.assertEqual(row, (0, "low_quality", 1)) + self.assertEqual(connection.execute("SELECT COUNT(*) FROM game_replays").fetchone()[0], 0) + + def test_script_migrates_in_place_and_drops_legacy_tables(self): + with tempfile.TemporaryDirectory() as temp_dir: + database = Path(temp_dir) / "gameplay.sqlite3" + create_legacy_database(database) + script = [sys.executable, "scripts/migrate_gameplay_to_compact.py", "--source", str(database)] + + migrated = subprocess.run(script, cwd=REPO_ROOT, text=True, capture_output=True) + dropped = subprocess.run(script + ["--drop-legacy"], cwd=REPO_ROOT, text=True, capture_output=True) + + self.assertEqual(migrated.returncode, 0, migrated.stderr) + self.assertEqual(dropped.returncode, 0, dropped.stderr) + with closing(sqlite3.connect(database)) as connection: + tables = {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type = 'table'")} + self.assertFalse({"turns", "snake_turns"} & tables) + replay = asyncio.run(SqliteGameplayBackend(str(database)).get_game_replay("g1")) + self.assertEqual(len(replay["turns"]), 3) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_MergeGameplayDatabases.py b/tests/test_MergeGameplayDatabases.py index f7242e5..48dce87 100644 --- a/tests/test_MergeGameplayDatabases.py +++ b/tests/test_MergeGameplayDatabases.py @@ -1,104 +1,98 @@ +import json import sqlite3 import subprocess import sys import tempfile -from pathlib import Path import unittest +from contextlib import closing +from datetime import datetime, timezone +from pathlib import Path from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend +from server.database.compact.records import GameBundle, SnakeIdentity, SnakeState, TurnRecord + +REPO_ROOT = Path(__file__).resolve().parents[1] +CUSTOMIZATIONS = {"color": "#663399", "head": "ferret", "tail": "swirl"} + +def _bundle(game_id:str, winner_you:bool, cleaned:bool=False) -> GameBundle: + turns = [ + TurnRecord(turn, datetime(2026, 1, 1, 0, 0, turn, tzinfo=timezone.utc), "up", {"score": turn}, [] if turn == 1 else None, [] if turn == 1 else None) + for turn in range(1, 6) + ] + states = [ + SnakeState(snake_id, turn, 90, 10, [{"x": x, "y": turn}]) + for turn in range(1, 6) + for snake_id, x in (("me", 1), ("a", 3), ("b", 5)) + ] + return GameBundle( + game={ + "game_id": game_id, "started_at": "2026-01-01T00:00:00+00:00", + "ended_at": "2026-01-01T00:01:00+00:00", "width": 11, "height": 11, + "source": "league", "map_name": "standard", "ruleset_name": "standard", + "ruleset_version": "v1", "your_snake_id": "me", "your_snake_name": "PrismBattleSnake", + "your_snake_type": "PrismBattleSnake_GPT_5_6_Sol", "your_snake_version": "1.0.0", + "game_type": "standard", "winner_name": "PrismBattleSnake", "winner_you": winner_you, + "final_turn": 5, "status": "finished", "has_replay": True, "quality_status": "retained", + "quality_score": 90 if cleaned else None, "quality_tier": "high" if cleaned else None, + "quality_reasons": ["already_scored"] if cleaned else None, + }, + snakes=[ + SnakeIdentity("me", "PrismBattleSnake", True, CUSTOMIZATIONS), + SnakeIdentity("a", "A", False, {}), + SnakeIdentity("b", "B", False, {}), + ], + turns=turns, + states=states, + ) class TestMergeGameplayDatabases(unittest.TestCase): - def _create_game(self, path:Path, game_id:str, winner_you:int, cleaned:bool=False) -> None: - SqliteGameplayBackend(str(path)) - with sqlite3.connect(path) as connection: - connection.execute(""" - INSERT INTO games ( - game_id, started_at, ended_at, width, height, source, map_name, - ruleset_name, ruleset_version, your_snake_id, your_snake_name, - your_snake_type, your_snake_version, game_type, winner_name, - winner_you, final_turn, status, has_replay, quality_status, - quality_score, quality_tier, quality_reasons_json - ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) - """, ( - game_id, "2026-01-01T00:00:00Z", "2026-01-01T00:01:00Z", 11, 11, - "league", "standard", "standard", "v1", "me", "PrismBattleSnake", - "PrismBattleSnake_GPT_5_6_Sol", "1.0.0", "duel", "PrismBattleSnake", - winner_you, 1, "finished", 1, "retained", - 90 if cleaned else None, "high" if cleaned else None, - '["already_scored"]' if cleaned else None, - )) - connection.execute(""" - INSERT INTO game_snakes ( - game_id,snake_id,snake_name,is_you,customizations_json - ) VALUES (?,?,?,?,?) - """, ( - game_id, "me", "PrismBattleSnake", 1, - '{"color":"#663399","head":"ferret","tail":"swirl"}', - )) - connection.execute(""" - INSERT INTO turns ( - game_id,turn,observed_at,my_move,my_thinking_json, - board_state_json,snakes_json,you_json,food_json,hazards_json - ) VALUES (?,?,?,?,?,?,?,?,?,?) - """, (game_id, 1, "2026-01-01T00:00:01Z", "up", '{"score":1}', '{}', '[]', '{}', '[]', '[]')) - connection.execute(""" - INSERT INTO snake_turns ( - game_id,turn,snake_id,health,length,head_x,head_y,body_json,is_you,inferred_move - ) VALUES (?,?,?,?,?,?,?,?,?,?) - """, (game_id, 1, "me", 90, 3, 1, 1, '[{"x":1,"y":1}]', 0, "up")) + def _create(self, path:Path, bundle:GameBundle) -> None: + SqliteGameplayBackend(str(path)).import_game_bundle_sync(bundle) - def test_merges_base_and_delta_and_regenerates_row_ids(self): + def _merge(self, root:Path, *extra:str): + return subprocess.run([ + sys.executable, "scripts/merge_gameplay_databases.py", + "--base", str(root / "base.sqlite3"), "--delta", str(root / "delta.sqlite3"), + "--destination", str(root / "merged.sqlite3"), *extra, + ], cwd=REPO_ROOT, text=True, capture_output=True) + + def test_merges_base_and_delta_with_fresh_ids(self): with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) - base = root / "base.sqlite3" - delta = root / "delta.sqlite3" - merged = root / "merged.sqlite3" - self._create_game(base, "base-game", 1, cleaned=True) - self._create_game(delta, "delta-game", 0) + self._create(root / "base.sqlite3", _bundle("base-game", True, cleaned=True)) + self._create(root / "delta.sqlite3", _bundle("delta-game", False)) - result = subprocess.run([ - sys.executable, "scripts/merge_gameplay_databases.py", - "--base", str(base), "--delta", str(delta), - "--destination", str(merged), "--minimum-quality", "medium", - ], cwd=Path(__file__).resolve().parents[1], text=True, capture_output=True) + result = self._merge(root, "--minimum-quality", "medium") self.assertEqual(result.returncode, 0, result.stderr) - with sqlite3.connect(merged) as connection: + merged = SqliteGameplayBackend(str(root / "merged.sqlite3")) + with closing(sqlite3.connect(root / "merged.sqlite3")) as connection: games = connection.execute( - "SELECT game_id, winner_you, has_replay, quality_tier FROM games ORDER BY game_id" + "SELECT game_id, gid, winner_you, has_replay, quality_tier FROM games ORDER BY game_id" ).fetchall() - self.assertEqual(games, [ - ("base-game", 1, 1, "high"), - ("delta-game", 0, 1, "medium"), - ]) - self.assertEqual(connection.execute("SELECT COUNT(*) FROM turns").fetchone()[0], 2) - self.assertEqual(connection.execute("SELECT COUNT(*) FROM snake_turns").fetchone()[0], 2) - customizations = connection.execute(""" - SELECT game_id, customizations_json FROM game_snakes ORDER BY game_id - """).fetchall() - self.assertEqual(customizations, [ - ("base-game", '{"color":"#663399","head":"ferret","tail":"swirl"}'), - ("delta-game", '{"color":"#663399","head":"ferret","tail":"swirl"}'), - ]) + self.assertEqual(connection.execute("SELECT COUNT(*) FROM game_replays").fetchone()[0], 2) self.assertEqual(connection.execute("PRAGMA foreign_key_check").fetchall(), []) + self.assertEqual([(g[0], g[2], g[3]) for g in games], [("base-game", 1, 1), ("delta-game", 0, 1)]) + self.assertEqual(games[0][4], "high") + self.assertEqual(len({g[1] for g in games}), 2) + for game_id in ("base-game", "delta-game"): + bundle = merged.read_game_bundle_sync(game_id) + self.assertEqual(len(bundle.turns), 5) + self.assertEqual(len(bundle.states), 15) + me = next(snake for snake in bundle.snakes if snake.snake_id == "me") + self.assertEqual(me.customizations, CUSTOMIZATIONS) def test_conflicting_duplicate_aborts_without_destination(self): with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) - base = root / "base.sqlite3" - delta = root / "delta.sqlite3" - merged = root / "merged.sqlite3" - self._create_game(base, "same-game", 1, cleaned=True) - self._create_game(delta, "same-game", 0) + self._create(root / "base.sqlite3", _bundle("same-game", True, cleaned=True)) + self._create(root / "delta.sqlite3", _bundle("same-game", False)) - result = subprocess.run([ - sys.executable, "scripts/merge_gameplay_databases.py", - "--base", str(base), "--delta", str(delta), "--destination", str(merged), - ], cwd=Path(__file__).resolve().parents[1], text=True, capture_output=True) + result = self._merge(root) self.assertNotEqual(result.returncode, 0) self.assertIn("Conflicting duplicate game_id", result.stderr) - self.assertFalse(merged.exists()) + self.assertFalse((root / "merged.sqlite3").exists()) if __name__ == "__main__": unittest.main() diff --git a/tests/test_MigrateGameplayDatabase.py b/tests/test_MigrateGameplayDatabase.py deleted file mode 100644 index 81d9fcf..0000000 --- a/tests/test_MigrateGameplayDatabase.py +++ /dev/null @@ -1,137 +0,0 @@ -import sqlite3 -import tempfile -import unittest -from pathlib import Path - -from scripts.migrate_gameplay_database import copy_game_snakes -from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend - -class TestMigrateGameplayDatabase(unittest.TestCase): - def test_copy_game_snakes_preserves_customizations(self): - with tempfile.TemporaryDirectory() as temp_dir: - root = Path(temp_dir) - source_path = root / "source.sqlite3" - destination_path = root / "destination.sqlite3" - SqliteGameplayBackend(str(source_path)) - SqliteGameplayBackend(str(destination_path)) - - with sqlite3.connect(source_path) as source: - source.execute("PRAGMA foreign_keys = OFF") - source.execute(""" - INSERT INTO game_snakes ( - game_id, snake_id, snake_name, is_you, customizations_json - ) VALUES (?, ?, ?, ?, ?) - """, ( - "game-1", "snake-1", "PrismBattleSnake", 1, - '{"color":"#663399","head":"ferret","tail":"swirl"}', - )) - - source = sqlite3.connect(source_path) - destination = sqlite3.connect(destination_path) - try: - copied = copy_game_snakes( - source, destination, batch_size=10, retained_ids={"game-1"}, - ) - destination.commit() - row = destination.execute(""" - SELECT snake_name, is_you, customizations_json - FROM game_snakes WHERE game_id = ? AND snake_id = ? - """, ("game-1", "snake-1")).fetchone() - finally: - source.close() - destination.close() - - self.assertEqual(copied, 1) - self.assertEqual(row, ( - "PrismBattleSnake", 1, - '{"color":"#663399","head":"ferret","tail":"swirl"}', - )) - - def test_copy_game_snakes_synthesizes_rows_when_table_is_empty(self): - source = sqlite3.connect(":memory:") - destination = sqlite3.connect(":memory:") - try: - source.execute(""" - CREATE TABLE game_snakes ( - game_id TEXT, snake_id TEXT, snake_name TEXT, is_you INTEGER, - customizations_json TEXT NOT NULL DEFAULT '{}' - ) - """) - source.execute(""" - CREATE TABLE snake_turns ( - game_id TEXT, snake_id TEXT, snake_name TEXT, is_you INTEGER - ) - """) - source.executemany( - "INSERT INTO snake_turns VALUES (?, ?, ?, ?)", - [ - ("game-1", "snake-1", "PrismBattleSnake", 1), - ("game-1", "snake-1", "PrismBattleSnake", 1), - ("game-1", "snake-2", "Enemy", 0), - ], - ) - destination.execute(""" - CREATE TABLE game_snakes ( - game_id TEXT, snake_id TEXT, snake_name TEXT, is_you INTEGER, - customizations_json TEXT NOT NULL DEFAULT '{}', - PRIMARY KEY (game_id, snake_id) - ) - """) - - copied = copy_game_snakes( - source, destination, batch_size=10, retained_ids={"game-1"}, - ) - rows = destination.execute(""" - SELECT snake_id, snake_name, is_you, customizations_json - FROM game_snakes ORDER BY snake_id - """).fetchall() - finally: - source.close() - destination.close() - - self.assertEqual(copied, 2) - self.assertEqual(rows, [ - ("snake-1", "PrismBattleSnake", 1, "{}"), - ("snake-2", "Enemy", 0, "{}"), - ]) - - def test_copy_game_snakes_defaults_legacy_schema_to_empty_customizations(self): - source = sqlite3.connect(":memory:") - destination = sqlite3.connect(":memory:") - try: - source.execute(""" - CREATE TABLE game_snakes ( - game_id TEXT, snake_id TEXT, snake_name TEXT, is_you INTEGER - ) - """) - source.execute( - "INSERT INTO game_snakes VALUES (?, ?, ?, ?)", - ("game-1", "snake-1", "LegacySnake", 0), - ) - source.execute(""" - CREATE TABLE snake_turns ( - game_id TEXT, snake_id TEXT, snake_name TEXT, is_you INTEGER - ) - """) - destination.execute(""" - CREATE TABLE game_snakes ( - game_id TEXT, snake_id TEXT, snake_name TEXT, is_you INTEGER, - customizations_json TEXT NOT NULL DEFAULT '{}' - ) - """) - - copied = copy_game_snakes( - source, destination, batch_size=10, retained_ids={"game-1"}, - ) - row = destination.execute( - "SELECT customizations_json FROM game_snakes" - ).fetchone() - finally: - source.close() - destination.close() - - self.assertEqual(copied, 1) - self.assertEqual(row, ("{}",)) - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_MigrateSqliteToPostgresql.py b/tests/test_MigrateSqliteToPostgresql.py deleted file mode 100644 index a5258b7..0000000 --- a/tests/test_MigrateSqliteToPostgresql.py +++ /dev/null @@ -1,34 +0,0 @@ -import unittest - -from scripts.migrate_sqlite_to_postgresql import parse_json, parse_timestamp, transform_row - -class TestMigrateSqliteToPostgresql(unittest.TestCase): - def test_transform_row_converts_timestamp_boolean_and_json(self): - columns = ("started_at", "winner_you", "quality_reasons", "game_id") - row = ("2026-08-01T10:20:30+00:00", 1, '["complete"]', "game-1") - - transformed = transform_row(columns, row) - - self.assertEqual(transformed[0].isoformat(), "2026-08-01T10:20:30+00:00") - self.assertIs(transformed[1], True) - self.assertEqual(transformed[2], '["complete"]') - self.assertEqual(transformed[3], "game-1") - - def test_json_defaults_match_not_null_postgresql_columns(self): - columns = ("customizations", "board_state", "snakes", "you", "food", "hazards", "body") - self.assertEqual( - transform_row(columns, (None,) * len(columns)), - ("{}", "{}", "[]", "{}", "[]", "[]", "[]"), - ) - - def test_invalid_json_aborts_instead_of_silently_losing_data(self): - with self.assertRaisesRegex(ValueError, "Invalid JSON"): - parse_json("{invalid") - - def test_z_suffix_timestamp_is_timezone_aware(self): - parsed = parse_timestamp("2026-08-01T10:20:30Z") - self.assertIsNotNone(parsed.tzinfo) - self.assertEqual(parsed.utcoffset().total_seconds(), 0) - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_PostgresqlGameplayBackend.py b/tests/test_PostgresqlGameplayBackend.py new file mode 100644 index 0000000..571b3df --- /dev/null +++ b/tests/test_PostgresqlGameplayBackend.py @@ -0,0 +1,90 @@ +"""Integration tests against a disposable PostgreSQL database. + +Set GAMEPLAY_TEST_PG_DSN to a database that may be wiped, e.g. +postgresql://user:pass@127.0.0.1:5432/battlesnake_test +""" +import os +import unittest + +from server.database.backend.PostgresqlGameplayBackend import PostgresqlGameplayBackend +import test_GameplayDatabase as sqlite_tests + +DSN = os.environ.get("GAMEPLAY_TEST_PG_DSN") + +@unittest.skipUnless(DSN, "GAMEPLAY_TEST_PG_DSN not set") +class TestPostgresqlGameplayBackend(unittest.IsolatedAsyncioTestCase): + _build_state = sqlite_tests.TestGameplayDatabase._build_state + _play_game = sqlite_tests.TestGameplayDatabase._play_game + + async def asyncSetUp(self): + self.backend = PostgresqlGameplayBackend(DSN) + pool = await self.backend._get_pool() + async with pool.acquire() as conn: + await conn.execute("TRUNCATE games CASCADE") + + async def asyncTearDown(self): + await self.backend.close() + + async def _count(self, table:str) -> int: + pool = await self.backend._get_pool() + async with pool.acquire() as conn: + return int(await conn.fetchval(f"SELECT COUNT(*) FROM {table}")) + + async def test_running_rows_then_packed_replay_is_identical(self): + await self._play_game(self.backend) + self.assertEqual(await self._count("game_turns"), 2) + self.assertEqual(await self._count("snake_states"), 4) + running = await self.backend.get_game_replay("game-abc") + + await self.backend.record_game_end( + self._build_state(turn=2, me_head=(2, 2), enemy_head=(4, 4), include_enemy=False) + ) + + self.assertEqual(await self._count("game_turns"), 0) + self.assertEqual(await self._count("snake_states"), 0) + self.assertEqual(await self._count("game_replays"), 1) + finished = await self.backend.get_game_replay("game-abc") + self.assertEqual(finished["turns"], running["turns"]) + self.assertEqual(finished["game"]["status"], "finished") + turn = finished["turns"][1] + self.assertEqual(turn["my_thinking"], {"turn": 2, "reason": "food", "scores": {"up": 1.4}}) + self.assertEqual(finished["turns"][0]["my_thinking"]["health"], 90) + me = next(snake for snake in turn["snakes"] if snake["snake_id"] == "me") + self.assertEqual(me["inferred_move"], "up") + self.assertEqual(me["customizations"]["head"], "ferret") + + async def test_stale_game_is_finalized_and_packed(self): + await self._play_game(self.backend) + pool = await self.backend._get_pool() + async with pool.acquire() as conn: + await conn.execute("UPDATE games SET started_at = now() - interval '1 day'") + + self.assertEqual(await self.backend.finalize_stale_running_games(600), 1) + + self.assertEqual(await self._count("game_turns"), 0) + replay = await self.backend.get_game_replay("game-abc") + self.assertEqual(len(replay["turns"]), 2) + + async def test_late_turn_after_end_is_merged_and_repacked(self): + await self._play_game(self.backend) + await self.backend.record_game_end(self._build_state(turn=2, me_head=(2, 2), enemy_head=(4, 4))) + await self.backend.record_turn(self._build_state(turn=3, me_head=(3, 2), enemy_head=(3, 4)), my_move="right") + + self.assertEqual(len((await self.backend.get_game_replay("game-abc"))["turns"]), 3) + await self.backend.finalize_stale_running_games(600) + self.assertEqual(await self._count("game_turns"), 0) + self.assertEqual(len((await self.backend.get_game_replay("game-abc"))["turns"]), 3) + + async def test_bundle_roundtrip(self): + await self._play_game(self.backend) + await self.backend.record_game_end(self._build_state(turn=2, me_head=(2, 2), enemy_head=(4, 4))) + bundle = await self.backend.read_game_bundle("game-abc") + bundle.game["game_id"] = "copy" + await self.backend.import_game_bundle(bundle) + + copy = await self.backend.read_game_bundle("copy") + self.assertEqual(copy.turns, bundle.turns) + self.assertEqual(sorted(copy.states, key=lambda s: (s.turn, s.snake_id)), sorted(bundle.states, key=lambda s: (s.turn, s.snake_id))) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_PostgresqlGameplayMigration.py b/tests/test_PostgresqlGameplayMigration.py deleted file mode 100644 index e2a09d4..0000000 --- a/tests/test_PostgresqlGameplayMigration.py +++ /dev/null @@ -1,65 +0,0 @@ -import sqlite3 -import tempfile -import unittest -from pathlib import Path - -from server.database.backend.PostgresqlGameplayBackend import PostgresqlGameplayBackend - -class TestPostgresqlGameplayMigration(unittest.TestCase): - def _create_source(self, path:Path, winner_column:str, winner_value:str|None) -> None: - with sqlite3.connect(path) as connection: - connection.executescript(f""" - CREATE TABLE games ( - game_id TEXT PRIMARY KEY, started_at TEXT NOT NULL, ended_at TEXT, - width INTEGER, height INTEGER, source TEXT, map_name TEXT, - ruleset_name TEXT, ruleset_version TEXT, your_snake_id TEXT, - your_snake_name TEXT, your_snake_type TEXT, your_snake_version TEXT, - game_type TEXT, {winner_column} TEXT, winner_you INTEGER, - final_turn INTEGER, status TEXT - ); - CREATE TABLE turns ( - game_id TEXT, turn INTEGER, observed_at TEXT, my_move TEXT, - my_thinking_json TEXT, board_state_json TEXT, snakes_json TEXT, - you_json TEXT, food_json TEXT, hazards_json TEXT - ); - CREATE TABLE snake_turns ( - game_id TEXT, turn INTEGER, snake_id TEXT, snake_name TEXT, - health INTEGER, length INTEGER, head_x INTEGER, head_y INTEGER, - body_json TEXT, is_you INTEGER, inferred_move TEXT, latency TEXT - ); - """) - connection.execute( - f"""INSERT INTO games ( - game_id, started_at, {winner_column}, winner_you, final_turn, status - ) VALUES (?, ?, ?, ?, ?, ?)""", - ("game-1", "2026-08-01T10:00:00+00:00", winner_value, 1, 4, "finished"), - ) - - def test_reads_current_winner_name_schema(self): - with tempfile.TemporaryDirectory() as temp_dir: - path = Path(temp_dir) / "gameplay.sqlite3" - self._create_source(path, "winner_name", "Prism") - backend = PostgresqlGameplayBackend("postgresql://example", sqlite_migration_path=str(path)) - - games, turns, snake_turns = backend._read_sqlite_data_sync(str(path)) - - self.assertEqual(games[0]["winner_name"], "Prism") - self.assertEqual(turns, []) - self.assertEqual(snake_turns, []) - self.assertEqual(backend._migrated_winner_name(games[0]["winner_name"]), "Prism") - - def test_reads_legacy_winner_names_json_schema(self): - with tempfile.TemporaryDirectory() as temp_dir: - path = Path(temp_dir) / "gameplay.sqlite3" - self._create_source(path, "winner_names_json", '["Prism"]') - backend = PostgresqlGameplayBackend("postgresql://example", sqlite_migration_path=str(path)) - - games, _, _ = backend._read_sqlite_data_sync(str(path)) - - self.assertEqual( - backend._migrated_winner_name(games[0]["winner_name"]), - "Prism", - ) - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_ZstdCompression.py b/tests/test_ZstdCompression.py deleted file mode 100644 index 592c182..0000000 --- a/tests/test_ZstdCompression.py +++ /dev/null @@ -1,92 +0,0 @@ -import unittest, sqlite3, os -from pathlib import Path - -from server.database import GameplayDatabase - -EXT_PATH = Path(os.environ.get("SQLITE_ZSTD_EXT", "lib/libsqlite_zstd.so")).expanduser().resolve() - -def _open(db_path: str) -> sqlite3.Connection: - conn = sqlite3.connect(db_path) - conn.row_factory = sqlite3.Row - if EXT_PATH.exists(): - conn.enable_load_extension(True) - conn.load_extension(str(EXT_PATH)) - conn.enable_load_extension(False) - return conn - -@unittest.skipUnless(EXT_PATH.exists(), f"sqlite-zstd extension not found at {EXT_PATH}") -class TestZstdCompression(unittest.TestCase): - - def test_extension_loads(self): - conn = sqlite3.connect(":memory:") - conn.enable_load_extension(True) - conn.load_extension(str(EXT_PATH)) - conn.enable_load_extension(False) - result = conn.execute("SELECT zstd_compress('hello world', 6)").fetchone()[0] - self.assertIsInstance(result, bytes) - self.assertGreater(len(result), 0) - conn.close() - - def test_compress_decompress_roundtrip(self): - # Use a long repetitive string so compression actually reduces size - original = '{"board": {"width": 11, "height": 11, "snakes": []}, "turn": 42} ' * 20 - conn = sqlite3.connect(":memory:") - conn.enable_load_extension(True) - conn.load_extension(str(EXT_PATH)) - conn.enable_load_extension(False) - compressed = conn.execute("SELECT zstd_compress(?, 6)", (original,)).fetchone()[0] - self.assertIsInstance(compressed, bytes) - self.assertLess(len(compressed), len(original)) - conn.close() - - def test_transparent_compression_enabled(self): - db_path = os.environ.get("GAMEPLAY_DB_PATH", "data/database/gameplay.sqlite3") - if not Path(db_path).exists(): - self.skipTest(f"Database not found at {db_path}") - - conn = _open(db_path) - tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master").fetchall()} - self.assertIn("_zstd_configs", tables, "_zstd_configs table missing — compression not enabled") - - configs = conn.execute("SELECT config FROM _zstd_configs").fetchall() - compressed_columns = [row["config"] for row in configs] - self.assertEqual(len(compressed_columns), 6) - - expected = ["board_state_json", "snakes_json", "you_json", "food_json", "hazards_json", "body_json"] - for col in expected: - self.assertTrue( - any(col in cfg for cfg in compressed_columns), - f"{col} not found in _zstd_configs", - ) - conn.close() - - def test_turns_and_snake_turns_are_views(self): - db_path = os.environ.get("GAMEPLAY_DB_PATH", "data/database/gameplay.sqlite3") - if not Path(db_path).exists(): - self.skipTest(f"Database not found at {db_path}") - - conn = _open(db_path) - views = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='view'").fetchall()} - self.assertIn("turns", views) - self.assertIn("snake_turns", views) - conn.close() - - def test_board_state_json_is_stored_compressed(self): - db_path = os.environ.get("GAMEPLAY_DB_PATH", "data/database/gameplay.sqlite3") - if not Path(db_path).exists(): - self.skipTest(f"Database not found at {db_path}") - - conn = _open(db_path) - tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()} - if "_turns_zstd" not in tables: - self.skipTest("_turns_zstd table not found") - - row = conn.execute("SELECT board_state_json FROM _turns_zstd LIMIT 1").fetchone() - if row is None: - self.skipTest("No rows in _turns_zstd") - - self.assertIsInstance(row[0], (bytes, bytearray), "board_state_json should be stored as compressed bytes") - conn.close() - -if __name__ == "__main__": - unittest.main()