Files
snake-python/tests/test_GameplaySchema.py
daniel156161 39a7a53873
Build and Push Docker Container / build-and-push (push) Successful in 3m52s
feat(database): store gameplay replays in compact form
- Add compact storage package: codec, records, hydrate, replay blobs, cache.
- Running games write game_turns and snake_states keyed by integer ids.
- Encode moves as 1-byte codes and bodies as head plus step codes.
- Store food and hazards only when they changed since the previous turn.
- Strip reasoning fields that duplicate stored values and restore on read.
- Pack finished games into one zstd blob per game in game_replays.
- Merge late turns into packed blobs during the stale-game sweep.
- Extract SQLite and PostgreSQL schemas into dedicated modules.
- Add read/import game bundle and list_game_ids to the backend template.
- Add legacy readers and converter for the old turns/snake_turns layout.
- Add migrate_gameplay_to_compact.py with in-place or cross-DB migration.
- Verify every migrated game and keep legacy rows on failure.
- Support --since, --minimum-quality and guarded --drop-legacy.
- Add just recipe migrate-gameplay-postgres.
- Rewrite merge script to copy game by game via backend bundles.
- Move benchmark state loading into benchmark_states for both backends.
- Update PostgreSQL cleanup script to the new replay tables.
- Drop sqlite-zstd build stages from the Dockerfile.
- Remove old migration scripts, normalized_turn and their tests.
- Add tests for compact storage, schema, legacy migration and PostgreSQL.
- Document the compact layout and migration workflow in the README.
2026-11-05 00:02:56 +01:00

121 lines
5.2 KiB
Python

import sqlite3
import tempfile
import unittest
from contextlib import closing
from pathlib import Path
from server.database.backend import postgresql_schema
from server.database.backend.PostgresqlGameplayBackend import PostgresqlGameplayBackend
from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend
REDUNDANT = {"idx_turns_game_turn", "idx_snake_turns_game_turn"}
class _RecordingConnection:
def __init__(self) -> None:
self.statements:list[str] = []
async def execute(self, statement:str, *args) -> str:
self.statements.append(statement)
return "OK"
class _AcquireContext:
def __init__(self, connection) -> None:
self._connection = connection
async def __aenter__(self):
return self._connection
async def __aexit__(self, *exc) -> None:
return None
class _Pool:
def __init__(self, connection) -> None:
self._connection = connection
def acquire(self):
return _AcquireContext(self._connection)
class TestPostgresqlSchema(unittest.IsolatedAsyncioTestCase):
def test_ddl_defines_compact_tables_without_legacy_indexes(self):
ddl = postgresql_schema.TABLES_DDL
for table in ("game_turns", "snake_states", "game_replays"):
self.assertIn(f"CREATE TABLE IF NOT EXISTS {table}", ddl)
for name in REDUNDANT:
self.assertNotIn(f"CREATE INDEX IF NOT EXISTS {name}", ddl)
self.assertIn("SET STORAGE EXTERNAL", ddl)
async def test_schema_init_is_locked_and_drops_redundant_indexes_separately(self):
connection = _RecordingConnection()
backend = PostgresqlGameplayBackend("postgresql://example")
backend._pool = _Pool(connection)
await backend._initialize_schema()
self.assertIn("pg_advisory_lock", connection.statements[0])
drops = [s for s in connection.statements if "DROP INDEX" in s]
self.assertEqual({s.split()[-1] for s in drops}, REDUNDANT)
for statement in drops:
# CONCURRENTLY only works as a standalone statement outside a transaction.
self.assertIn("CONCURRENTLY IF EXISTS", statement)
self.assertNotIn(";", statement)
class TestSqliteSchema(unittest.TestCase):
def _objects(self, db_path:Path, kind:str) -> set[str]:
with closing(sqlite3.connect(db_path)) as connection:
return {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type = ?", (kind,))}
def test_new_database_has_compact_tables(self):
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "gameplay.sqlite3"
SqliteGameplayBackend(str(db_path))
tables = self._objects(db_path, "table")
self.assertTrue({"games", "game_snakes", "game_turns", "snake_states", "game_replays"} <= tables)
self.assertFalse({"turns", "snake_turns"} & tables)
self.assertFalse(self._objects(db_path, "index") & REDUNDANT)
def test_legacy_database_gets_ids_and_keeps_legacy_tables(self):
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "gameplay.sqlite3"
with closing(sqlite3.connect(db_path)) as connection, connection:
connection.executescript("""
CREATE TABLE games (game_id TEXT PRIMARY KEY, started_at TEXT NOT NULL,
winner_you INTEGER NOT NULL DEFAULT 0, final_turn INTEGER NOT NULL DEFAULT 0,
status TEXT NOT NULL DEFAULT 'running', width INTEGER, height INTEGER,
source TEXT, map_name TEXT, ruleset_name TEXT, ruleset_version TEXT,
your_snake_id TEXT, your_snake_name TEXT);
CREATE TABLE game_snakes (game_id TEXT NOT NULL, snake_id TEXT NOT NULL,
snake_name TEXT, is_you INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (game_id, snake_id));
CREATE TABLE turns (game_id TEXT, turn INTEGER);
CREATE TABLE snake_turns (game_id TEXT, turn INTEGER, snake_id TEXT);
CREATE INDEX idx_turns_game_turn ON turns(game_id, turn);
INSERT INTO games (game_id, started_at) VALUES ('a', '2026-01-01'), ('b', '2026-01-02');
INSERT INTO game_snakes (game_id, snake_id) VALUES ('a', 's1'), ('a', 's2');
""")
SqliteGameplayBackend(str(db_path))
with closing(sqlite3.connect(db_path)) as connection:
gids = [row[0] for row in connection.execute("SELECT gid FROM games ORDER BY game_id")]
gsids = [row[0] for row in connection.execute("SELECT gsid FROM game_snakes ORDER BY snake_id")]
self.assertEqual(len(set(gids)), 2)
self.assertNotIn(None, gids)
self.assertEqual(len(set(gsids)), 2)
self.assertNotIn(None, gsids)
self.assertTrue({"turns", "snake_turns"} <= self._objects(db_path, "table"))
self.assertFalse(self._objects(db_path, "index") & REDUNDANT)
def test_running_game_lookups_use_primary_keys(self):
with tempfile.TemporaryDirectory() as temp_dir:
db_path = Path(temp_dir) / "gameplay.sqlite3"
SqliteGameplayBackend(str(db_path))
with closing(sqlite3.connect(db_path)) as connection:
for query in (
"SELECT * FROM game_turns WHERE gid = 1 ORDER BY turn",
"SELECT * FROM snake_states WHERE gsid = 1 AND turn = 3",
):
plan = " ".join(str(row[-1]) for row in connection.execute(f"EXPLAIN QUERY PLAN {query}"))
self.assertIn("PRIMARY KEY", plan, plan)
if __name__ == "__main__":
unittest.main()