feat(database): store gameplay replays in compact form
Build and Push Docker Container / build-and-push (push) Successful in 3m52s
Build and Push Docker Container / build-and-push (push) Successful in 3m52s
- Add compact storage package: codec, records, hydrate, replay blobs, cache. - Running games write game_turns and snake_states keyed by integer ids. - Encode moves as 1-byte codes and bodies as head plus step codes. - Store food and hazards only when they changed since the previous turn. - Strip reasoning fields that duplicate stored values and restore on read. - Pack finished games into one zstd blob per game in game_replays. - Merge late turns into packed blobs during the stale-game sweep. - Extract SQLite and PostgreSQL schemas into dedicated modules. - Add read/import game bundle and list_game_ids to the backend template. - Add legacy readers and converter for the old turns/snake_turns layout. - Add migrate_gameplay_to_compact.py with in-place or cross-DB migration. - Verify every migrated game and keep legacy rows on failure. - Support --since, --minimum-quality and guarded --drop-legacy. - Add just recipe migrate-gameplay-postgres. - Rewrite merge script to copy game by game via backend bundles. - Move benchmark state loading into benchmark_states for both backends. - Update PostgreSQL cleanup script to the new replay tables. - Drop sqlite-zstd build stages from the Dockerfile. - Remove old migration scripts, normalized_turn and their tests. - Add tests for compact storage, schema, legacy migration and PostgreSQL. - Document the compact layout and migration workflow in the README.
This commit is contained in:
@@ -0,0 +1,90 @@
|
||||
"""Integration tests against a disposable PostgreSQL database.
|
||||
|
||||
Set GAMEPLAY_TEST_PG_DSN to a database that may be wiped, e.g.
|
||||
postgresql://user:pass@127.0.0.1:5432/battlesnake_test
|
||||
"""
|
||||
import os
|
||||
import unittest
|
||||
|
||||
from server.database.backend.PostgresqlGameplayBackend import PostgresqlGameplayBackend
|
||||
import test_GameplayDatabase as sqlite_tests
|
||||
|
||||
DSN = os.environ.get("GAMEPLAY_TEST_PG_DSN")
|
||||
|
||||
@unittest.skipUnless(DSN, "GAMEPLAY_TEST_PG_DSN not set")
|
||||
class TestPostgresqlGameplayBackend(unittest.IsolatedAsyncioTestCase):
|
||||
_build_state = sqlite_tests.TestGameplayDatabase._build_state
|
||||
_play_game = sqlite_tests.TestGameplayDatabase._play_game
|
||||
|
||||
async def asyncSetUp(self):
|
||||
self.backend = PostgresqlGameplayBackend(DSN)
|
||||
pool = await self.backend._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute("TRUNCATE games CASCADE")
|
||||
|
||||
async def asyncTearDown(self):
|
||||
await self.backend.close()
|
||||
|
||||
async def _count(self, table:str) -> int:
|
||||
pool = await self.backend._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
return int(await conn.fetchval(f"SELECT COUNT(*) FROM {table}"))
|
||||
|
||||
async def test_running_rows_then_packed_replay_is_identical(self):
|
||||
await self._play_game(self.backend)
|
||||
self.assertEqual(await self._count("game_turns"), 2)
|
||||
self.assertEqual(await self._count("snake_states"), 4)
|
||||
running = await self.backend.get_game_replay("game-abc")
|
||||
|
||||
await self.backend.record_game_end(
|
||||
self._build_state(turn=2, me_head=(2, 2), enemy_head=(4, 4), include_enemy=False)
|
||||
)
|
||||
|
||||
self.assertEqual(await self._count("game_turns"), 0)
|
||||
self.assertEqual(await self._count("snake_states"), 0)
|
||||
self.assertEqual(await self._count("game_replays"), 1)
|
||||
finished = await self.backend.get_game_replay("game-abc")
|
||||
self.assertEqual(finished["turns"], running["turns"])
|
||||
self.assertEqual(finished["game"]["status"], "finished")
|
||||
turn = finished["turns"][1]
|
||||
self.assertEqual(turn["my_thinking"], {"turn": 2, "reason": "food", "scores": {"up": 1.4}})
|
||||
self.assertEqual(finished["turns"][0]["my_thinking"]["health"], 90)
|
||||
me = next(snake for snake in turn["snakes"] if snake["snake_id"] == "me")
|
||||
self.assertEqual(me["inferred_move"], "up")
|
||||
self.assertEqual(me["customizations"]["head"], "ferret")
|
||||
|
||||
async def test_stale_game_is_finalized_and_packed(self):
|
||||
await self._play_game(self.backend)
|
||||
pool = await self.backend._get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute("UPDATE games SET started_at = now() - interval '1 day'")
|
||||
|
||||
self.assertEqual(await self.backend.finalize_stale_running_games(600), 1)
|
||||
|
||||
self.assertEqual(await self._count("game_turns"), 0)
|
||||
replay = await self.backend.get_game_replay("game-abc")
|
||||
self.assertEqual(len(replay["turns"]), 2)
|
||||
|
||||
async def test_late_turn_after_end_is_merged_and_repacked(self):
|
||||
await self._play_game(self.backend)
|
||||
await self.backend.record_game_end(self._build_state(turn=2, me_head=(2, 2), enemy_head=(4, 4)))
|
||||
await self.backend.record_turn(self._build_state(turn=3, me_head=(3, 2), enemy_head=(3, 4)), my_move="right")
|
||||
|
||||
self.assertEqual(len((await self.backend.get_game_replay("game-abc"))["turns"]), 3)
|
||||
await self.backend.finalize_stale_running_games(600)
|
||||
self.assertEqual(await self._count("game_turns"), 0)
|
||||
self.assertEqual(len((await self.backend.get_game_replay("game-abc"))["turns"]), 3)
|
||||
|
||||
async def test_bundle_roundtrip(self):
|
||||
await self._play_game(self.backend)
|
||||
await self.backend.record_game_end(self._build_state(turn=2, me_head=(2, 2), enemy_head=(4, 4)))
|
||||
bundle = await self.backend.read_game_bundle("game-abc")
|
||||
bundle.game["game_id"] = "copy"
|
||||
await self.backend.import_game_bundle(bundle)
|
||||
|
||||
copy = await self.backend.read_game_bundle("copy")
|
||||
self.assertEqual(copy.turns, bundle.turns)
|
||||
self.assertEqual(sorted(copy.states, key=lambda s: (s.turn, s.snake_id)), sorted(bundle.states, key=lambda s: (s.turn, s.snake_id)))
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user