feat(database): store gameplay replays in compact form
Build and Push Docker Container / build-and-push (push) Successful in 3m52s

- Add compact storage package: codec, records, hydrate, replay blobs, cache.
- Running games write game_turns and snake_states keyed by integer ids.
- Encode moves as 1-byte codes and bodies as head plus step codes.
- Store food and hazards only when they changed since the previous turn.
- Strip reasoning fields that duplicate stored values and restore on read.
- Pack finished games into one zstd blob per game in game_replays.
- Merge late turns into packed blobs during the stale-game sweep.
- Extract SQLite and PostgreSQL schemas into dedicated modules.
- Add read/import game bundle and list_game_ids to the backend template.
- Add legacy readers and converter for the old turns/snake_turns layout.
- Add migrate_gameplay_to_compact.py with in-place or cross-DB migration.
- Verify every migrated game and keep legacy rows on failure.
- Support --since, --minimum-quality and guarded --drop-legacy.
- Add just recipe migrate-gameplay-postgres.
- Rewrite merge script to copy game by game via backend bundles.
- Move benchmark state loading into benchmark_states for both backends.
- Update PostgreSQL cleanup script to the new replay tables.
- Drop sqlite-zstd build stages from the Dockerfile.
- Remove old migration scripts, normalized_turn and their tests.
- Add tests for compact storage, schema, legacy migration and PostgreSQL.
- Document the compact layout and migration workflow in the README.
This commit is contained in:
2026-11-05 00:02:56 +01:00
parent 4403978537
commit 39a7a53873
41 changed files with 2779 additions and 2493 deletions
+2 -50
View File
@@ -1,17 +1,14 @@
#!/usr/bin/env python3
"""Benchmark snake move latency against sampled states from gameplay SQLite."""
"""Benchmark snake move latency against sampled states from a gameplay database."""
import argparse
from pathlib import Path
import sqlite3
from statistics import mean, median
import sys
from time import perf_counter
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from server.database.benchmark_states import (
_build_state, is_postgresql_source, load_postgresql_states,
)
from server.database.benchmark_states import load_states
lazy from server.GameBoard import GameBoard
lazy from snakes import SnakeBuilder
@@ -20,51 +17,6 @@ def percentile(values: list[float], quantile: float) -> float:
index = min(len(ordered) - 1, round((len(ordered) - 1) * quantile))
return ordered[index]
def load_states(db_path: str, samples: int, stride: int) -> list[tuple[dict, dict]]:
if is_postgresql_source(db_path):
return load_postgresql_states(db_path, samples, stride)
connection = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
connection.execute("PRAGMA query_only = ON")
max_id = int(connection.execute("SELECT max(id) FROM turns").fetchone()[0] or 0)
if max_id == 0:
return []
states: list[tuple[dict, dict]] = []
next_id = max(1, max_id - (samples - 1) * stride)
query = """
SELECT t.id, t.board_state_json, t.you_json, t.food_json, t.hazards_json,
g.your_snake_id, g.your_snake_name, g.width, g.height,
g.game_id, g.source, g.map_name,
g.ruleset_name, g.ruleset_version, t.turn
FROM turns AS t
JOIN games AS g ON g.game_id = t.game_id
WHERE t.id >= ?
ORDER BY t.id
LIMIT 1
"""
snake_query = """
SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name),
st.health, st.length, st.head_x, st.head_y, st.body_json,
COALESCE(gs.customizations_json, '{}')
FROM snake_turns AS st
LEFT JOIN game_snakes AS gs
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
WHERE st.game_id = ? AND st.turn = ?
ORDER BY st.id
"""
while len(states) < samples and next_id <= max_id:
row = connection.execute(query, (next_id,)).fetchone()
if row is None:
break
snake_rows = connection.execute(snake_query, (row[9], row[14])).fetchall()
state = _build_state(row, snake_rows)
if state is not None:
states.append(state)
next_id = int(row[0]) + stride
connection.close()
return states
def benchmark(snake_name: str, states: list[tuple[dict, dict]], repeat: int) -> dict:
durations: list[float] = []
moves = 0