feat(database): store gameplay replays in compact form
Build and Push Docker Container / build-and-push (push) Successful in 3m52s
Build and Push Docker Container / build-and-push (push) Successful in 3m52s
- Add compact storage package: codec, records, hydrate, replay blobs, cache. - Running games write game_turns and snake_states keyed by integer ids. - Encode moves as 1-byte codes and bodies as head plus step codes. - Store food and hazards only when they changed since the previous turn. - Strip reasoning fields that duplicate stored values and restore on read. - Pack finished games into one zstd blob per game in game_replays. - Merge late turns into packed blobs during the stale-game sweep. - Extract SQLite and PostgreSQL schemas into dedicated modules. - Add read/import game bundle and list_game_ids to the backend template. - Add legacy readers and converter for the old turns/snake_turns layout. - Add migrate_gameplay_to_compact.py with in-place or cross-DB migration. - Verify every migrated game and keep legacy rows on failure. - Support --since, --minimum-quality and guarded --drop-legacy. - Add just recipe migrate-gameplay-postgres. - Rewrite merge script to copy game by game via backend bundles. - Move benchmark state loading into benchmark_states for both backends. - Update PostgreSQL cleanup script to the new replay tables. - Drop sqlite-zstd build stages from the Dockerfile. - Remove old migration scripts, normalized_turn and their tests. - Add tests for compact storage, schema, legacy migration and PostgreSQL. - Document the compact layout and migration workflow in the README.
This commit is contained in:
@@ -1,17 +1,14 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Benchmark snake move latency against sampled states from gameplay SQLite."""
|
||||
"""Benchmark snake move latency against sampled states from a gameplay database."""
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
from statistics import mean, median
|
||||
import sys
|
||||
from time import perf_counter
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from server.database.benchmark_states import (
|
||||
_build_state, is_postgresql_source, load_postgresql_states,
|
||||
)
|
||||
from server.database.benchmark_states import load_states
|
||||
lazy from server.GameBoard import GameBoard
|
||||
lazy from snakes import SnakeBuilder
|
||||
|
||||
@@ -20,51 +17,6 @@ def percentile(values: list[float], quantile: float) -> float:
|
||||
index = min(len(ordered) - 1, round((len(ordered) - 1) * quantile))
|
||||
return ordered[index]
|
||||
|
||||
def load_states(db_path: str, samples: int, stride: int) -> list[tuple[dict, dict]]:
|
||||
if is_postgresql_source(db_path):
|
||||
return load_postgresql_states(db_path, samples, stride)
|
||||
|
||||
connection = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
|
||||
connection.execute("PRAGMA query_only = ON")
|
||||
max_id = int(connection.execute("SELECT max(id) FROM turns").fetchone()[0] or 0)
|
||||
if max_id == 0:
|
||||
return []
|
||||
|
||||
states: list[tuple[dict, dict]] = []
|
||||
next_id = max(1, max_id - (samples - 1) * stride)
|
||||
query = """
|
||||
SELECT t.id, t.board_state_json, t.you_json, t.food_json, t.hazards_json,
|
||||
g.your_snake_id, g.your_snake_name, g.width, g.height,
|
||||
g.game_id, g.source, g.map_name,
|
||||
g.ruleset_name, g.ruleset_version, t.turn
|
||||
FROM turns AS t
|
||||
JOIN games AS g ON g.game_id = t.game_id
|
||||
WHERE t.id >= ?
|
||||
ORDER BY t.id
|
||||
LIMIT 1
|
||||
"""
|
||||
snake_query = """
|
||||
SELECT st.snake_id, COALESCE(gs.snake_name, st.snake_name),
|
||||
st.health, st.length, st.head_x, st.head_y, st.body_json,
|
||||
COALESCE(gs.customizations_json, '{}')
|
||||
FROM snake_turns AS st
|
||||
LEFT JOIN game_snakes AS gs
|
||||
ON gs.game_id = st.game_id AND gs.snake_id = st.snake_id
|
||||
WHERE st.game_id = ? AND st.turn = ?
|
||||
ORDER BY st.id
|
||||
"""
|
||||
while len(states) < samples and next_id <= max_id:
|
||||
row = connection.execute(query, (next_id,)).fetchone()
|
||||
if row is None:
|
||||
break
|
||||
snake_rows = connection.execute(snake_query, (row[9], row[14])).fetchall()
|
||||
state = _build_state(row, snake_rows)
|
||||
if state is not None:
|
||||
states.append(state)
|
||||
next_id = int(row[0]) + stride
|
||||
connection.close()
|
||||
return states
|
||||
|
||||
def benchmark(snake_name: str, states: list[tuple[dict, dict]], repeat: int) -> dict:
|
||||
durations: list[float] = []
|
||||
moves = 0
|
||||
|
||||
@@ -21,20 +21,20 @@ async def cleanup(dsn: str, older_than_days: int, dry_run: bool, vacuum: bool) -
|
||||
AND COALESCE(ended_at, started_at) < $1
|
||||
""", cutoff))
|
||||
rows = await connection.fetchrow("""
|
||||
WITH candidates AS (
|
||||
SELECT game_id, gid FROM games
|
||||
WHERE has_replay
|
||||
AND quality_status = 'low_quality'
|
||||
AND COALESCE(ended_at, started_at) < $1
|
||||
)
|
||||
SELECT
|
||||
(SELECT count(*) FROM turns t JOIN games g USING (game_id)
|
||||
WHERE g.has_replay AND g.quality_status = 'low_quality'
|
||||
AND COALESCE(g.ended_at, g.started_at) < $1) AS turns,
|
||||
(SELECT count(*) FROM snake_turns t JOIN games g USING (game_id)
|
||||
WHERE g.has_replay AND g.quality_status = 'low_quality'
|
||||
AND COALESCE(g.ended_at, g.started_at) < $1) AS snake_turns,
|
||||
(SELECT count(*) FROM game_snakes t JOIN games g USING (game_id)
|
||||
WHERE g.has_replay AND g.quality_status = 'low_quality'
|
||||
AND COALESCE(g.ended_at, g.started_at) < $1) AS game_snakes
|
||||
(SELECT count(*) FROM game_replays r JOIN candidates c USING (gid)) AS replays,
|
||||
(SELECT count(*) FROM game_turns t JOIN candidates c USING (gid)) AS turns,
|
||||
(SELECT count(*) FROM game_snakes s JOIN candidates c USING (game_id)) AS game_snakes
|
||||
""", cutoff)
|
||||
print(
|
||||
f"candidates before {cutoff.isoformat()}: games={candidates:,}, "
|
||||
f"turns={rows['turns']:,}, snake_turns={rows['snake_turns']:,}, "
|
||||
f"replays={rows['replays']:,}, running turns={rows['turns']:,}, "
|
||||
f"game_snakes={rows['game_snakes']:,}"
|
||||
)
|
||||
if dry_run or candidates == 0:
|
||||
@@ -50,8 +50,13 @@ async def cleanup(dsn: str, older_than_days: int, dry_run: bool, vacuum: bool) -
|
||||
FOR UPDATE
|
||||
""", cutoff)
|
||||
ids = [row["game_id"] for row in game_ids]
|
||||
await connection.execute("DELETE FROM snake_turns WHERE game_id = ANY($1::text[])", ids)
|
||||
await connection.execute("DELETE FROM turns WHERE game_id = ANY($1::text[])", ids)
|
||||
# snake_states rows cascade from game_snakes.
|
||||
await connection.execute("""
|
||||
DELETE FROM game_replays WHERE gid IN (SELECT gid FROM games WHERE game_id = ANY($1::text[]))
|
||||
""", ids)
|
||||
await connection.execute("""
|
||||
DELETE FROM game_turns WHERE gid IN (SELECT gid FROM games WHERE game_id = ANY($1::text[]))
|
||||
""", ids)
|
||||
await connection.execute("DELETE FROM game_snakes WHERE game_id = ANY($1::text[])", ids)
|
||||
await connection.execute("""
|
||||
UPDATE games
|
||||
@@ -62,8 +67,9 @@ async def cleanup(dsn: str, older_than_days: int, dry_run: bool, vacuum: bool) -
|
||||
if vacuum:
|
||||
await connection.execute("VACUUM (ANALYZE) games")
|
||||
await connection.execute("VACUUM (ANALYZE) game_snakes")
|
||||
await connection.execute("VACUUM (ANALYZE) turns")
|
||||
await connection.execute("VACUUM (ANALYZE) snake_turns")
|
||||
await connection.execute("VACUUM (ANALYZE) game_replays")
|
||||
await connection.execute("VACUUM (ANALYZE) game_turns")
|
||||
await connection.execute("VACUUM (ANALYZE) snake_states")
|
||||
print("vacuum/analyze complete")
|
||||
finally:
|
||||
await connection.close()
|
||||
|
||||
@@ -1,40 +1,23 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Safely merge a cleaned gameplay database with a temporary delta database.
|
||||
"""Safely merge a gameplay SQLite database with a temporary delta database.
|
||||
|
||||
The base and delta are opened read-only. A new destination is created, numeric
|
||||
row IDs are regenerated, delta games are quality-rated, and all result rows are
|
||||
kept even when their replay is excluded. The base can be replaced only after
|
||||
verification and a timestamped backup.
|
||||
Base and delta are opened read-only and copied game by game into a new
|
||||
destination, which assigns fresh integer ids. Delta games are quality-rated;
|
||||
all game results are kept even when their replay is excluded. The base can be
|
||||
replaced only after verification and a timestamped backup.
|
||||
"""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import sys
|
||||
from time import perf_counter
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from scripts.migrate_gameplay_database import (
|
||||
analyze_quality, column_or_null, create_destination, object_columns,
|
||||
open_source, retained_game_ids,
|
||||
)
|
||||
from server.database.game_quality import quality_meets_minimum
|
||||
from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend
|
||||
from server.database.legacy.migrate import apply_quality
|
||||
|
||||
GAME_COLUMNS = (
|
||||
"game_id", "started_at", "ended_at", "width", "height", "source",
|
||||
"map_name", "ruleset_name", "ruleset_version", "your_snake_id",
|
||||
"your_snake_name", "your_snake_type", "your_snake_version", "game_type",
|
||||
"winner_name", "winner_you", "final_turn", "status", "has_replay",
|
||||
"quality_status", "quality_score", "quality_tier", "quality_reasons_json",
|
||||
)
|
||||
GAME_DEFAULTS = {
|
||||
"winner_you": "0 AS winner_you", "final_turn": "0 AS final_turn",
|
||||
"status": "'running' AS status", "has_replay": "1 AS has_replay",
|
||||
"quality_status": "'retained' AS quality_status",
|
||||
}
|
||||
CORE_GAME_FIELDS = (
|
||||
"game_id", "width", "height", "source", "map_name", "ruleset_name",
|
||||
"ruleset_version", "your_snake_id", "your_snake_name", "your_snake_type",
|
||||
@@ -42,26 +25,13 @@ CORE_GAME_FIELDS = (
|
||||
"final_turn", "status",
|
||||
)
|
||||
|
||||
def select_expression(columns:set[str], name:str) -> str:
|
||||
if name in columns:
|
||||
return name
|
||||
return GAME_DEFAULTS.get(name, f"NULL AS {name}")
|
||||
|
||||
def read_games(connection:sqlite3.Connection) -> dict[str, sqlite3.Row]:
|
||||
columns = object_columns(connection, "games")
|
||||
selected = ", ".join(select_expression(columns, name) for name in GAME_COLUMNS)
|
||||
return {
|
||||
row["game_id"]: row
|
||||
for row in connection.execute(f"SELECT {selected} FROM games")
|
||||
}
|
||||
|
||||
def duplicate_ids(base_games:dict[str, sqlite3.Row], delta_games:dict[str, sqlite3.Row]) -> set[str]:
|
||||
duplicates = set(base_games) & set(delta_games)
|
||||
def duplicate_ids(base:SqliteGameplayBackend, delta:SqliteGameplayBackend, base_ids:list[str], delta_ids:list[str]) -> set[str]:
|
||||
duplicates = set(base_ids) & set(delta_ids)
|
||||
conflicts = []
|
||||
for game_id in duplicates:
|
||||
base_signature = tuple(base_games[game_id][name] for name in CORE_GAME_FIELDS)
|
||||
delta_signature = tuple(delta_games[game_id][name] for name in CORE_GAME_FIELDS)
|
||||
if base_signature != delta_signature:
|
||||
base_game = base.read_game_bundle_sync(game_id).game
|
||||
delta_game = delta.read_game_bundle_sync(game_id).game
|
||||
if any(base_game[name] != delta_game[name] for name in CORE_GAME_FIELDS):
|
||||
conflicts.append(game_id)
|
||||
if conflicts:
|
||||
examples = ", ".join(sorted(conflicts)[:5])
|
||||
@@ -71,131 +41,32 @@ def duplicate_ids(base_games:dict[str, sqlite3.Row], delta_games:dict[str, sqlit
|
||||
)
|
||||
return duplicates
|
||||
|
||||
def insert_games(
|
||||
destination:sqlite3.Connection,
|
||||
rows:dict[str, sqlite3.Row],
|
||||
excluded:set[str],
|
||||
quality:dict[str, object]|None=None,
|
||||
minimum_quality:str="medium",
|
||||
) -> tuple[int, set[str], Counter]:
|
||||
placeholders = ",".join("?" for _ in GAME_COLUMNS)
|
||||
sql = f"INSERT INTO games ({','.join(GAME_COLUMNS)}) VALUES ({placeholders})"
|
||||
values = []
|
||||
replay_ids:set[str] = set()
|
||||
def copy_games(source:SqliteGameplayBackend, destination:SqliteGameplayBackend, game_ids:list[str], minimum_quality:str|None=None) -> tuple[Counter, Counter]:
|
||||
counts:Counter = Counter()
|
||||
tiers:Counter = Counter()
|
||||
for game_id in game_ids:
|
||||
bundle = source.read_game_bundle_sync(game_id)
|
||||
if minimum_quality is not None:
|
||||
apply_quality(bundle, minimum_quality)
|
||||
tiers[bundle.game["quality_tier"]] += 1
|
||||
destination.import_game_bundle_sync(bundle)
|
||||
counts["games"] += 1
|
||||
counts["game_snakes"] += len(bundle.snakes)
|
||||
counts["turns"] += len(bundle.turns)
|
||||
counts["snake_states"] += len(bundle.states)
|
||||
return counts, tiers
|
||||
|
||||
for game_id, row in rows.items():
|
||||
if game_id in excluded:
|
||||
continue
|
||||
output = [row[name] for name in GAME_COLUMNS]
|
||||
if quality is not None:
|
||||
result = quality[game_id]
|
||||
keep_replay = quality_meets_minimum(result.tier, minimum_quality)
|
||||
replacements = {
|
||||
"has_replay": int(keep_replay),
|
||||
"quality_status": "retained" if keep_replay else "low_quality",
|
||||
"quality_score": result.score,
|
||||
"quality_tier": result.tier,
|
||||
"quality_reasons_json": json.dumps(result.reasons, separators=(",", ":")),
|
||||
}
|
||||
output = [replacements.get(name, row[name]) for name in GAME_COLUMNS]
|
||||
tiers[result.tier] += 1
|
||||
if keep_replay:
|
||||
replay_ids.add(game_id)
|
||||
elif bool(row["has_replay"]):
|
||||
replay_ids.add(game_id)
|
||||
values.append(tuple(output))
|
||||
|
||||
destination.executemany(sql, values)
|
||||
return len(values), replay_ids, tiers
|
||||
|
||||
def copy_game_snakes(source:sqlite3.Connection, destination:sqlite3.Connection, allowed:set[str], batch_size:int) -> int:
|
||||
has_table = source.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='game_snakes'"
|
||||
).fetchone()
|
||||
if has_table:
|
||||
columns = object_columns(source, "game_snakes")
|
||||
customizations = (
|
||||
"COALESCE(customizations_json, '{}') AS customizations_json"
|
||||
if "customizations_json" in columns else "'{}' AS customizations_json"
|
||||
)
|
||||
cursor = source.execute(f"""
|
||||
SELECT game_id, snake_id, snake_name, is_you, {customizations}
|
||||
FROM game_snakes ORDER BY game_id, snake_id
|
||||
""")
|
||||
else:
|
||||
cursor = source.execute("""
|
||||
SELECT game_id, snake_id, MAX(snake_name), MAX(is_you),
|
||||
'{}' AS customizations_json
|
||||
FROM snake_turns GROUP BY game_id, snake_id ORDER BY game_id, snake_id
|
||||
""")
|
||||
sql = """
|
||||
INSERT INTO game_snakes (
|
||||
game_id,snake_id,snake_name,is_you,customizations_json
|
||||
) VALUES (?,?,?,?,?)
|
||||
"""
|
||||
count = 0
|
||||
while rows := cursor.fetchmany(batch_size):
|
||||
values = [tuple(row) for row in rows if row[0] in allowed]
|
||||
destination.executemany(sql, values)
|
||||
count += len(values)
|
||||
return count
|
||||
|
||||
def copy_turns(source:sqlite3.Connection, destination:sqlite3.Connection, allowed:set[str], batch_size:int) -> int:
|
||||
columns = object_columns(source, "turns")
|
||||
names = (
|
||||
"game_id", "turn", "observed_at", "my_move", "my_thinking_json",
|
||||
"board_state_json", "snakes_json", "you_json", "food_json", "hazards_json",
|
||||
)
|
||||
defaults = {
|
||||
"my_thinking_json": "NULL AS my_thinking_json", "board_state_json": "'{}' AS board_state_json",
|
||||
"snakes_json": "'[]' AS snakes_json", "you_json": "'{}' AS you_json",
|
||||
"food_json": "'[]' AS food_json", "hazards_json": "'[]' AS hazards_json",
|
||||
}
|
||||
selected = ",".join(name if name in columns else defaults[name] for name in names)
|
||||
cursor = source.execute(f"SELECT {selected} FROM turns ORDER BY id")
|
||||
sql = f"INSERT INTO turns ({','.join(names)}) VALUES ({','.join('?' for _ in names)})"
|
||||
count = 0
|
||||
while rows := cursor.fetchmany(batch_size):
|
||||
values = [tuple(row) for row in rows if row[0] in allowed]
|
||||
destination.executemany(sql, values)
|
||||
count += len(values)
|
||||
return count
|
||||
|
||||
def copy_snake_turns(source:sqlite3.Connection, destination:sqlite3.Connection, allowed:set[str], batch_size:int) -> int:
|
||||
columns = object_columns(source, "snake_turns")
|
||||
names = (
|
||||
"game_id", "turn", "snake_id", "snake_name", "health", "length",
|
||||
"head_x", "head_y", "body_json", "is_you", "inferred_move", "latency",
|
||||
)
|
||||
selected = ",".join(column_or_null(columns, name) for name in names)
|
||||
cursor = source.execute(f"SELECT {selected} FROM snake_turns ORDER BY id")
|
||||
sql = f"INSERT INTO snake_turns ({','.join(names)}) VALUES ({','.join('?' for _ in names)})"
|
||||
count = 0
|
||||
while rows := cursor.fetchmany(batch_size):
|
||||
values = [tuple(row) for row in rows if row[0] in allowed]
|
||||
destination.executemany(sql, values)
|
||||
count += len(values)
|
||||
return count
|
||||
|
||||
def copy_replays(source, destination, allowed:set[str], batch_size:int) -> dict[str, int]:
|
||||
return {
|
||||
"game_snakes": copy_game_snakes(source, destination, allowed, batch_size),
|
||||
"turns": copy_turns(source, destination, allowed, batch_size),
|
||||
"snake_turns": copy_snake_turns(source, destination, allowed, batch_size),
|
||||
}
|
||||
|
||||
def verify(destination:sqlite3.Connection, expected:dict[str, int]) -> None:
|
||||
for table, count in expected.items():
|
||||
actual = int(destination.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0])
|
||||
if actual != count:
|
||||
raise RuntimeError(f"{table} count mismatch: expected {count}, got {actual}")
|
||||
foreign_keys = destination.execute("PRAGMA foreign_key_check").fetchall()
|
||||
if foreign_keys:
|
||||
raise RuntimeError(f"Foreign-key verification failed with {len(foreign_keys)} errors")
|
||||
integrity = destination.execute("PRAGMA integrity_check").fetchone()[0]
|
||||
if integrity != "ok":
|
||||
raise RuntimeError(f"Integrity check failed: {integrity}")
|
||||
def verify(destination:SqliteGameplayBackend, expected:Counter) -> None:
|
||||
actual:Counter = Counter()
|
||||
for game_id in destination.list_game_ids_sync():
|
||||
bundle = destination.read_game_bundle_sync(game_id)
|
||||
actual["games"] += 1
|
||||
actual["game_snakes"] += len(bundle.snakes)
|
||||
actual["turns"] += len(bundle.turns)
|
||||
actual["snake_states"] += len(bundle.states)
|
||||
for key, count in expected.items():
|
||||
if actual[key] != count:
|
||||
raise RuntimeError(f"{key} count mismatch: expected {count}, got {actual[key]}")
|
||||
|
||||
def replace_base(base_path:Path, destination_path:Path) -> Path:
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||
@@ -214,8 +85,6 @@ def main() -> None:
|
||||
parser.add_argument("--delta", required=True, type=Path, help="Database written during cleanup")
|
||||
parser.add_argument("--destination", required=True, type=Path)
|
||||
parser.add_argument("--minimum-quality", choices=("low", "medium", "high"), default="medium")
|
||||
parser.add_argument("--batch-size", type=int, default=10_000)
|
||||
parser.add_argument("--busy-timeout-ms", type=int, default=60_000)
|
||||
parser.add_argument("--replace-base", action="store_true", help="Replace base after verification and keep a timestamped backup")
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -224,51 +93,32 @@ def main() -> None:
|
||||
destination_path = args.destination.expanduser().resolve()
|
||||
if len({base_path, delta_path, destination_path}) != 3:
|
||||
raise SystemExit("Base, delta, and destination must be different paths")
|
||||
if destination_path.exists():
|
||||
raise SystemExit(f"Destination already exists: {destination_path}")
|
||||
|
||||
started = perf_counter()
|
||||
base = open_source(base_path)
|
||||
delta = open_source(delta_path)
|
||||
destination = create_destination(destination_path, max(1000, args.busy_timeout_ms))
|
||||
try:
|
||||
base_games = read_games(base)
|
||||
delta_games = read_games(delta)
|
||||
duplicates = duplicate_ids(base_games, delta_games)
|
||||
delta_quality = analyze_quality(delta)
|
||||
base = SqliteGameplayBackend(str(base_path), read_only=True)
|
||||
delta = SqliteGameplayBackend(str(delta_path), read_only=True)
|
||||
base_ids = base.list_game_ids_sync()
|
||||
delta_ids = delta.list_game_ids_sync()
|
||||
duplicates = duplicate_ids(base, delta, base_ids, delta_ids)
|
||||
|
||||
base_count, base_replays, _ = insert_games(destination, base_games, set())
|
||||
delta_count, delta_replays, tiers = insert_games(
|
||||
destination, delta_games, duplicates, delta_quality, args.minimum_quality,
|
||||
destination = SqliteGameplayBackend(str(destination_path), journal_mode="DELETE")
|
||||
try:
|
||||
base_counts, _ = copy_games(base, destination, base_ids)
|
||||
delta_counts, tiers = copy_games(
|
||||
delta, destination, [game_id for game_id in delta_ids if game_id not in duplicates],
|
||||
args.minimum_quality,
|
||||
)
|
||||
base_rows = copy_replays(base, destination, base_replays, max(1, args.batch_size))
|
||||
delta_rows = copy_replays(delta, destination, delta_replays - duplicates, max(1, args.batch_size))
|
||||
destination.commit()
|
||||
destination.executescript("""
|
||||
CREATE INDEX IF NOT EXISTS idx_turns_game_turn ON turns(game_id, turn);
|
||||
CREATE INDEX IF NOT EXISTS idx_games_status ON games(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_snake_turns_game_turn ON snake_turns(game_id, turn);
|
||||
""")
|
||||
destination.execute("PRAGMA foreign_keys = ON")
|
||||
expected = {
|
||||
"games": base_count + delta_count,
|
||||
"game_snakes": base_rows["game_snakes"] + delta_rows["game_snakes"],
|
||||
"turns": base_rows["turns"] + delta_rows["turns"],
|
||||
"snake_turns": base_rows["snake_turns"] + delta_rows["snake_turns"],
|
||||
}
|
||||
verify(destination, expected)
|
||||
verify(destination, base_counts + delta_counts)
|
||||
except Exception:
|
||||
destination.close()
|
||||
base.close()
|
||||
delta.close()
|
||||
for suffix in ("", "-wal", "-shm"):
|
||||
Path(f"{destination_path}{suffix}").unlink(missing_ok=True)
|
||||
raise
|
||||
destination.close()
|
||||
base.close()
|
||||
delta.close()
|
||||
|
||||
print(f"merged: base games={base_count:,}, delta games={delta_count:,}, identical duplicates skipped={len(duplicates):,}")
|
||||
print(f"merged: base games={base_counts['games']:,}, delta games={delta_counts['games']:,}, identical duplicates skipped={len(duplicates):,}")
|
||||
print(f"delta quality: {dict(tiers)}")
|
||||
print(f"verified: {expected}; elapsed: {perf_counter() - started:.1f}s")
|
||||
print(f"verified: {dict(base_counts + delta_counts)}; elapsed: {perf_counter() - started:.1f}s")
|
||||
if args.replace_base:
|
||||
backup = replace_base(base_path, destination_path)
|
||||
print(f"base replaced; backup kept at: {backup}")
|
||||
|
||||
@@ -1,355 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create a compact, normalized copy of a gameplay SQLite database.
|
||||
|
||||
The source is always opened read-only. The destination is written separately,
|
||||
verified, and can optionally replace the source after a timestamped backup is
|
||||
created. No in-place schema rewrite is performed.
|
||||
"""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import sys
|
||||
from time import perf_counter
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend
|
||||
from server.database.game_quality import GameQualityInput, quality_meets_minimum, rate_game_quality
|
||||
|
||||
def open_source(path:Path) -> sqlite3.Connection:
|
||||
connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=60)
|
||||
connection.row_factory = sqlite3.Row
|
||||
connection.execute("PRAGMA query_only = ON")
|
||||
return connection
|
||||
|
||||
def object_columns(connection:sqlite3.Connection, name:str) -> set[str]:
|
||||
return {row[1] for row in connection.execute(f"PRAGMA table_info({name})")}
|
||||
|
||||
def column_or_null(columns:set[str], name:str) -> str:
|
||||
return name if name in columns else f"NULL AS {name}"
|
||||
|
||||
def create_destination(path:Path, busy_timeout_ms:int) -> sqlite3.Connection:
|
||||
if path.exists():
|
||||
raise FileExistsError(f"Destination already exists: {path}")
|
||||
SqliteGameplayBackend(
|
||||
str(path), busy_timeout_ms=busy_timeout_ms,
|
||||
initialize_indexes=False, journal_mode="DELETE",
|
||||
)
|
||||
connection = sqlite3.connect(str(path), timeout=max(1, busy_timeout_ms // 1000))
|
||||
connection.execute("PRAGMA foreign_keys = OFF")
|
||||
connection.execute("PRAGMA synchronous = OFF")
|
||||
connection.execute("PRAGMA locking_mode = EXCLUSIVE")
|
||||
connection.execute("PRAGMA temp_store = MEMORY")
|
||||
connection.execute("PRAGMA cache_size = -262144")
|
||||
connection.execute(f"PRAGMA busy_timeout = {busy_timeout_ms}")
|
||||
return connection
|
||||
|
||||
def analyze_quality(source:sqlite3.Connection) -> dict[str, object]:
|
||||
games = {
|
||||
row["game_id"]: row
|
||||
for row in source.execute("""
|
||||
SELECT game_id, status, final_turn, winner_name
|
||||
FROM games
|
||||
""")
|
||||
}
|
||||
aggregates = {
|
||||
row["game_id"]: row
|
||||
for row in source.execute("""
|
||||
SELECT t.game_id,
|
||||
COUNT(*) AS turn_rows,
|
||||
MIN(t.turn) AS min_turn,
|
||||
MAX(t.turn) AS max_turn,
|
||||
SUM(CASE WHEN t.my_move IN ('up','down','left','right') THEN 1 ELSE 0 END) AS valid_moves,
|
||||
SUM(CASE WHEN t.my_thinking_json IS NOT NULL AND t.my_thinking_json NOT IN ('', '{}', 'null') THEN 1 ELSE 0 END) AS thinking_rows,
|
||||
COUNT(DISTINCT t.my_move) AS distinct_moves
|
||||
FROM turns AS t
|
||||
GROUP BY t.game_id
|
||||
""")
|
||||
}
|
||||
snake_counts = {
|
||||
row["game_id"]: int(row["snake_turn_rows"])
|
||||
for row in source.execute("""
|
||||
SELECT game_id, COUNT(*) AS snake_turn_rows
|
||||
FROM snake_turns GROUP BY game_id
|
||||
""")
|
||||
}
|
||||
|
||||
quality = {}
|
||||
for game_id, game in games.items():
|
||||
aggregate = aggregates.get(game_id)
|
||||
quality[game_id] = rate_game_quality(GameQualityInput(
|
||||
status=game["status"],
|
||||
final_turn=int(game["final_turn"] or 0),
|
||||
turn_rows=int(aggregate["turn_rows"] if aggregate else 0),
|
||||
min_turn=int(aggregate["min_turn"]) if aggregate and aggregate["min_turn"] is not None else None,
|
||||
max_turn=int(aggregate["max_turn"]) if aggregate and aggregate["max_turn"] is not None else None,
|
||||
valid_moves=int(aggregate["valid_moves"] if aggregate else 0),
|
||||
thinking_rows=int(aggregate["thinking_rows"] if aggregate else 0),
|
||||
distinct_moves=int(aggregate["distinct_moves"] if aggregate else 0),
|
||||
snake_turn_rows=snake_counts.get(game_id, 0),
|
||||
winner_name=game["winner_name"],
|
||||
))
|
||||
return quality
|
||||
|
||||
def copy_games(source:sqlite3.Connection, destination:sqlite3.Connection, batch_size:int, quality:dict[str, object], minimum_quality:str) -> tuple[int, int]:
|
||||
columns = object_columns(source, "games")
|
||||
selected = [
|
||||
"game_id", "started_at", "ended_at", "width", "height", "source", "map_name",
|
||||
"ruleset_name", "ruleset_version", "your_snake_id", "your_snake_name",
|
||||
column_or_null(columns, "your_snake_type"), column_or_null(columns, "your_snake_version"),
|
||||
column_or_null(columns, "game_type"), column_or_null(columns, "winner_name"),
|
||||
"winner_you", "final_turn", "status",
|
||||
]
|
||||
cursor = source.execute(f"SELECT {', '.join(selected)} FROM games ORDER BY game_id")
|
||||
sql = """
|
||||
INSERT INTO games (
|
||||
game_id, started_at, ended_at, width, height, source, map_name,
|
||||
ruleset_name, ruleset_version, your_snake_id, your_snake_name,
|
||||
your_snake_type, your_snake_version, game_type, winner_name,
|
||||
winner_you, final_turn, status, has_replay, quality_status,
|
||||
quality_score, quality_tier, quality_reasons_json
|
||||
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
"""
|
||||
count = 0
|
||||
retained = 0
|
||||
while rows := cursor.fetchmany(batch_size):
|
||||
values = []
|
||||
for row in rows:
|
||||
game_quality = quality[row[0]]
|
||||
keep_replay = quality_meets_minimum(game_quality.tier, minimum_quality)
|
||||
values.append((
|
||||
*tuple(row), 1 if keep_replay else 0,
|
||||
"retained" if keep_replay else "low_quality",
|
||||
game_quality.score, game_quality.tier,
|
||||
json.dumps(game_quality.reasons, separators=(",", ":")),
|
||||
))
|
||||
retained += int(keep_replay)
|
||||
destination.executemany(sql, values)
|
||||
count += len(rows)
|
||||
return count, retained
|
||||
|
||||
def retained_game_ids(quality:dict[str, object], minimum_quality:str) -> set[str]:
|
||||
return {
|
||||
game_id for game_id, result in quality.items()
|
||||
if quality_meets_minimum(result.tier, minimum_quality)
|
||||
}
|
||||
|
||||
def copy_game_snakes(source:sqlite3.Connection, destination:sqlite3.Connection, batch_size:int, retained_ids:set[str]) -> int:
|
||||
has_game_snakes = source.execute("""
|
||||
SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'game_snakes'
|
||||
""").fetchone() is not None
|
||||
sql = """
|
||||
INSERT OR IGNORE INTO game_snakes (
|
||||
game_id, snake_id, snake_name, is_you, customizations_json
|
||||
) VALUES (?, ?, ?, ?, ?)
|
||||
"""
|
||||
count = 0
|
||||
if has_game_snakes:
|
||||
columns = object_columns(source, "game_snakes")
|
||||
customizations = (
|
||||
"COALESCE(customizations_json, '{}') AS customizations_json"
|
||||
if "customizations_json" in columns else "'{}' AS customizations_json"
|
||||
)
|
||||
cursor = source.execute(f"""
|
||||
SELECT game_id, snake_id, snake_name, is_you, {customizations}
|
||||
FROM game_snakes ORDER BY game_id, snake_id
|
||||
""")
|
||||
while rows := cursor.fetchmany(batch_size):
|
||||
retained_rows = [tuple(row) for row in rows if row[0] in retained_ids]
|
||||
before = destination.total_changes
|
||||
destination.executemany(sql, retained_rows)
|
||||
count += destination.total_changes - before
|
||||
|
||||
# Older databases can contain an empty or only partially populated
|
||||
# game_snakes table. Always synthesize missing identities from snake_turns.
|
||||
cursor = source.execute("""
|
||||
SELECT game_id, snake_id, MAX(snake_name), MAX(is_you),
|
||||
'{}' AS customizations_json
|
||||
FROM snake_turns
|
||||
GROUP BY game_id, snake_id
|
||||
ORDER BY game_id, snake_id
|
||||
""")
|
||||
while rows := cursor.fetchmany(batch_size):
|
||||
retained_rows = [tuple(row) for row in rows if row[0] in retained_ids]
|
||||
before = destination.total_changes
|
||||
destination.executemany(sql, retained_rows)
|
||||
count += destination.total_changes - before
|
||||
return count
|
||||
|
||||
def decode_json(value:str|None, fallback):
|
||||
if not value:
|
||||
return fallback
|
||||
try:
|
||||
return json.loads(value)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return fallback
|
||||
|
||||
def compact_board(row:sqlite3.Row) -> tuple[str, str, str]:
|
||||
board = decode_json(row["board_state_json"], {})
|
||||
food = board.get("food") if isinstance(board, dict) else None
|
||||
hazards = board.get("hazards") if isinstance(board, dict) else None
|
||||
if food is None:
|
||||
food = decode_json(row["food_json"], [])
|
||||
if hazards is None:
|
||||
hazards = decode_json(row["hazards_json"], [])
|
||||
compact = json.dumps({}, separators=(",", ":"))
|
||||
return (
|
||||
compact,
|
||||
json.dumps(food or [], separators=(",", ":")),
|
||||
json.dumps(hazards or [], separators=(",", ":")),
|
||||
)
|
||||
|
||||
def copy_turns(source:sqlite3.Connection, destination:sqlite3.Connection, batch_size:int, retained_ids:set[str]) -> int:
|
||||
cursor = source.execute("""
|
||||
SELECT t.id, t.game_id, t.turn, t.observed_at, t.my_move, t.my_thinking_json,
|
||||
t.board_state_json, t.food_json, t.hazards_json
|
||||
FROM turns AS t
|
||||
ORDER BY t.id
|
||||
""")
|
||||
sql = """
|
||||
INSERT INTO turns (
|
||||
id, game_id, turn, observed_at, my_move, my_thinking_json,
|
||||
board_state_json, snakes_json, you_json, food_json, hazards_json
|
||||
) VALUES (?,?,?,?,?,?,?,'[]','{}',?,?)
|
||||
"""
|
||||
count = 0
|
||||
while rows := cursor.fetchmany(batch_size):
|
||||
values = []
|
||||
for row in rows:
|
||||
if row["game_id"] not in retained_ids:
|
||||
continue
|
||||
compact, food, hazards = compact_board(row)
|
||||
values.append((
|
||||
row["id"], row["game_id"], row["turn"], row["observed_at"],
|
||||
row["my_move"], row["my_thinking_json"], compact, food, hazards,
|
||||
))
|
||||
destination.executemany(sql, values)
|
||||
count += len(values)
|
||||
if count % max(batch_size, 100_000) == 0:
|
||||
print(f"turns: {count:,}", flush=True)
|
||||
return count
|
||||
|
||||
def copy_snake_turns(source:sqlite3.Connection, destination:sqlite3.Connection, batch_size:int, retained_ids:set[str]) -> int:
|
||||
columns = object_columns(source, "snake_turns")
|
||||
latency = column_or_null(columns, "latency")
|
||||
cursor = source.execute(f"""
|
||||
SELECT st.id, st.game_id, st.turn, st.snake_id, st.health, st.length,
|
||||
st.head_x, st.head_y, st.body_json, st.inferred_move, st.{latency}
|
||||
FROM snake_turns AS st
|
||||
ORDER BY st.id
|
||||
""")
|
||||
sql = """
|
||||
INSERT INTO snake_turns (
|
||||
id, game_id, turn, snake_id, snake_name, health, length,
|
||||
head_x, head_y, body_json, is_you, inferred_move, latency
|
||||
) VALUES (?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, 0, ?, ?)
|
||||
"""
|
||||
count = 0
|
||||
while rows := cursor.fetchmany(batch_size):
|
||||
destination.executemany(sql, [
|
||||
(
|
||||
row["id"], row["game_id"], row["turn"], row["snake_id"],
|
||||
row["health"], row["length"], row["head_x"], row["head_y"],
|
||||
row["body_json"], row["inferred_move"], row["latency"],
|
||||
)
|
||||
for row in rows if row["game_id"] in retained_ids
|
||||
])
|
||||
count += sum(1 for row in rows if row["game_id"] in retained_ids)
|
||||
if count % max(batch_size, 100_000) == 0:
|
||||
print(f"snake_turns: {count:,}", flush=True)
|
||||
return count
|
||||
|
||||
def verify(source:sqlite3.Connection, destination:sqlite3.Connection, retained_ids:set[str]) -> dict[str, int]:
|
||||
result = {}
|
||||
retained_turns = sum(
|
||||
1 for row in source.execute("SELECT game_id FROM turns")
|
||||
if row["game_id"] in retained_ids
|
||||
)
|
||||
retained_snake_turns = sum(
|
||||
1 for row in source.execute("SELECT game_id FROM snake_turns")
|
||||
if row["game_id"] in retained_ids
|
||||
)
|
||||
expected = {
|
||||
"games": int(source.execute("SELECT COUNT(*) FROM games").fetchone()[0]),
|
||||
"turns": retained_turns,
|
||||
"snake_turns": retained_snake_turns,
|
||||
}
|
||||
for table, source_count in expected.items():
|
||||
destination_count = int(destination.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0])
|
||||
if source_count != destination_count:
|
||||
raise RuntimeError(f"{table} count mismatch: {source_count} != {destination_count}")
|
||||
result[table] = destination_count
|
||||
integrity = destination.execute("PRAGMA integrity_check").fetchone()[0]
|
||||
if integrity != "ok":
|
||||
raise RuntimeError(f"Destination integrity check failed: {integrity}")
|
||||
return result
|
||||
|
||||
def replace_with_backup(source_path:Path, destination_path:Path) -> Path:
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||
backup = source_path.with_name(f"{source_path.name}.backup-{timestamp}")
|
||||
source_path.rename(backup)
|
||||
try:
|
||||
destination_path.rename(source_path)
|
||||
except Exception:
|
||||
backup.rename(source_path)
|
||||
raise
|
||||
return backup
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--source", required=True, type=Path)
|
||||
parser.add_argument("--destination", type=Path)
|
||||
parser.add_argument("--batch-size", type=int, default=10_000)
|
||||
parser.add_argument("--busy-timeout-ms", type=int, default=60_000)
|
||||
parser.add_argument("--minimum-quality", choices=("low", "medium", "high"), default="medium", help="Minimum quality tier whose replay rows are retained")
|
||||
parser.add_argument("--replace", action="store_true", help="Backup source and replace it after verification")
|
||||
args = parser.parse_args()
|
||||
|
||||
source_path = args.source.expanduser().resolve()
|
||||
destination_path = (args.destination or source_path.with_name(f"{source_path.stem}.compact{source_path.suffix}")).expanduser().resolve()
|
||||
if source_path == destination_path:
|
||||
raise SystemExit("Source and destination must be different paths")
|
||||
|
||||
started = perf_counter()
|
||||
source = open_source(source_path)
|
||||
destination = create_destination(destination_path, max(1000, args.busy_timeout_ms))
|
||||
try:
|
||||
quality = analyze_quality(source)
|
||||
retained_ids = retained_game_ids(quality, args.minimum_quality)
|
||||
games, retained_games = copy_games(source, destination, max(1, args.batch_size), quality, args.minimum_quality)
|
||||
game_snakes = copy_game_snakes(source, destination, max(1, args.batch_size), retained_ids)
|
||||
turns = copy_turns(source, destination, max(1, args.batch_size), retained_ids)
|
||||
snake_turns = copy_snake_turns(source, destination, max(1, args.batch_size), retained_ids)
|
||||
destination.commit()
|
||||
destination.executescript("""
|
||||
CREATE INDEX IF NOT EXISTS idx_turns_game_turn ON turns(game_id, turn);
|
||||
CREATE INDEX IF NOT EXISTS idx_games_status ON games(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_snake_turns_game_turn ON snake_turns(game_id, turn);
|
||||
""")
|
||||
destination.execute("PRAGMA foreign_keys = ON")
|
||||
counts = verify(source, destination, retained_ids)
|
||||
except Exception:
|
||||
destination.close()
|
||||
source.close()
|
||||
for suffix in ("", "-wal", "-shm"):
|
||||
Path(f"{destination_path}{suffix}").unlink(missing_ok=True)
|
||||
raise
|
||||
destination.close()
|
||||
source.close()
|
||||
|
||||
source_bytes = source_path.stat().st_size
|
||||
destination_bytes = destination_path.stat().st_size
|
||||
print(f"copied: game rates={games:,}, retained replays={retained_games:,}, game_snakes={game_snakes:,}, turns={turns:,}, snake_turns={snake_turns:,}")
|
||||
print(f"verified: {counts}; size {source_bytes / 2**30:.2f} GiB -> {destination_bytes / 2**30:.2f} GiB")
|
||||
print(f"elapsed: {perf_counter() - started:.1f}s")
|
||||
|
||||
if args.replace:
|
||||
backup = replace_with_backup(source_path, destination_path)
|
||||
print(f"source replaced; backup kept at: {backup}")
|
||||
else:
|
||||
print(f"compact database created at: {destination_path}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Migrate gameplay data from the legacy turns/snake_turns layout to compact storage.
|
||||
|
||||
Source and destination are a SQLite path or a postgresql:// DSN. Without
|
||||
--destination the migration runs in place (same database, new tables). Every
|
||||
game is read back and compared after writing; legacy tables are only removed
|
||||
with --drop-legacy, and only when every legacy replay exists in compact form.
|
||||
|
||||
Online PostgreSQL procedure:
|
||||
1. run the migration while the old bot version keeps recording,
|
||||
2. deploy the new version (it records into the compact tables),
|
||||
3. run again with --since <start of step 1> to pick up games recorded meanwhile,
|
||||
4. run with --drop-legacy.
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import sys
|
||||
from time import perf_counter
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from server.database.benchmark_states import is_postgresql_source
|
||||
from server.database.backend.PostgresqlGameplayBackend import PostgresqlGameplayBackend
|
||||
from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend
|
||||
from server.database.legacy.migrate import migrate_legacy
|
||||
from server.database.legacy.reader_postgresql import LegacyPostgresqlReader
|
||||
from server.database.legacy.reader_sqlite import LegacySqliteReader
|
||||
|
||||
_MISSING_REPLAYS = """
|
||||
SELECT COUNT(*) FROM games AS g
|
||||
WHERE g.has_replay
|
||||
AND EXISTS (SELECT 1 FROM turns AS t WHERE t.game_id = g.game_id)
|
||||
AND NOT EXISTS (SELECT 1 FROM game_replays AS r WHERE r.gid = g.gid)
|
||||
AND NOT EXISTS (SELECT 1 FROM game_turns AS c WHERE c.gid = g.gid)
|
||||
"""
|
||||
|
||||
def _progress(done:int, total:int, stats) -> None:
|
||||
print(
|
||||
f"{done:,}/{total:,} games — replays={stats.replays:,} turns={stats.turns:,} "
|
||||
f"snake states={stats.states:,} failed={stats.failed:,}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
async def _drop_legacy(destination:str) -> None:
|
||||
if is_postgresql_source(destination):
|
||||
backend = PostgresqlGameplayBackend(destination)
|
||||
pool = await backend._get_pool()
|
||||
try:
|
||||
async with pool.acquire() as conn:
|
||||
missing = int(await conn.fetchval(_MISSING_REPLAYS))
|
||||
if missing:
|
||||
raise SystemExit(f"{missing} games still lack a compact replay; legacy tables kept")
|
||||
await conn.execute("DROP TABLE snake_turns")
|
||||
await conn.execute("DROP TABLE turns")
|
||||
finally:
|
||||
await backend.close()
|
||||
else:
|
||||
connection = sqlite3.connect(destination)
|
||||
try:
|
||||
missing = int(connection.execute(_MISSING_REPLAYS.replace("g.has_replay", "g.has_replay = 1")).fetchone()[0])
|
||||
if missing:
|
||||
raise SystemExit(f"{missing} games still lack a compact replay; legacy tables kept")
|
||||
connection.execute("DROP TABLE snake_turns")
|
||||
connection.execute("DROP TABLE turns")
|
||||
connection.commit()
|
||||
connection.execute("VACUUM")
|
||||
finally:
|
||||
connection.close()
|
||||
print("legacy tables turns and snake_turns dropped")
|
||||
|
||||
async def migrate(args:argparse.Namespace) -> None:
|
||||
source = args.source
|
||||
destination = args.destination or source
|
||||
if args.drop_legacy:
|
||||
await _drop_legacy(destination)
|
||||
return
|
||||
|
||||
if is_postgresql_source(destination):
|
||||
backend = PostgresqlGameplayBackend(destination)
|
||||
await backend.initialize()
|
||||
else:
|
||||
backend = SqliteGameplayBackend(destination)
|
||||
|
||||
if is_postgresql_source(source):
|
||||
reader = await LegacyPostgresqlReader.connect(source)
|
||||
close_reader = reader.close
|
||||
else:
|
||||
reader = LegacySqliteReader(source)
|
||||
async def close_reader() -> None:
|
||||
reader.close()
|
||||
|
||||
started = perf_counter()
|
||||
try:
|
||||
stats = await migrate_legacy(
|
||||
reader, backend, since=args.since, minimum_quality=args.minimum_quality,
|
||||
verify=not args.no_verify, progress=_progress,
|
||||
)
|
||||
finally:
|
||||
await close_reader()
|
||||
await backend.close()
|
||||
|
||||
print(
|
||||
f"migrated {stats.games:,} games ({stats.replays:,} replays, {stats.turns:,} turns, "
|
||||
f"{stats.states:,} snake states) in {perf_counter() - started:.1f}s"
|
||||
)
|
||||
if args.minimum_quality:
|
||||
print(f"replays dropped below '{args.minimum_quality}' quality: {stats.dropped_replays:,}")
|
||||
if stats.failed:
|
||||
print(f"FAILED games ({stats.failed}); their legacy rows are untouched:", file=sys.stderr)
|
||||
for failure in stats.failures[:50]:
|
||||
print(f" {failure}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--source", required=True, help="SQLite path or postgresql:// DSN with legacy tables")
|
||||
parser.add_argument("--destination", help="SQLite path or postgresql:// DSN (default: in place)")
|
||||
parser.add_argument("--since", help="Only games started at or after this ISO timestamp")
|
||||
parser.add_argument("--minimum-quality", choices=("low", "medium", "high"), help="Re-rate replays and drop those below this tier")
|
||||
parser.add_argument("--no-verify", action="store_true", help="Skip reading each game back after writing")
|
||||
parser.add_argument("--drop-legacy", action="store_true", help="Drop turns/snake_turns after checking every replay was migrated")
|
||||
asyncio.run(migrate(parser.parse_args()))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,207 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Stream a normalized Battlesnake SQLite database into PostgreSQL.
|
||||
|
||||
The source is opened read-only. Rows are copied in bounded batches through
|
||||
temporary PostgreSQL tables, then inserted idempotently with ON CONFLICT. The
|
||||
script verifies source/inserted row counts and never modifies the SQLite file.
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
from time import perf_counter
|
||||
|
||||
from server.database.backend.PostgresqlGameplayBackend import PostgresqlGameplayBackend
|
||||
|
||||
TABLES = (
|
||||
(
|
||||
"games",
|
||||
(
|
||||
"game_id", "started_at", "ended_at", "width", "height", "source",
|
||||
"map_name", "ruleset_name", "ruleset_version", "your_snake_id",
|
||||
"your_snake_name", "your_snake_type", "your_snake_version", "game_type",
|
||||
"winner_name", "winner_you", "final_turn", "status", "has_replay",
|
||||
"quality_status", "quality_score", "quality_tier", "quality_reasons",
|
||||
),
|
||||
"game_id",
|
||||
),
|
||||
(
|
||||
"game_snakes",
|
||||
("game_id", "snake_id", "snake_name", "is_you", "customizations"),
|
||||
"game_id, snake_id",
|
||||
),
|
||||
(
|
||||
"turns",
|
||||
(
|
||||
"game_id", "turn", "observed_at", "my_move", "my_thinking",
|
||||
"board_state", "snakes", "you", "food", "hazards",
|
||||
),
|
||||
"game_id, turn",
|
||||
),
|
||||
(
|
||||
"snake_turns",
|
||||
(
|
||||
"game_id", "turn", "snake_id", "snake_name", "health", "length",
|
||||
"head_x", "head_y", "body", "is_you", "inferred_move", "latency",
|
||||
),
|
||||
"game_id, turn, snake_id",
|
||||
),
|
||||
)
|
||||
|
||||
SQLITE_SELECTS = {
|
||||
"games": """
|
||||
SELECT game_id, started_at, ended_at, width, height, source, map_name,
|
||||
ruleset_name, ruleset_version, your_snake_id, your_snake_name,
|
||||
your_snake_type, your_snake_version, game_type, winner_name, winner_you,
|
||||
final_turn, status, has_replay, quality_status, quality_score,
|
||||
quality_tier, quality_reasons_json
|
||||
FROM games ORDER BY game_id
|
||||
""",
|
||||
"game_snakes": """
|
||||
SELECT game_id, snake_id, snake_name, is_you, customizations_json
|
||||
FROM game_snakes ORDER BY game_id, snake_id
|
||||
""",
|
||||
"turns": """
|
||||
SELECT game_id, turn, observed_at, my_move, my_thinking_json,
|
||||
board_state_json, snakes_json, you_json, food_json, hazards_json
|
||||
FROM turns ORDER BY id
|
||||
""",
|
||||
"snake_turns": """
|
||||
SELECT game_id, turn, snake_id, snake_name, health, length, head_x, head_y,
|
||||
body_json, is_you, inferred_move, latency
|
||||
FROM snake_turns ORDER BY id
|
||||
""",
|
||||
}
|
||||
|
||||
TIMESTAMP_FIELDS = {"started_at", "ended_at", "observed_at"}
|
||||
BOOLEAN_FIELDS = {"winner_you", "has_replay", "is_you"}
|
||||
JSON_FIELDS = {
|
||||
"quality_reasons", "customizations", "my_thinking", "board_state",
|
||||
"snakes", "you", "food", "hazards", "body",
|
||||
}
|
||||
JSON_DEFAULTS = {
|
||||
"customizations": {}, "board_state": {}, "snakes": [], "you": {},
|
||||
"food": [], "hazards": [], "body": [],
|
||||
}
|
||||
|
||||
def parse_timestamp(value: str | None) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
return parsed
|
||||
|
||||
def parse_json(value: str | None, default: object = None) -> object:
|
||||
if value in (None, ""):
|
||||
return default
|
||||
try:
|
||||
return json.loads(value)
|
||||
except (json.JSONDecodeError, TypeError) as exc:
|
||||
raise ValueError(f"Invalid JSON value: {str(value)[:120]}") from exc
|
||||
|
||||
def transform_row(columns: tuple[str, ...], row: sqlite3.Row) -> tuple:
|
||||
output = []
|
||||
for column, value in zip(columns, row, strict=True):
|
||||
if column in TIMESTAMP_FIELDS:
|
||||
value = parse_timestamp(value)
|
||||
elif column in BOOLEAN_FIELDS:
|
||||
value = bool(value)
|
||||
elif column in JSON_FIELDS:
|
||||
parsed = parse_json(value, JSON_DEFAULTS.get(column))
|
||||
value = None if parsed is None else json.dumps(parsed, separators=(",", ":"))
|
||||
output.append(value)
|
||||
return tuple(output)
|
||||
|
||||
def open_source(path: Path) -> sqlite3.Connection:
|
||||
connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=60)
|
||||
connection.row_factory = sqlite3.Row
|
||||
return connection
|
||||
|
||||
async def import_table(pool, source: sqlite3.Connection, table: str,
|
||||
columns: tuple[str, ...], conflict_columns: str,
|
||||
batch_size: int) -> tuple[int, int]:
|
||||
source_count = int(source.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0])
|
||||
async with pool.acquire() as connection:
|
||||
before = int(await connection.fetchval(f"SELECT COUNT(*) FROM {table}"))
|
||||
stage = f"migration_{table}"
|
||||
await connection.execute(
|
||||
f"CREATE TEMP TABLE {stage} (LIKE {table} INCLUDING DEFAULTS)"
|
||||
)
|
||||
json_columns = [column for column in columns if column in JSON_FIELDS]
|
||||
for column in json_columns:
|
||||
await connection.execute(
|
||||
f"ALTER TABLE {stage} ALTER COLUMN {column} TYPE TEXT USING {column}::text"
|
||||
)
|
||||
cursor = source.execute(SQLITE_SELECTS[table])
|
||||
copied = 0
|
||||
while rows := cursor.fetchmany(batch_size):
|
||||
records = [transform_row(columns, row) for row in rows]
|
||||
async with connection.transaction():
|
||||
await connection.copy_records_to_table(stage, records=records, columns=columns)
|
||||
selected = ", ".join(columns)
|
||||
source_expressions = ", ".join(
|
||||
f"{column}::jsonb" if column in JSON_FIELDS else column
|
||||
for column in columns
|
||||
)
|
||||
await connection.execute(
|
||||
f"INSERT INTO {table} ({selected}) SELECT {source_expressions} FROM {stage} "
|
||||
f"ON CONFLICT ({conflict_columns}) DO NOTHING"
|
||||
)
|
||||
await connection.execute(f"TRUNCATE {stage}")
|
||||
copied += len(records)
|
||||
print(f"{table}: streamed {copied:,}/{source_count:,}", flush=True)
|
||||
after = int(await connection.fetchval(f"SELECT COUNT(*) FROM {table}"))
|
||||
inserted = after - before
|
||||
if copied != source_count:
|
||||
raise RuntimeError(f"{table}: source changed while reading ({source_count} -> {copied})")
|
||||
print(f"{table}: source={source_count:,}, inserted={inserted:,}, conflicts={source_count - inserted:,}")
|
||||
return source_count, inserted
|
||||
|
||||
async def migrate(source_path: Path, dsn: str, batch_size: int) -> None:
|
||||
source = open_source(source_path)
|
||||
try:
|
||||
quick_check = source.execute("PRAGMA quick_check").fetchone()[0]
|
||||
if quick_check != "ok":
|
||||
raise RuntimeError(f"SQLite quick_check failed: {quick_check}")
|
||||
|
||||
backend = PostgresqlGameplayBackend(dsn=dsn)
|
||||
await backend.initialize()
|
||||
pool = await backend._get_pool()
|
||||
started = perf_counter()
|
||||
results = {}
|
||||
try:
|
||||
for table, columns, conflicts in TABLES:
|
||||
results[table] = await import_table(
|
||||
pool, source, table, columns, conflicts, max(100, batch_size),
|
||||
)
|
||||
async with pool.acquire() as connection:
|
||||
invalid = int(await connection.fetchval("""
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM turns t LEFT JOIN games g USING (game_id) WHERE g.game_id IS NULL) +
|
||||
(SELECT COUNT(*) FROM game_snakes s LEFT JOIN games g USING (game_id) WHERE g.game_id IS NULL) +
|
||||
(SELECT COUNT(*) FROM snake_turns s LEFT JOIN games g USING (game_id) WHERE g.game_id IS NULL)
|
||||
"""))
|
||||
if invalid:
|
||||
raise RuntimeError(f"PostgreSQL foreign-key verification found {invalid} orphan rows")
|
||||
counts = {
|
||||
table: int(await connection.fetchval(f"SELECT COUNT(*) FROM {table}"))
|
||||
for table, _, _ in TABLES
|
||||
}
|
||||
print(f"verified PostgreSQL counts: {counts}")
|
||||
print(f"migration elapsed: {perf_counter() - started:.1f}s")
|
||||
finally:
|
||||
await backend.close()
|
||||
finally:
|
||||
source.close()
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--source", required=True, type=Path)
|
||||
parser.add_argument("--dsn", required=True)
|
||||
parser.add_argument("--batch-size", type=int, default=10_000)
|
||||
args = parser.parse_args()
|
||||
asyncio.run(migrate(args.source.expanduser().resolve(), args.dsn, args.batch_size))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user