snake-python/scripts/merge_gameplay_databases.py

#!/usr/bin/env python3
"""Safely merge a gameplay SQLite database with a temporary delta database.

Base and delta are opened read-only and copied game by game into a new
destination, which assigns fresh integer ids. Delta games are quality-rated;
all game results are kept even when their replay is excluded. The base can be
replaced only after verification and a timestamped backup.
"""
import argparse
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
import sys
from time import perf_counter

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from server.database.backend.SqliteGameplayBackend import SqliteGameplayBackend
from server.database.legacy.migrate import apply_quality

CORE_GAME_FIELDS = (
  "game_id", "width", "height", "source", "map_name", "ruleset_name",
  "ruleset_version", "your_snake_id", "your_snake_name", "your_snake_type",
  "your_snake_version", "game_type", "winner_name", "winner_you",
  "final_turn", "status",
)

def duplicate_ids(base:SqliteGameplayBackend, delta:SqliteGameplayBackend, base_ids:list[str], delta_ids:list[str]) -> set[str]:
  duplicates = set(base_ids) & set(delta_ids)
  conflicts = []
  for game_id in duplicates:
    base_game = base.read_game_bundle_sync(game_id).game
    delta_game = delta.read_game_bundle_sync(game_id).game
    if any(base_game[name] != delta_game[name] for name in CORE_GAME_FIELDS):
      conflicts.append(game_id)
  if conflicts:
    examples = ", ".join(sorted(conflicts)[:5])
    raise RuntimeError(
      f"Conflicting duplicate game_id values ({len(conflicts)}): {examples}. "
      "Nothing was merged."
    )
  return duplicates

def copy_games(source:SqliteGameplayBackend, destination:SqliteGameplayBackend, game_ids:list[str], minimum_quality:str|None=None) -> tuple[Counter, Counter]:
  counts:Counter = Counter()
  tiers:Counter = Counter()
  for game_id in game_ids:
    bundle = source.read_game_bundle_sync(game_id)
    if minimum_quality is not None:
      apply_quality(bundle, minimum_quality)
      tiers[bundle.game["quality_tier"]] += 1
    destination.import_game_bundle_sync(bundle)
    counts["games"] += 1
    counts["game_snakes"] += len(bundle.snakes)
    counts["turns"] += len(bundle.turns)
    counts["snake_states"] += len(bundle.states)
  return counts, tiers

def verify(destination:SqliteGameplayBackend, expected:Counter) -> None:
  actual:Counter = Counter()
  for game_id in destination.list_game_ids_sync():
    bundle = destination.read_game_bundle_sync(game_id)
    actual["games"] += 1
    actual["game_snakes"] += len(bundle.snakes)
    actual["turns"] += len(bundle.turns)
    actual["snake_states"] += len(bundle.states)
  for key, count in expected.items():
    if actual[key] != count:
      raise RuntimeError(f"{key} count mismatch: expected {count}, got {actual[key]}")

def replace_base(base_path:Path, destination_path:Path) -> Path:
  timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
  backup = base_path.with_name(f"{base_path.name}.backup-{timestamp}")
  base_path.rename(backup)
  try:
    destination_path.rename(base_path)
  except Exception:
    backup.rename(base_path)
    raise
  return backup

def main() -> None:
  parser = argparse.ArgumentParser(description=__doc__)
  parser.add_argument("--base", required=True, type=Path, help="Cleaned database")
  parser.add_argument("--delta", required=True, type=Path, help="Database written during cleanup")
  parser.add_argument("--destination", required=True, type=Path)
  parser.add_argument("--minimum-quality", choices=("low", "medium", "high"), default="medium")
  parser.add_argument("--replace-base", action="store_true", help="Replace base after verification and keep a timestamped backup")
  args = parser.parse_args()

  base_path = args.base.expanduser().resolve()
  delta_path = args.delta.expanduser().resolve()
  destination_path = args.destination.expanduser().resolve()
  if len({base_path, delta_path, destination_path}) != 3:
    raise SystemExit("Base, delta, and destination must be different paths")
  if destination_path.exists():
    raise SystemExit(f"Destination already exists: {destination_path}")

  started = perf_counter()
  base = SqliteGameplayBackend(str(base_path), read_only=True)
  delta = SqliteGameplayBackend(str(delta_path), read_only=True)
  base_ids = base.list_game_ids_sync()
  delta_ids = delta.list_game_ids_sync()
  duplicates = duplicate_ids(base, delta, base_ids, delta_ids)

  destination = SqliteGameplayBackend(str(destination_path), journal_mode="DELETE")
  try:
    base_counts, _ = copy_games(base, destination, base_ids)
    delta_counts, tiers = copy_games(
      delta, destination, [game_id for game_id in delta_ids if game_id not in duplicates],
      args.minimum_quality,
    )
    verify(destination, base_counts + delta_counts)
  except Exception:
    for suffix in ("", "-wal", "-shm"):
      Path(f"{destination_path}{suffix}").unlink(missing_ok=True)
    raise

  print(f"merged: base games={base_counts['games']:,}, delta games={delta_counts['games']:,}, identical duplicates skipped={len(duplicates):,}")
  print(f"delta quality: {dict(tiers)}")
  print(f"verified: {dict(base_counts + delta_counts)}; elapsed: {perf_counter() - started:.1f}s")
  if args.replace_base:
    backup = replace_base(base_path, destination_path)
    print(f"base replaced; backup kept at: {backup}")
  else:
    print(f"merged database created at: {destination_path}")

if __name__ == "__main__":
  main()