daniel156161
39a7a53873
Build and Push Docker Container / build-and-push (push) Successful in 3m52s
- Add compact storage package: codec, records, hydrate, replay blobs, cache. - Running games write game_turns and snake_states keyed by integer ids. - Encode moves as 1-byte codes and bodies as head plus step codes. - Store food and hazards only when they changed since the previous turn. - Strip reasoning fields that duplicate stored values and restore on read. - Pack finished games into one zstd blob per game in game_replays. - Merge late turns into packed blobs during the stale-game sweep. - Extract SQLite and PostgreSQL schemas into dedicated modules. - Add read/import game bundle and list_game_ids to the backend template. - Add legacy readers and converter for the old turns/snake_turns layout. - Add migrate_gameplay_to_compact.py with in-place or cross-DB migration. - Verify every migrated game and keep legacy rows on failure. - Support --since, --minimum-quality and guarded --drop-legacy. - Add just recipe migrate-gameplay-postgres. - Rewrite merge script to copy game by game via backend bundles. - Move benchmark state loading into benchmark_states for both backends. - Update PostgreSQL cleanup script to the new replay tables. - Drop sqlite-zstd build stages from the Dockerfile. - Remove old migration scripts, normalized_turn and their tests. - Add tests for compact storage, schema, legacy migration and PostgreSQL. - Document the compact layout and migration workflow in the README.
88 lines
3.5 KiB
Python
88 lines
3.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Delete old low-quality replay payloads while preserving game results."""
|
|
import argparse
|
|
import asyncio
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
async def cleanup(dsn: str, older_than_days: int, dry_run: bool, vacuum: bool) -> None:
|
|
try:
|
|
import asyncpg
|
|
except ImportError as exc:
|
|
raise RuntimeError("asyncpg is required for PostgreSQL cleanup") from exc
|
|
|
|
cutoff = datetime.now(timezone.utc) - timedelta(days=max(1, older_than_days))
|
|
connection = await asyncpg.connect(dsn=dsn)
|
|
try:
|
|
candidates = int(await connection.fetchval("""
|
|
SELECT count(*)
|
|
FROM games
|
|
WHERE has_replay
|
|
AND quality_status = 'low_quality'
|
|
AND COALESCE(ended_at, started_at) < $1
|
|
""", cutoff))
|
|
rows = await connection.fetchrow("""
|
|
WITH candidates AS (
|
|
SELECT game_id, gid FROM games
|
|
WHERE has_replay
|
|
AND quality_status = 'low_quality'
|
|
AND COALESCE(ended_at, started_at) < $1
|
|
)
|
|
SELECT
|
|
(SELECT count(*) FROM game_replays r JOIN candidates c USING (gid)) AS replays,
|
|
(SELECT count(*) FROM game_turns t JOIN candidates c USING (gid)) AS turns,
|
|
(SELECT count(*) FROM game_snakes s JOIN candidates c USING (game_id)) AS game_snakes
|
|
""", cutoff)
|
|
print(
|
|
f"candidates before {cutoff.isoformat()}: games={candidates:,}, "
|
|
f"replays={rows['replays']:,}, running turns={rows['turns']:,}, "
|
|
f"game_snakes={rows['game_snakes']:,}"
|
|
)
|
|
if dry_run or candidates == 0:
|
|
print("dry run: no rows changed" if dry_run else "nothing to clean")
|
|
return
|
|
|
|
async with connection.transaction():
|
|
game_ids = await connection.fetch("""
|
|
SELECT game_id FROM games
|
|
WHERE has_replay
|
|
AND quality_status = 'low_quality'
|
|
AND COALESCE(ended_at, started_at) < $1
|
|
FOR UPDATE
|
|
""", cutoff)
|
|
ids = [row["game_id"] for row in game_ids]
|
|
# snake_states rows cascade from game_snakes.
|
|
await connection.execute("""
|
|
DELETE FROM game_replays WHERE gid IN (SELECT gid FROM games WHERE game_id = ANY($1::text[]))
|
|
""", ids)
|
|
await connection.execute("""
|
|
DELETE FROM game_turns WHERE gid IN (SELECT gid FROM games WHERE game_id = ANY($1::text[]))
|
|
""", ids)
|
|
await connection.execute("DELETE FROM game_snakes WHERE game_id = ANY($1::text[])", ids)
|
|
await connection.execute("""
|
|
UPDATE games
|
|
SET has_replay = FALSE, quality_status = 'low_quality'
|
|
WHERE game_id = ANY($1::text[])
|
|
""", ids)
|
|
print(f"cleaned replay payloads for {len(ids):,} games; result rows preserved")
|
|
if vacuum:
|
|
await connection.execute("VACUUM (ANALYZE) games")
|
|
await connection.execute("VACUUM (ANALYZE) game_snakes")
|
|
await connection.execute("VACUUM (ANALYZE) game_replays")
|
|
await connection.execute("VACUUM (ANALYZE) game_turns")
|
|
await connection.execute("VACUUM (ANALYZE) snake_states")
|
|
print("vacuum/analyze complete")
|
|
finally:
|
|
await connection.close()
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--dsn", required=True)
|
|
parser.add_argument("--older-than-days", type=int, default=30)
|
|
parser.add_argument("--execute", action="store_true", help="Apply deletion; default is dry-run")
|
|
parser.add_argument("--vacuum", action="store_true")
|
|
args = parser.parse_args()
|
|
asyncio.run(cleanup(args.dsn, args.older_than_days, not args.execute, args.vacuum))
|
|
|
|
if __name__ == "__main__":
|
|
main()
|