daniel156161
39a7a53873
Build and Push Docker Container / build-and-push (push) Successful in 3m52s
- Add compact storage package: codec, records, hydrate, replay blobs, cache. - Running games write game_turns and snake_states keyed by integer ids. - Encode moves as 1-byte codes and bodies as head plus step codes. - Store food and hazards only when they changed since the previous turn. - Strip reasoning fields that duplicate stored values and restore on read. - Pack finished games into one zstd blob per game in game_replays. - Merge late turns into packed blobs during the stale-game sweep. - Extract SQLite and PostgreSQL schemas into dedicated modules. - Add read/import game bundle and list_game_ids to the backend template. - Add legacy readers and converter for the old turns/snake_turns layout. - Add migrate_gameplay_to_compact.py with in-place or cross-DB migration. - Verify every migrated game and keep legacy rows on failure. - Support --since, --minimum-quality and guarded --drop-legacy. - Add just recipe migrate-gameplay-postgres. - Rewrite merge script to copy game by game via backend bundles. - Move benchmark state loading into benchmark_states for both backends. - Update PostgreSQL cleanup script to the new replay tables. - Drop sqlite-zstd build stages from the Dockerfile. - Remove old migration scripts, normalized_turn and their tests. - Add tests for compact storage, schema, legacy migration and PostgreSQL. - Document the compact layout and migration workflow in the README.
85 lines
2.7 KiB
Python
85 lines
2.7 KiB
Python
"""Copy legacy-layout games into the compact layout, verifying each game."""
|
|
|
|
from dataclasses import dataclass, field
|
|
import inspect
|
|
import json
|
|
|
|
from server.database.compact.quality import rate_bundle
|
|
from server.database.compact.records import GameBundle
|
|
from server.database.game_quality import quality_meets_minimum
|
|
|
|
from .convert import LegacyDataError
|
|
|
|
@dataclass
|
|
class MigrationStats:
|
|
games:int = 0
|
|
replays:int = 0
|
|
turns:int = 0
|
|
states:int = 0
|
|
dropped_replays:int = 0
|
|
failed:int = 0
|
|
failures:list[str] = field(default_factory=list)
|
|
|
|
async def _resolve(value):
|
|
return await value if inspect.isawaitable(value) else value
|
|
|
|
def _signature(bundle:GameBundle) -> tuple:
|
|
turns = sorted(
|
|
(record.turn, record.move, json.dumps(record.thinking, sort_keys=True), json.dumps(record.food), json.dumps(record.hazards))
|
|
for record in bundle.turns
|
|
)
|
|
states = sorted(
|
|
(state.snake_id, state.turn, state.health, state.latency, json.dumps(state.body))
|
|
for state in bundle.states
|
|
)
|
|
return turns, states
|
|
|
|
def apply_quality(bundle:GameBundle, minimum_quality:str) -> bool:
|
|
"""Rate the replay; drop its turns below the minimum. Returns whether it is kept."""
|
|
quality = rate_bundle(bundle)
|
|
keep = quality_meets_minimum(quality.tier, minimum_quality)
|
|
bundle.game.update({
|
|
"has_replay": keep,
|
|
"quality_status": "retained" if keep else "low_quality",
|
|
"quality_score": quality.score,
|
|
"quality_tier": quality.tier,
|
|
"quality_reasons": list(quality.reasons),
|
|
})
|
|
if not keep:
|
|
bundle.turns = []
|
|
bundle.states = []
|
|
return keep
|
|
|
|
async def migrate_legacy(
|
|
reader,
|
|
backend,
|
|
since:str|None=None,
|
|
minimum_quality:str|None=None,
|
|
verify:bool=True,
|
|
progress=None,
|
|
) -> MigrationStats:
|
|
stats = MigrationStats()
|
|
game_ids = await _resolve(reader.game_ids(since))
|
|
for index, game_id in enumerate(game_ids, start=1):
|
|
try:
|
|
bundle = await _resolve(reader.read_bundle(game_id))
|
|
if bundle is None:
|
|
continue
|
|
if minimum_quality and not apply_quality(bundle, minimum_quality):
|
|
stats.dropped_replays += 1
|
|
await backend.import_game_bundle(bundle)
|
|
if verify and bundle.turns and bundle.game["has_replay"]:
|
|
stored = await backend.read_game_bundle(game_id)
|
|
if stored is None or _signature(stored) != _signature(bundle):
|
|
raise LegacyDataError("stored replay differs from source")
|
|
stats.games += 1
|
|
stats.replays += int(bool(bundle.turns))
|
|
stats.turns += len(bundle.turns)
|
|
stats.states += len(bundle.states)
|
|
except LegacyDataError as error:
|
|
stats.failed += 1
|
|
stats.failures.append(f"{game_id}: {error}")
|
|
if progress is not None and (index % 500 == 0 or index == len(game_ids)):
|
|
progress(index, len(game_ids), stats)
|
|
return stats
|