daniel156161
39a7a53873
Build and Push Docker Container / build-and-push (push) Successful in 3m52s
- Add compact storage package: codec, records, hydrate, replay blobs, cache. - Running games write game_turns and snake_states keyed by integer ids. - Encode moves as 1-byte codes and bodies as head plus step codes. - Store food and hazards only when they changed since the previous turn. - Strip reasoning fields that duplicate stored values and restore on read. - Pack finished games into one zstd blob per game in game_replays. - Merge late turns into packed blobs during the stale-game sweep. - Extract SQLite and PostgreSQL schemas into dedicated modules. - Add read/import game bundle and list_game_ids to the backend template. - Add legacy readers and converter for the old turns/snake_turns layout. - Add migrate_gameplay_to_compact.py with in-place or cross-DB migration. - Verify every migrated game and keep legacy rows on failure. - Support --since, --minimum-quality and guarded --drop-legacy. - Add just recipe migrate-gameplay-postgres. - Rewrite merge script to copy game by game via backend bundles. - Move benchmark state loading into benchmark_states for both backends. - Update PostgreSQL cleanup script to the new replay tables. - Drop sqlite-zstd build stages from the Dockerfile. - Remove old migration scripts, normalized_turn and their tests. - Add tests for compact storage, schema, legacy migration and PostgreSQL. - Document the compact layout and migration workflow in the README.
64 lines
2.2 KiB
Python
64 lines
2.2 KiB
Python
"""Pack a finished game's turns into one zstd-compressed blob.
|
|
|
|
Consecutive turns are nearly identical, so compressing a whole game at once is
|
|
far smaller than storing one row per turn and snake.
|
|
"""
|
|
|
|
from compression import zstd
|
|
import json
|
|
|
|
from .codec import (
|
|
decode_move, encode_move, flatten_points, from_epoch_us, to_epoch_us,
|
|
unflatten_points,
|
|
)
|
|
from .records import SnakeState, TurnRecord
|
|
|
|
REPLAY_FORMAT = 1
|
|
_LEVEL = 10
|
|
|
|
def _optional_points(points:list|None) -> list|None:
|
|
return None if points is None else flatten_points(points)
|
|
|
|
def _optional_unflatten(values:list|None) -> list|None:
|
|
return None if values is None else unflatten_points(values)
|
|
|
|
def pack_replay(turns:list[TurnRecord], states:list[SnakeState]) -> bytes:
|
|
by_snake:dict[str, list] = {}
|
|
for state in sorted(states, key=lambda s: (s.snake_id, s.turn)):
|
|
by_snake.setdefault(state.snake_id, []).append(
|
|
[state.turn, state.health, state.latency, flatten_points(state.body)]
|
|
)
|
|
payload = {
|
|
"t": [
|
|
[
|
|
record.turn, to_epoch_us(record.observed_at), encode_move(record.move),
|
|
record.thinking, _optional_points(record.food), _optional_points(record.hazards),
|
|
]
|
|
for record in sorted(turns, key=lambda item: item.turn)
|
|
],
|
|
"s": by_snake,
|
|
}
|
|
raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode()
|
|
return zstd.compress(raw, level=_LEVEL)
|
|
|
|
def unpack_replay(data:bytes|memoryview, format_version:int=REPLAY_FORMAT) -> tuple[list[TurnRecord], list[SnakeState]]:
|
|
if format_version != REPLAY_FORMAT:
|
|
raise ValueError(f"Unsupported replay format: {format_version}")
|
|
payload = json.loads(zstd.decompress(bytes(data)))
|
|
turns = [
|
|
TurnRecord(
|
|
turn=turn, observed_at=from_epoch_us(observed), move=decode_move(move),
|
|
thinking=thinking, food=_optional_unflatten(food), hazards=_optional_unflatten(hazards),
|
|
)
|
|
for turn, observed, move, thinking, food, hazards in payload["t"]
|
|
]
|
|
states = [
|
|
SnakeState(
|
|
snake_id=snake_id, turn=turn, health=health, latency=latency,
|
|
body=unflatten_points(body),
|
|
)
|
|
for snake_id, rows in payload["s"].items()
|
|
for turn, health, latency, body in rows
|
|
]
|
|
return turns, states
|