daniel156161
39a7a53873
Build and Push Docker Container / build-and-push (push) Successful in 3m52s
- Add compact storage package: codec, records, hydrate, replay blobs, cache. - Running games write game_turns and snake_states keyed by integer ids. - Encode moves as 1-byte codes and bodies as head plus step codes. - Store food and hazards only when they changed since the previous turn. - Strip reasoning fields that duplicate stored values and restore on read. - Pack finished games into one zstd blob per game in game_replays. - Merge late turns into packed blobs during the stale-game sweep. - Extract SQLite and PostgreSQL schemas into dedicated modules. - Add read/import game bundle and list_game_ids to the backend template. - Add legacy readers and converter for the old turns/snake_turns layout. - Add migrate_gameplay_to_compact.py with in-place or cross-DB migration. - Verify every migrated game and keep legacy rows on failure. - Support --since, --minimum-quality and guarded --drop-legacy. - Add just recipe migrate-gameplay-postgres. - Rewrite merge script to copy game by game via backend bundles. - Move benchmark state loading into benchmark_states for both backends. - Update PostgreSQL cleanup script to the new replay tables. - Drop sqlite-zstd build stages from the Dockerfile. - Remove old migration scripts, normalized_turn and their tests. - Add tests for compact storage, schema, legacy migration and PostgreSQL. - Document the compact layout and migration workflow in the README.
99 lines
4.3 KiB
Python
99 lines
4.3 KiB
Python
"""Convert rows of the legacy layout (turns/snake_turns) into a compact bundle."""
|
|
|
|
import json
|
|
|
|
from server.database.compact.codec import encode_latency
|
|
from server.database.compact.games import normalize_game
|
|
from server.database.compact.records import (
|
|
GameBundle, PointsState, SnakeIdentity, SnakeState, compact_turn,
|
|
)
|
|
|
|
class LegacyDataError(ValueError):
|
|
"""Legacy rows that cannot be converted without losing data."""
|
|
|
|
def parse_json(value:object, default:object=None) -> object:
|
|
if value is None or value == "":
|
|
return default
|
|
if not isinstance(value, str):
|
|
return value
|
|
try:
|
|
return json.loads(value)
|
|
except (json.JSONDecodeError, TypeError) as exc:
|
|
raise LegacyDataError(f"Invalid JSON value: {value[:120]}") from exc
|
|
|
|
def _state_from_row(row:dict) -> SnakeState:
|
|
body = parse_json(row.get("body"), []) or []
|
|
head_x, head_y, length = row.get("head_x"), row.get("head_y"), row.get("length")
|
|
if length is not None and int(length) != len(body):
|
|
raise LegacyDataError(f"snake {row['snake_id']} turn {row['turn']}: length {length} != body {len(body)}")
|
|
if body and head_x is not None and (head_x, head_y) != (body[0].get("x"), body[0].get("y")):
|
|
raise LegacyDataError(f"snake {row['snake_id']} turn {row['turn']}: head differs from body[0]")
|
|
return SnakeState(
|
|
snake_id=row["snake_id"], turn=int(row["turn"]), health=row.get("health"),
|
|
latency=encode_latency(row.get("latency")), body=body,
|
|
)
|
|
|
|
def _states_from_board(board:dict, turn:int) -> list[SnakeState]:
|
|
return [
|
|
SnakeState(
|
|
snake_id=snake["id"], turn=turn, health=snake.get("health"),
|
|
latency=encode_latency(snake.get("latency")), body=list(snake.get("body", [])),
|
|
)
|
|
for snake in board.get("snakes", []) if snake.get("id") is not None
|
|
]
|
|
|
|
def _identities(game:dict, identity_rows:list[dict], snake_rows:list[dict], board_snakes:list[dict]) -> list[SnakeIdentity]:
|
|
identities:dict[str, SnakeIdentity] = {}
|
|
for row in identity_rows:
|
|
identities[row["snake_id"]] = SnakeIdentity(
|
|
row["snake_id"], row.get("snake_name"), bool(row.get("is_you")),
|
|
parse_json(row.get("customizations"), {}) or {},
|
|
)
|
|
# Older databases have an empty or partial game_snakes table.
|
|
for row in snake_rows + board_snakes:
|
|
snake_id = row.get("snake_id") or row.get("id")
|
|
if snake_id is None or snake_id in identities:
|
|
continue
|
|
name = row.get("snake_name") or row.get("name")
|
|
is_you = bool(row.get("is_you")) or snake_id == game.get("your_snake_id")
|
|
identities[snake_id] = SnakeIdentity(snake_id, name, is_you, row.get("customizations") or {})
|
|
for snake_id, identity in list(identities.items()):
|
|
if identity.name is None:
|
|
name = next((row.get("snake_name") for row in snake_rows if row.get("snake_id") == snake_id and row.get("snake_name")), None)
|
|
if name:
|
|
identities[snake_id] = SnakeIdentity(snake_id, name, identity.is_you, identity.customizations)
|
|
return list(identities.values())
|
|
|
|
def build_legacy_bundle(game_row:dict, identity_rows:list[dict], turn_rows:list[dict], snake_rows:list[dict]) -> GameBundle:
|
|
game = normalize_game(game_row)
|
|
states_by_turn:dict[int, list[SnakeState]] = {}
|
|
for row in snake_rows:
|
|
state = _state_from_row(row)
|
|
states_by_turn.setdefault(state.turn, []).append(state)
|
|
|
|
turns = []
|
|
states = []
|
|
board_snakes = []
|
|
previous:PointsState|None = None
|
|
for row in sorted(turn_rows, key=lambda item: int(item["turn"])):
|
|
turn = int(row["turn"])
|
|
board = parse_json(row.get("board_state"), {}) or {}
|
|
food = board["food"] if "food" in board else parse_json(row.get("food"), []) or []
|
|
hazards = board["hazards"] if "hazards" in board else parse_json(row.get("hazards"), []) or []
|
|
turn_states = states_by_turn.pop(turn, None)
|
|
if turn_states is None:
|
|
turn_states = _states_from_board(board, turn)
|
|
board_snakes.extend(board.get("snakes", []))
|
|
states.extend(turn_states)
|
|
record, previous = compact_turn(
|
|
turn, row.get("observed_at"), row.get("my_move"), parse_json(row.get("my_thinking")),
|
|
food, hazards, turn_states, game.get("your_snake_id"), game.get("width"), game.get("height"),
|
|
previous,
|
|
)
|
|
turns.append(record)
|
|
# Snake rows without a turn row are kept as well.
|
|
for orphan in states_by_turn.values():
|
|
states.extend(orphan)
|
|
|
|
return GameBundle(game, _identities(game, identity_rows, snake_rows, board_snakes), turns, states)
|