feat(database): store gameplay replays in compact form
Build and Push Docker Container / build-and-push (push) Successful in 3m52s
Build and Push Docker Container / build-and-push (push) Successful in 3m52s
- Add compact storage package: codec, records, hydrate, replay blobs, cache. - Running games write game_turns and snake_states keyed by integer ids. - Encode moves as 1-byte codes and bodies as head plus step codes. - Store food and hazards only when they changed since the previous turn. - Strip reasoning fields that duplicate stored values and restore on read. - Pack finished games into one zstd blob per game in game_replays. - Merge late turns into packed blobs during the stale-game sweep. - Extract SQLite and PostgreSQL schemas into dedicated modules. - Add read/import game bundle and list_game_ids to the backend template. - Add legacy readers and converter for the old turns/snake_turns layout. - Add migrate_gameplay_to_compact.py with in-place or cross-DB migration. - Verify every migrated game and keep legacy rows on failure. - Support --since, --minimum-quality and guarded --drop-legacy. - Add just recipe migrate-gameplay-postgres. - Rewrite merge script to copy game by game via backend bundles. - Move benchmark state loading into benchmark_states for both backends. - Update PostgreSQL cleanup script to the new replay tables. - Drop sqlite-zstd build stages from the Dockerfile. - Remove old migration scripts, normalized_turn and their tests. - Add tests for compact storage, schema, legacy migration and PostgreSQL. - Document the compact layout and migration workflow in the README.
This commit is contained in:
@@ -0,0 +1,57 @@
|
||||
"""Strip move-reasoning fields that can be rebuilt from the stored turn.
|
||||
|
||||
Snakes put values like ``turn``, ``health`` or ``head`` into their reasoning
|
||||
payload although the same values are stored in the turn and snake rows. Such a
|
||||
field is only removed when it equals the derived value, and a bitmask records
|
||||
which fields were removed, so restoring is exact.
|
||||
"""
|
||||
|
||||
DERIVED_KEYS = ("turn", "move", "head", "health", "length", "food_count", "board_size", "snakes")
|
||||
MASK_KEY = "_derived"
|
||||
_RAW_KEY = "_raw"
|
||||
|
||||
def derived_values(turn:int, move:str|None, you:dict|None, food:list, width:int|None, height:int|None, snake_count:int) -> dict:
|
||||
you = you or {}
|
||||
return {
|
||||
"turn": turn,
|
||||
"move": move,
|
||||
"head": you.get("head"),
|
||||
"health": you.get("health"),
|
||||
"length": you.get("length"),
|
||||
"food_count": len(food),
|
||||
"board_size": f"{width}x{height}",
|
||||
"snakes": snake_count,
|
||||
}
|
||||
|
||||
def _same(stored:object, derived:object) -> bool:
|
||||
# bool is an int subclass; never treat True == 1 as derivable.
|
||||
return type(stored) is type(derived) and stored == derived
|
||||
|
||||
def slim_thinking(thinking:object, derived:dict) -> object:
|
||||
if not isinstance(thinking, dict):
|
||||
return thinking
|
||||
if MASK_KEY in thinking:
|
||||
# A payload that already uses the marker key is stored verbatim.
|
||||
return {MASK_KEY: 0, _RAW_KEY: thinking}
|
||||
mask = 0
|
||||
slim = {}
|
||||
for key, value in thinking.items():
|
||||
if key in DERIVED_KEYS and _same(value, derived[key]):
|
||||
mask |= 1 << DERIVED_KEYS.index(key)
|
||||
else:
|
||||
slim[key] = value
|
||||
if mask:
|
||||
slim[MASK_KEY] = mask
|
||||
return slim
|
||||
|
||||
def restore_thinking(thinking:object, derived:dict) -> object:
|
||||
if not isinstance(thinking, dict) or MASK_KEY not in thinking:
|
||||
return thinking
|
||||
mask = int(thinking[MASK_KEY])
|
||||
if mask == 0 and _RAW_KEY in thinking:
|
||||
return thinking[_RAW_KEY]
|
||||
restored = {key: value for key, value in thinking.items() if key != MASK_KEY}
|
||||
for index, key in enumerate(DERIVED_KEYS):
|
||||
if mask & (1 << index):
|
||||
restored[key] = derived[key]
|
||||
return restored
|
||||
Reference in New Issue
Block a user