daniel156161
cb6c8d4dc8
- Share duel search contexts and transpositions across candidate moves. - Add aspiration windows, principal variation ordering, and body caches. - Model simultaneous multiplayer responses with a compact beam rollout. - Adapt search depth and response breadth to the remaining deadline. - Add a deterministic arena benchmark with optional JSON reporting. - Expose search metrics, document benchmarking, and bump Prism to 1.2.0.
118 lines
4.2 KiB
Python
Executable File
118 lines
4.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Compare snake decisions and latency on deterministic synthetic positions.
|
|
|
|
For outcome/win-rate tournaments use the local Battlesnake CLI. This harness is
|
|
fast enough for CI and detects move disagreements, crashes, and latency changes.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from statistics import mean, median
|
|
from time import perf_counter
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from scripts.benchmark_snakes_from_db import load_states, percentile
|
|
from server.GameBoard import GameBoard
|
|
from snakes import SnakeBuilder
|
|
from tests.bench_best_battle_snake import build_game_state
|
|
|
|
def synthetic_states(count: int) -> list[tuple[dict, dict]]:
|
|
states: list[tuple[dict, dict]] = []
|
|
for index in range(count):
|
|
payload = build_game_state()
|
|
payload["game"]["id"] = f"arena-{index}"
|
|
payload["turn"] = 20 + index
|
|
# Rotate food deterministically to exercise routing without creating
|
|
# invalid bodies or relying on private/real network values.
|
|
payload["board"]["food"] = [
|
|
{"x": 1 + index % 3, "y": 9},
|
|
{"x": 9, "y": 1 + (index // 3) % 3},
|
|
]
|
|
states.append((payload["board"], {
|
|
"game_id": payload["game"]["id"],
|
|
"source": "custom", "map": "standard",
|
|
"ruleset": payload["game"]["ruleset"], "turn": payload["turn"],
|
|
"you": payload["you"],
|
|
}))
|
|
return states
|
|
|
|
def evaluate(name: str, states: list[tuple[dict, dict]]) -> tuple[list[str], dict]:
|
|
moves: list[str] = []
|
|
durations: list[float] = []
|
|
depths: list[int] = []
|
|
for index, (board_data, metadata) in enumerate(states):
|
|
snake = SnakeBuilder.build(name)
|
|
game_id = f"arena-{name}-{index}-{metadata['game_id']}"
|
|
board = GameBoard(
|
|
game_id=game_id, width=board_data["width"], height=board_data["height"],
|
|
ruleset=metadata["ruleset"], source=metadata["source"],
|
|
map=metadata["map"], snake_class=snake,
|
|
)
|
|
board.read_game_data({
|
|
"game": {
|
|
"id": game_id, "ruleset": metadata["ruleset"],
|
|
"source": metadata["source"], "map": metadata["map"], "timeout": 500,
|
|
},
|
|
"turn": metadata["turn"], "board": board_data, "you": metadata["you"],
|
|
})
|
|
started = perf_counter()
|
|
moves.append(snake.choose_move(board))
|
|
durations.append((perf_counter() - started) * 1000.0)
|
|
history = snake.get_history() if hasattr(snake, "get_history") else []
|
|
if history:
|
|
depths.append(int(history[-1].get("minimax_depth_reached", 0)))
|
|
return moves, {
|
|
"snake": name, "positions": len(states),
|
|
"mean_ms": mean(durations), "median_ms": median(durations),
|
|
"p95_ms": percentile(durations, 0.95), "max_ms": max(durations),
|
|
"mean_minimax_depth": mean(depths) if depths else 0.0,
|
|
}
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--snake", action="append", default=[])
|
|
parser.add_argument("--database")
|
|
parser.add_argument("--positions", type=int, default=100)
|
|
parser.add_argument("--stride", type=int, default=997)
|
|
parser.add_argument("--json-output")
|
|
args = parser.parse_args()
|
|
|
|
states = (
|
|
load_states(args.database, max(1, args.positions), max(1, args.stride))
|
|
if args.database else synthetic_states(max(1, args.positions))
|
|
)
|
|
if not states:
|
|
raise SystemExit("No benchmark positions found")
|
|
|
|
names = args.snake or ["ApexBattleSnake", "PrismBattleSnake_GPT_5_6_Sol"]
|
|
move_sets: dict[str, list[str]] = {}
|
|
reports: list[dict] = []
|
|
for name in names:
|
|
moves, report = evaluate(name, states)
|
|
move_sets[name] = moves
|
|
reports.append(report)
|
|
print(
|
|
f"{name}: mean={report['mean_ms']:.3f} ms, "
|
|
f"p95={report['p95_ms']:.3f} ms, max={report['max_ms']:.3f} ms, "
|
|
f"depth={report['mean_minimax_depth']:.2f}"
|
|
)
|
|
baseline = names[0]
|
|
disagreements = {
|
|
name: sum(a != b for a, b in zip(move_sets[baseline], move_sets[name]))
|
|
for name in names[1:]
|
|
}
|
|
if disagreements:
|
|
print(f"Move disagreements versus {baseline}: {disagreements}")
|
|
|
|
payload = {"reports": reports, "baseline": baseline, "disagreements": disagreements}
|
|
if args.json_output:
|
|
Path(args.json_output).write_text(json.dumps(payload, indent=2) + "\n")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|