feat(snake): modularize engine and add tournament tools
- Split active strategies, reusable engine code, core classes, and legacy snakes. - Replace implicit snake imports with explicit module registrations. - Extract Prism duel, spatial, and survival behavior into focused mixins. - Improve duel scoring with food races, pressure, caches, and depth metrics. - Add deterministic arena scenarios and paired seeded engine tournaments. - Expand benchmark telemetry and bump Prism to version 1.3.0. - Update documentation and tests for the new package layout and tooling.
This commit is contained in:
@@ -17,34 +17,20 @@ from time import perf_counter
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from scripts.benchmark_snakes_from_db import load_states, percentile
|
||||
from scripts.snake_arena_scenarios import SCENARIOS, synthetic_states
|
||||
from server.GameBoard import GameBoard
|
||||
from snakes import SnakeBuilder
|
||||
from tests.bench_best_battle_snake import build_game_state
|
||||
|
||||
def synthetic_states(count: int) -> list[tuple[dict, dict]]:
|
||||
states: list[tuple[dict, dict]] = []
|
||||
for index in range(count):
|
||||
payload = build_game_state()
|
||||
payload["game"]["id"] = f"arena-{index}"
|
||||
payload["turn"] = 20 + index
|
||||
# Rotate food deterministically to exercise routing without creating
|
||||
# invalid bodies or relying on private/real network values.
|
||||
payload["board"]["food"] = [
|
||||
{"x": 1 + index % 3, "y": 9},
|
||||
{"x": 9, "y": 1 + (index // 3) % 3},
|
||||
]
|
||||
states.append((payload["board"], {
|
||||
"game_id": payload["game"]["id"],
|
||||
"source": "custom", "map": "standard",
|
||||
"ruleset": payload["game"]["ruleset"], "turn": payload["turn"],
|
||||
"you": payload["you"],
|
||||
}))
|
||||
return states
|
||||
|
||||
def evaluate(name: str, states: list[tuple[dict, dict]]) -> tuple[list[str], dict]:
|
||||
moves: list[str] = []
|
||||
durations: list[float] = []
|
||||
depths: list[int] = []
|
||||
duel_depths: list[int] = []
|
||||
rollout_depths: list[int] = []
|
||||
duel_nodes = 0
|
||||
rollout_nodes = 0
|
||||
cache_hits = 0
|
||||
deadline_exits = 0
|
||||
scenario_durations: dict[str, list[float]] = {}
|
||||
for index, (board_data, metadata) in enumerate(states):
|
||||
snake = SnakeBuilder.build(name)
|
||||
game_id = f"arena-{name}-{index}-{metadata['game_id']}"
|
||||
@@ -62,15 +48,32 @@ def evaluate(name: str, states: list[tuple[dict, dict]]) -> tuple[list[str], dic
|
||||
})
|
||||
started = perf_counter()
|
||||
moves.append(snake.choose_move(board))
|
||||
durations.append((perf_counter() - started) * 1000.0)
|
||||
duration = (perf_counter() - started) * 1000.0
|
||||
durations.append(duration)
|
||||
scenario = metadata.get("scenario", "recorded")
|
||||
scenario_durations.setdefault(scenario, []).append(duration)
|
||||
history = snake.get_history() if hasattr(snake, "get_history") else []
|
||||
if history:
|
||||
depths.append(int(history[-1].get("minimax_depth_reached", 0)))
|
||||
thinking = history[-1]
|
||||
duel_depths.append(int(thinking.get("prism_duel_depth", thinking.get("minimax_depth_reached", 0))))
|
||||
rollout_depths.append(int(thinking.get("prism_rollout_depth", 0)))
|
||||
duel_nodes += int(thinking.get("prism_duel_nodes", 0))
|
||||
rollout_nodes += int(thinking.get("prism_rollout_nodes", 0))
|
||||
cache_hits += int(thinking.get("prism_duel_cache_hits", 0))
|
||||
cache_hits += int(thinking.get("prism_rollout_cache_hits", 0))
|
||||
deadline_exits += int(thinking.get("prism_duel_deadline_exits", 0))
|
||||
deadline_exits += int(thinking.get("prism_rollout_deadline_exits", 0))
|
||||
return moves, {
|
||||
"snake": name, "positions": len(states),
|
||||
"mean_ms": mean(durations), "median_ms": median(durations),
|
||||
"p95_ms": percentile(durations, 0.95), "max_ms": max(durations),
|
||||
"mean_minimax_depth": mean(depths) if depths else 0.0,
|
||||
"mean_duel_depth": mean(duel_depths) if duel_depths else 0.0,
|
||||
"mean_rollout_depth": mean(rollout_depths) if rollout_depths else 0.0,
|
||||
"duel_nodes": duel_nodes, "rollout_nodes": rollout_nodes,
|
||||
"cache_hits": cache_hits, "deadline_exits": deadline_exits,
|
||||
"scenario_mean_ms": {
|
||||
scenario: mean(values) for scenario, values in scenario_durations.items()
|
||||
},
|
||||
}
|
||||
|
||||
def main() -> None:
|
||||
@@ -79,12 +82,13 @@ def main() -> None:
|
||||
parser.add_argument("--database")
|
||||
parser.add_argument("--positions", type=int, default=100)
|
||||
parser.add_argument("--stride", type=int, default=997)
|
||||
parser.add_argument("--scenario", action="append", choices=sorted(SCENARIOS))
|
||||
parser.add_argument("--json-output")
|
||||
args = parser.parse_args()
|
||||
|
||||
states = (
|
||||
load_states(args.database, max(1, args.positions), max(1, args.stride))
|
||||
if args.database else synthetic_states(max(1, args.positions))
|
||||
if args.database else synthetic_states(max(1, args.positions), args.scenario)
|
||||
)
|
||||
if not states:
|
||||
raise SystemExit("No benchmark positions found")
|
||||
@@ -99,7 +103,10 @@ def main() -> None:
|
||||
print(
|
||||
f"{name}: mean={report['mean_ms']:.3f} ms, "
|
||||
f"p95={report['p95_ms']:.3f} ms, max={report['max_ms']:.3f} ms, "
|
||||
f"depth={report['mean_minimax_depth']:.2f}"
|
||||
f"duel-depth={report['mean_duel_depth']:.2f}, "
|
||||
f"rollout-depth={report['mean_rollout_depth']:.2f}, "
|
||||
f"nodes={report['duel_nodes'] + report['rollout_nodes']}, "
|
||||
f"cache-hits={report['cache_hits']}, deadline-exits={report['deadline_exits']}"
|
||||
)
|
||||
baseline = names[0]
|
||||
disagreements = {
|
||||
@@ -109,7 +116,25 @@ def main() -> None:
|
||||
if disagreements:
|
||||
print(f"Move disagreements versus {baseline}: {disagreements}")
|
||||
|
||||
payload = {"reports": reports, "baseline": baseline, "disagreements": disagreements}
|
||||
scenario_disagreements = {}
|
||||
for name in names[1:]:
|
||||
counts: dict[str, int] = {}
|
||||
for index, (baseline_move, candidate_move) in enumerate(
|
||||
zip(move_sets[baseline], move_sets[name])
|
||||
):
|
||||
if baseline_move != candidate_move:
|
||||
scenario = states[index][1].get("scenario", "recorded")
|
||||
counts[scenario] = counts.get(scenario, 0) + 1
|
||||
scenario_disagreements[name] = counts
|
||||
if any(scenario_disagreements.values()):
|
||||
print(f"Disagreements by scenario: {scenario_disagreements}")
|
||||
|
||||
payload = {
|
||||
"reports": reports,
|
||||
"baseline": baseline,
|
||||
"disagreements": disagreements,
|
||||
"scenario_disagreements": scenario_disagreements,
|
||||
}
|
||||
if args.json_output:
|
||||
Path(args.json_output).write_text(json.dumps(payload, indent=2) + "\n")
|
||||
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run paired seeded games through the official local Battlesnake rules engine."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from collections import Counter
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from threading import Thread
|
||||
from pathlib import Path
|
||||
from statistics import mean
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
ENGINE_USER_AGENT = "BattlesnakeEngine/local-tournament"
|
||||
RESULT_RE = re.compile(
|
||||
r"Game completed after (\d+) turns\.(?: (.+?) was the winner\.| It was a draw\.)"
|
||||
)
|
||||
|
||||
def wait_for_server(url: str, process: subprocess.Popen, timeout: float = 15.0) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if process.poll() is not None:
|
||||
raise RuntimeError(f"Snake server exited with status {process.returncode}")
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=0.5) as response:
|
||||
if response.status == 200:
|
||||
return
|
||||
except OSError:
|
||||
time.sleep(0.1)
|
||||
raise TimeoutError(f"Snake server did not become ready at {url}")
|
||||
|
||||
class _EngineHeaderProxy(BaseHTTPRequestHandler):
|
||||
target: str
|
||||
|
||||
def do_GET(self) -> None:
|
||||
self._forward()
|
||||
|
||||
def do_POST(self) -> None:
|
||||
self._forward()
|
||||
|
||||
def _forward(self) -> None:
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
body = self.rfile.read(length) if length else None
|
||||
request = urllib.request.Request(
|
||||
f"{self.target}{self.path}", data=body, method=self.command,
|
||||
headers={
|
||||
"Content-Type": self.headers.get("Content-Type", "application/json"),
|
||||
"User-Agent": ENGINE_USER_AGENT,
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=2.0) as response:
|
||||
payload = response.read()
|
||||
self.send_response(response.status)
|
||||
self.send_header("Content-Type", response.headers.get("Content-Type", "application/json"))
|
||||
except urllib.error.HTTPError as error:
|
||||
payload = error.read()
|
||||
self.send_response(error.code)
|
||||
self.send_header("Content-Type", error.headers.get("Content-Type", "text/plain"))
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
def log_message(self, format: str, *args) -> None:
|
||||
pass
|
||||
|
||||
def start_proxy(port: int, target_port: int) -> tuple[ThreadingHTTPServer, Thread]:
|
||||
handler = type(
|
||||
f"EngineHeaderProxy{port}",
|
||||
(_EngineHeaderProxy,),
|
||||
{"target": f"http://127.0.0.1:{target_port}"},
|
||||
)
|
||||
server = ThreadingHTTPServer(("127.0.0.1", port), handler)
|
||||
thread = Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
return server, thread
|
||||
|
||||
def start_server(snake: str, port: int) -> subprocess.Popen:
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"HOST": "127.0.0.1",
|
||||
"PORT": str(port),
|
||||
"SNAKE": snake,
|
||||
"DEBUG": "false",
|
||||
"DEBUG_SERVER": "false",
|
||||
"STORE_GAME_HISTORY": "false",
|
||||
"GAMEPLAY_DB_ENABLED": "false",
|
||||
"METRICS_CLEAR_WORKERS_ON_STARTUP": "false",
|
||||
})
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, str(ROOT / "main.py")],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
start_new_session=True,
|
||||
)
|
||||
wait_for_server(f"http://127.0.0.1:{port}", process)
|
||||
return process
|
||||
|
||||
def stop_server(process: subprocess.Popen) -> None:
|
||||
if process.poll() is not None:
|
||||
return
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
try:
|
||||
process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait(timeout=5)
|
||||
|
||||
def run_game(
|
||||
cli: Path,
|
||||
seed: int,
|
||||
game_type: str,
|
||||
map_name: str,
|
||||
players: list[tuple[str, str]],
|
||||
width: int,
|
||||
height: int,
|
||||
timeout_ms: int,
|
||||
) -> dict:
|
||||
with tempfile.NamedTemporaryFile(prefix="snake-arena-", suffix=".jsonl") as output:
|
||||
command = [
|
||||
str(cli), "play", "-W", str(width), "-H", str(height),
|
||||
"-g", game_type, "--map", map_name, "--seed", str(seed),
|
||||
"--timeout", str(timeout_ms), "--output", output.name,
|
||||
]
|
||||
for name, url in players:
|
||||
command.extend(("--name", name, "--url", url))
|
||||
completed = subprocess.run(
|
||||
command, cwd=ROOT, text=True, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT, timeout=180, check=False,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"Rules engine failed for seed {seed}:\n{completed.stdout[-2000:]}"
|
||||
)
|
||||
output.seek(0)
|
||||
lines = [json.loads(line) for line in output if line.strip()]
|
||||
|
||||
terminal = lines[-1] if lines else {}
|
||||
match = RESULT_RE.search(completed.stdout)
|
||||
turns = int(match.group(1)) if match else max(0, len(lines) - 2)
|
||||
winner = terminal.get("winnerName") or None
|
||||
draw = bool(terminal.get("isDraw", winner is None))
|
||||
return {"seed": seed, "winner": winner, "draw": draw, "turns": turns}
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--games", type=int, default=20, help="Number of unique seeds")
|
||||
parser.add_argument("--seed-start", type=int, default=1)
|
||||
parser.add_argument("--gametype", default="standard")
|
||||
parser.add_argument("--map", default="standard")
|
||||
parser.add_argument("--width", type=int, default=11)
|
||||
parser.add_argument("--height", type=int, default=11)
|
||||
parser.add_argument("--timeout", type=int, default=500)
|
||||
parser.add_argument("--base-port", type=int, default=9301)
|
||||
parser.add_argument("--cli", default=str(ROOT / ".testing/tools/battlesnake-cli/battlesnake"))
|
||||
parser.add_argument("--json-output")
|
||||
args = parser.parse_args()
|
||||
|
||||
cli = Path(args.cli)
|
||||
if not cli.is_file():
|
||||
raise SystemExit(f"Battlesnake CLI not found: {cli}. Run: just build-battlesnake-cli")
|
||||
|
||||
apex_port, prism_port = args.base_port, args.base_port + 1
|
||||
apex_proxy_port, prism_proxy_port = args.base_port + 2, args.base_port + 3
|
||||
servers: list[subprocess.Popen] = []
|
||||
proxies: list[tuple[ThreadingHTTPServer, Thread]] = []
|
||||
results: list[dict] = []
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
servers = [
|
||||
start_server("ApexBattleSnake", apex_port),
|
||||
start_server("PrismBattleSnake_GPT_5_6_Sol", prism_port),
|
||||
]
|
||||
proxies = [
|
||||
start_proxy(apex_proxy_port, apex_port),
|
||||
start_proxy(prism_proxy_port, prism_port),
|
||||
]
|
||||
urls = {
|
||||
"Apex": f"http://127.0.0.1:{apex_proxy_port}",
|
||||
"Prism": f"http://127.0.0.1:{prism_proxy_port}",
|
||||
}
|
||||
for offset in range(max(1, args.games)):
|
||||
seed = args.seed_start + offset
|
||||
# Swap engine slots for every seed. This controls for deterministic map
|
||||
# spawn positions and gives each strategy both initial placements.
|
||||
for order in (("Apex", "Prism"), ("Prism", "Apex")):
|
||||
result = run_game(
|
||||
cli=cli, seed=seed, game_type=args.gametype, map_name=args.map,
|
||||
players=[(name, urls[name]) for name in order],
|
||||
width=args.width, height=args.height, timeout_ms=args.timeout,
|
||||
)
|
||||
result["order"] = list(order)
|
||||
results.append(result)
|
||||
print(
|
||||
f"seed={seed} order={'/'.join(order)} winner={result['winner'] or 'draw'} "
|
||||
f"turns={result['turns']}"
|
||||
)
|
||||
finally:
|
||||
for proxy, thread in reversed(proxies):
|
||||
proxy.shutdown()
|
||||
proxy.server_close()
|
||||
thread.join(timeout=2)
|
||||
for server in reversed(servers):
|
||||
stop_server(server)
|
||||
|
||||
wins = Counter(result["winner"] or "draw" for result in results)
|
||||
summary = {
|
||||
"games": len(results),
|
||||
"unique_seeds": max(1, args.games),
|
||||
"gametype": args.gametype,
|
||||
"map": args.map,
|
||||
"wins": dict(wins),
|
||||
"win_rates": {
|
||||
key: value / len(results) for key, value in wins.items()
|
||||
},
|
||||
"mean_turns": mean(result["turns"] for result in results),
|
||||
"elapsed_seconds": time.perf_counter() - started,
|
||||
"results": results,
|
||||
}
|
||||
print(json.dumps({key: value for key, value in summary.items() if key != "results"}, indent=2))
|
||||
if args.json_output:
|
||||
Path(args.json_output).write_text(json.dumps(summary, indent=2) + "\n")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Deterministic scenario corpus for the local snake arena."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
|
||||
from tests.bench_best_battle_snake import build_game_state
|
||||
|
||||
def _state(payload: dict, scenario: str, index: int) -> tuple[dict, dict]:
|
||||
payload["game"]["id"] = f"arena-{scenario}-{index}"
|
||||
return payload["board"], {
|
||||
"game_id": payload["game"]["id"],
|
||||
"source": "custom",
|
||||
"map": payload["game"].get("map", "standard"),
|
||||
"ruleset": payload["game"]["ruleset"],
|
||||
"turn": payload["turn"],
|
||||
"you": payload["you"],
|
||||
"scenario": scenario,
|
||||
}
|
||||
|
||||
def _standard_duel(index: int) -> dict:
|
||||
payload = build_game_state()
|
||||
payload["turn"] = 20 + index
|
||||
payload["board"]["food"] = [
|
||||
{"x": 1 + index % 3, "y": 9},
|
||||
{"x": 9, "y": 1 + (index // 3) % 3},
|
||||
]
|
||||
return payload
|
||||
|
||||
def _hazard_duel(index: int) -> dict:
|
||||
payload = _standard_duel(index)
|
||||
payload["you"]["health"] = 38 + index % 12
|
||||
payload["board"]["snakes"][0]["health"] = payload["you"]["health"]
|
||||
hazard_x = 5 + index % 2
|
||||
payload["board"]["hazards"] = [
|
||||
{"x": hazard_x, "y": y} for y in range(1, 10) if y != 5
|
||||
]
|
||||
return payload
|
||||
|
||||
def _multiplayer(index: int) -> dict:
|
||||
payload = _standard_duel(index)
|
||||
third = {
|
||||
"id": "enemy-2",
|
||||
"name": "enemy-2",
|
||||
"health": 65,
|
||||
"length": 5,
|
||||
"head": {"x": 2, "y": 8},
|
||||
"body": [
|
||||
{"x": 2, "y": 8}, {"x": 2, "y": 9}, {"x": 2, "y": 10},
|
||||
{"x": 1, "y": 10}, {"x": 0, "y": 10},
|
||||
],
|
||||
}
|
||||
payload["board"]["snakes"].append(third)
|
||||
return payload
|
||||
|
||||
def _constrictor(index: int) -> dict:
|
||||
payload = _multiplayer(index)
|
||||
payload["game"]["ruleset"] = deepcopy(payload["game"]["ruleset"])
|
||||
payload["game"]["ruleset"]["name"] = "constrictor"
|
||||
payload["board"]["food"] = []
|
||||
return payload
|
||||
|
||||
def _cramped_duel(index: int) -> dict:
|
||||
payload = _standard_duel(index)
|
||||
payload["board"]["width"] = 7
|
||||
payload["board"]["height"] = 7
|
||||
mine = {
|
||||
"id": "me", "name": "me", "health": 72, "length": 7,
|
||||
"head": {"x": 2, "y": 3},
|
||||
"body": [
|
||||
{"x": 2, "y": 3}, {"x": 2, "y": 2}, {"x": 2, "y": 1},
|
||||
{"x": 1, "y": 1}, {"x": 1, "y": 2}, {"x": 1, "y": 3},
|
||||
{"x": 1, "y": 4},
|
||||
],
|
||||
}
|
||||
enemy = {
|
||||
"id": "enemy", "name": "enemy", "health": 72, "length": 7,
|
||||
"head": {"x": 4, "y": 3},
|
||||
"body": [
|
||||
{"x": 4, "y": 3}, {"x": 4, "y": 2}, {"x": 4, "y": 1},
|
||||
{"x": 5, "y": 1}, {"x": 5, "y": 2}, {"x": 5, "y": 3},
|
||||
{"x": 5, "y": 4},
|
||||
],
|
||||
}
|
||||
payload["you"] = mine
|
||||
payload["board"]["snakes"] = [mine, enemy]
|
||||
payload["board"]["food"] = [{"x": 3, "y": 5 + index % 2}]
|
||||
payload["board"]["hazards"] = []
|
||||
return payload
|
||||
|
||||
SCENARIOS = {
|
||||
"duel": _standard_duel,
|
||||
"hazard": _hazard_duel,
|
||||
"multi": _multiplayer,
|
||||
"constrictor": _constrictor,
|
||||
"cramped": _cramped_duel,
|
||||
}
|
||||
|
||||
def synthetic_states(count: int, scenarios: list[str] | None = None) -> list[tuple[dict, dict]]:
|
||||
selected = scenarios or list(SCENARIOS)
|
||||
unknown = set(selected) - set(SCENARIOS)
|
||||
if unknown:
|
||||
raise ValueError(f"Unknown arena scenarios: {', '.join(sorted(unknown))}")
|
||||
states: list[tuple[dict, dict]] = []
|
||||
for index in range(count):
|
||||
scenario = selected[index % len(selected)]
|
||||
states.append(_state(SCENARIOS[scenario](index), scenario, index))
|
||||
return states
|
||||
Reference in New Issue
Block a user