feat: add Zenith strategy and move latency tracing

- Register Zenith 1.1 with balanced duel verdicts and forced-outcome overrides.
- Prioritize permanent-trail space in Constrictor and health-aware hazard routes.
- Deduct executor queue delays and return fallback directions on move timeouts.
- Correlate response headers, replay timings, and sampled structured trace logs.
- Move shared metrics publication off the response path without losing local counts.
- Allow configurable tournament opponents and add a psql helper recipe.
- Preserve empty latency values through compact replay encoding and migration.
- Update Arena game links and use default assets for missing customizations.
- Cover strategy, HTTP lifecycle, tracing, replay codecs, and dashboard behavior.
- Validate with 260 passing Python tests, 4 skips, and 2 passing JavaScript tests.

server/blueprints/battlesnake.py

@@ -8,6 +8,8 @@ from quart_common.web.logger import await_log
 from snakes import DEFAULT_SNAKE_CONFIG
 from server.GameBoard import GameBoard
 from server.Files import read_file
+from server.services.move_safety import fallback_move, run_budgeted_move
+from server.services.move_trace import MoveTrace
 
 lazy from server.Server import Server
 
@@ -58,36 +60,68 @@ def create_battlesnake_blueprint(server:'Server') -> Blueprint:
   @blueprint.post('/move')
   @require_user_agent(("BattlesnakeEngine", "BattlesnakeArena"), abort_code=404)
   async def on_move():
+    move_started = time.perf_counter()
     server.metrics_collector.record_http_request('move')
     game_state = await request.get_json()
-    move_started = time.perf_counter()
+    trace = MoveTrace(game_state['game']['id'], game_state.get('turn'))
+    trace.set(parse_ms=round((time.perf_counter() - move_started) * 1000, 3), timed_out=False)
 
     game_id = game_state['game']['id']
     timeout_ms = int(game_state.get('game', {}).get('timeout', 500))
     budget_sec = max(0.05, (timeout_ms - 50) / 1000.0)
 
-    next_move = None
+    next_move = fallback_move(game_state)
     game_board = None
 
     try:
       async with asyncio.timeout(budget_sec):
-        game_board = cast(GameBoard, await server.game_runtime.get_game_board(game_state))
+        with trace.stage('runtime'):
+          game_board = cast(GameBoard, await server.game_runtime.get_game_board(game_state))
         loop = asyncio.get_running_loop()
-        next_move = await loop.run_in_executor(None, game_board.snake_neat_make_a_move)
+        submitted = time.perf_counter()
+        next_move = await loop.run_in_executor(
+          None, run_budgeted_move, game_board, move_started + budget_sec, next_move, trace, submitted,
+        )
     except TimeoutError:
-      await await_log(server.logger.warning(f'MOVE TIMEOUT: turn={game_state.get("turn")}, game={game_id}, returning fallback {next_move!r}'))
+      trace.set(timed_out=True)
 
-    thinking = server.gameplay_tracking._extract_latest_snake_thinking(game_board)
-    server.game_lifecycle.submit(
-      game_id, lambda: server.game_lifecycle.turn(game_state, next_move, thinking),
-    )
-    elapsed_ms = (time.perf_counter() - move_started) * 1000.0
-    await server.metrics_collector.record_move(next_move, elapsed_ms)
+    # Never persist another turn's history when the worker has timed out.
+    thinking = None if trace.snapshot()['timed_out'] else server.gameplay_tracking._extract_latest_snake_thinking(game_board)
+    thinking = dict(thinking or {})
+    trace.set(handler_ms=round((time.perf_counter() - move_started) * 1000, 3),
+              move=next_move, budget_ms=round(budget_sec * 1000, 3))
+    thinking['move_trace'] = trace.snapshot()
+    server.metrics_collector.record_move_local(next_move, trace.snapshot()['handler_ms'])
+
+    bookkeeping_submitted = time.perf_counter()
+
+    async def finish_move():
+      trace.set(bookkeeping_queue_ms=round((time.perf_counter() - bookkeeping_submitted) * 1000, 3))
+      try:
+        try:
+          with trace.stage('metrics_publish'):
+            await server.metrics_collector.publish_move_metrics()
+        except Exception:
+          trace.set(metrics_publish_failed=True)
+          server.app.logger.exception('Move metrics publish failed; trace=%s', trace.id)
+        with trace.stage('persistence'):
+          thinking['move_trace'] = trace.snapshot()
+          await server.gameplay_tracking.record_gameplay_turn_snapshot(game_state, next_move, thinking)
+        with trace.stage('dashboard_publish'):
+          await server.dashboard_query.push_dashboard_game_replay_update(game_id)
+      finally:
+        trace.log(server.app.logger)
+
+    server.game_lifecycle.submit(game_id, finish_move)
 
     if server.debug:
       await await_log(server.logger.debug(f'TURN: {game_state['turn']:3}, MOVE: {next_move:5}'))
 
-    return {'move': next_move}
+    with trace.stage('response_encode'):
+      response = jsonify(move=next_move)
+    trace.set(handler_ms=round((time.perf_counter() - move_started) * 1000, 3))
+    response.headers['X-Move-Trace-ID'] = trace.id
+    return response
 
   @blueprint.post('/end')
   @require_user_agent(("BattlesnakeEngine", "BattlesnakeArena"), abort_code=404)