perf(battlesnake): defer ordered lifecycle bookkeeping

- Acknowledge start and end without waiting for persistence or dashboards.
- Keep runtime creation and finished-game removal on the response path.
- Serialize start, turn, and end bookkeeping per game in Quart tasks.
- Capture turn thinking before scheduling deferred persistence.
- Defer start metrics, logging, and stale-game pruning.
- Run legacy history exports in a worker thread.
- Record results and end metrics even when history export fails.
- Drain pending bookkeeping before closing storage during shutdown.
- Test sub-500 ms responses with slow backends and real SQLite.
- Cover event ordering, export failures, and rejected user agents.

server/services/game_lifecycle.py

@@ -0,0 +1,75 @@
+"""Ordered, application-managed game bookkeeping outside HTTP response budgets."""
+
+import asyncio
+from collections.abc import Awaitable, Callable
+import os
+
+from quart_common.web.logger import await_log
+from server.database import StorageLoader
+
+class GameLifecycleService:
+  def __init__(self, server):
+    self.server = server
+    self.pending: dict[str, asyncio.Future] = {}
+
+  def submit(self, game_id:str, operation:Callable[[], Awaitable[None]]) -> None:
+    previous = self.pending.get(game_id)
+    completed = asyncio.get_running_loop().create_future()
+    self.pending[game_id] = completed
+
+    async def run():
+      try:
+        if previous is not None:
+          await asyncio.shield(previous)
+        await operation()
+      except Exception:
+        self.server.app.logger.exception('Game bookkeeping failed for %s', game_id)
+      finally:
+        if not completed.done():
+          completed.set_result(None)
+        if self.pending.get(game_id) is completed:
+          self.pending.pop(game_id, None)
+
+    self.server.app.add_background_task(run)
+
+  async def drain(self) -> None:
+    while self.pending:
+      await asyncio.gather(*(asyncio.shield(task) for task in list(self.pending.values())))
+
+  async def started(self, game_state:dict, game_board, active_count:int) -> None:
+    server = self.server
+    await server.metrics_collector.record_game_started(active_count)
+    await server.gameplay_tracking.record_gameplay_start(game_state, game_board)
+    await server.dashboard_query.push_dashboard_games_update(game_state, trigger='game_started')
+    await await_log(server.logger.info(f'GAME START: {game_state["game"]}'))
+    await server.game_runtime.prune_stale_games()
+
+  async def ended(self, game_state:dict, game_board) -> None:
+    server = self.server
+    try:
+      if server.store_game_state and game_board is not None:
+        kwargs = {
+          'file_path': os.path.join(server.data_path, 'data'),
+          'database': os.getenv('EDGEDB_DATABASE'),
+        }
+        if server.check_tls_security:
+          kwargs['tls_security'] = None
+        # Legacy exporters include synchronous serialization and database calls.
+        # Keep those off the request event loop as well as off the response path.
+        def export():
+          asyncio.run(game_board.save(StorageLoader.build(server.storage_type), **kwargs))
+
+        await asyncio.to_thread(export)
+    finally:
+      # A history export failure must not suppress the game result or metrics.
+      await server.gameplay_tracking.record_gameplay_end(game_state)
+      await server.metrics_collector.record_game_end(game_state)
+      await server.dashboard_query.push_dashboard_games_update(game_state)
+      await await_log(server.logger.info(
+        f'GAME ENDED: Winner is {[snake["name"] for snake in game_state["board"]["snakes"]]}'
+      ))
+
+  async def turn(self, game_state:dict, move:str|None, thinking:dict|None) -> None:
+    server = self.server
+    await server.gameplay_tracking.record_gameplay_turn_snapshot(game_state, move, thinking)
+    await server.dashboard_query.push_dashboard_game_replay_update(game_state['game']['id'])

server/services/game_runtime.py

@@ -18,7 +18,7 @@ class GameRuntimeService:
   def attach_metrics_collector(self, metrics_collector:MetricsCollector) -> None:
     self.metrics_collector = metrics_collector
 
-  async def create_game_board(self, game_state:dict) -> GameBoard:
+  async def create_game_board(self, game_state:dict, publish_metrics:bool=True) -> GameBoard:
     game_id = game_state['game']['id']
     new_game_board = GameBoard(
       game_id=game_id,
@@ -34,7 +34,7 @@ class GameRuntimeService:
     self.running_games[game_id] = new_game_board
     self.game_move_counts[game_id] = 0
     self.game_last_seen_unix[game_id] = int(time.time())
-    if self.metrics_collector is not None:
+    if publish_metrics and self.metrics_collector is not None:
       await self.metrics_collector.record_game_started(len(self.game_last_seen_unix))
     return new_game_board
 

server/services/gameplay_tracking.py

@@ -23,10 +23,13 @@ class GameplayTrackingService:
       await await_log(self.logger.warning(f"Gameplay DB start record failed:{error}"))
 
   async def record_gameplay_turn(self, game_state:dict, my_move:str, game_board:GameBoard) -> None:
+    thinking = self._extract_latest_snake_thinking(game_board)
+    await self.record_gameplay_turn_snapshot(game_state, my_move, thinking)
+
+  async def record_gameplay_turn_snapshot(self, game_state:dict, my_move:str|None, thinking:dict|None) -> None:
     if self.gameplay_database is None:
       return
     try:
-      thinking = self._extract_latest_snake_thinking(game_board)
       await self.gameplay_database.record_turn(game_state, my_move, thinking)
     except Exception as error:
       await await_log(self.logger.warning(f"Gameplay DB turn record failed:{error}"))