perf(battlesnake): defer ordered lifecycle bookkeeping

- Acknowledge start and end without waiting for persistence or dashboards.
- Keep runtime creation and finished-game removal on the response path.
- Serialize start, turn, and end bookkeeping per game in Quart tasks.
- Capture turn thinking before scheduling deferred persistence.
- Defer start metrics, logging, and stale-game pruning.
- Run legacy history exports in a worker thread.
- Record results and end metrics even when history export fails.
- Drain pending bookkeeping before closing storage during shutdown.
- Test sub-500 ms responses with slow backends and real SQLite.
- Cover event ordering, export failures, and rejected user agents.

server/Server.py

@@ -21,6 +21,7 @@ from server.blueprints import (
   create_metrics_blueprint,
   create_dashboard_blueprint,
 )
+from server.services.game_lifecycle import GameLifecycleService
 from server.services import (
   DashboardEventsService,
   DashboardWebSocketHub,
@@ -106,6 +107,7 @@ class Server:
 
     self.app = Quart('Battlesnake', template_folder=os.path.join(data_path, 'templates', 'side'), static_folder=os.path.join(data_path, 'templates', 'files'))
 
+    self.game_lifecycle = GameLifecycleService(self)
     self.app.register_blueprint(create_battlesnake_blueprint(self))
     self.app.register_blueprint(create_metrics_blueprint(self))
     self.app.register_blueprint(create_dashboard_blueprint(self))
@@ -143,6 +145,7 @@ class Server:
 
     @self.app.after_serving
     async def shutdown_state_storage():
+      await self.game_lifecycle.drain()
       await self.dashboard_events_service.stop_listener()
       await self.metrics_collector.close()
       if self.gameplay_database is not None:

server/blueprints/battlesnake.py

@@ -5,7 +5,6 @@ from quart import Blueprint, request, jsonify
 
 from quart_common.web.decorators import require_user_agent
 from quart_common.web.logger import await_log
-from server.database import StorageLoader
 from snakes import DEFAULT_SNAKE_CONFIG
 from server.GameBoard import GameBoard
 from server.Files import read_file
@@ -47,14 +46,13 @@ def create_battlesnake_blueprint(server:'Server') -> Blueprint:
   @require_user_agent(("BattlesnakeEngine", "BattlesnakeArena"), abort_code=404)
   async def on_start():
     server.metrics_collector.record_http_request('start')
-    await server.game_runtime.prune_stale_games()
     game_state = await request.get_json()
-    game_board = await server.game_runtime.create_game_board(game_state)
-    await server.gameplay_tracking.record_gameplay_start(game_state, game_board)
-    await server.dashboard_query.push_dashboard_games_update(
-      game_state, trigger='game_started',
+    game_board = await server.game_runtime.create_game_board(game_state, publish_metrics=False)
+    active_count = len(server.game_runtime.running_games)
+    server.game_lifecycle.submit(
+      game_state['game']['id'],
+      lambda: server.game_lifecycle.started(game_state, game_board, active_count),
     )
-    await await_log(server.logger.info(f'GAME START: {game_state['game']}'))
     return 'ok'
 
   @blueprint.post('/move')
@@ -79,8 +77,10 @@ def create_battlesnake_blueprint(server:'Server') -> Blueprint:
     except TimeoutError:
       await await_log(server.logger.warning(f'MOVE TIMEOUT: turn={game_state.get("turn")}, game={game_id}, returning fallback {next_move!r}'))
 
-    await server.gameplay_tracking.record_gameplay_turn(game_state, next_move, game_board)
-    await server.dashboard_query.push_dashboard_game_replay_update(game_id)
+    thinking = server.gameplay_tracking._extract_latest_snake_thinking(game_board)
+    server.game_lifecycle.submit(
+      game_id, lambda: server.game_lifecycle.turn(game_state, next_move, thinking),
+    )
     elapsed_ms = (time.perf_counter() - move_started) * 1000.0
     await server.metrics_collector.record_move(next_move, elapsed_ms)
 
@@ -93,29 +93,16 @@ def create_battlesnake_blueprint(server:'Server') -> Blueprint:
   @require_user_agent(("BattlesnakeEngine", "BattlesnakeArena"), abort_code=404)
   async def on_end():
     server.metrics_collector.record_http_request('end')
-    await server.game_runtime.prune_stale_games()
     game_state = await request.get_json()
-    if server.store_game_state:
-      game_board = cast(GameBoard, await server.game_runtime.get_game_board(game_state, end=True))
-      if server.check_tls_security:
-        await game_board.save(
-          StorageLoader.build(server.storage_type),
-          file_path=os.path.join(server.data_path, 'data'),
-          database=os.getenv('EDGEDB_DATABASE', None),
-          tls_security=None,
-        )
-      else:
-        await game_board.save(
-          StorageLoader.build(server.storage_type),
-          file_path=os.path.join(server.data_path, 'data'),
-          database=os.getenv('EDGEDB_DATABASE', None),
-        )
-
-    await server.gameplay_tracking.record_gameplay_end(game_state)
-    await server.dashboard_query.push_dashboard_games_update(game_state)
-    await await_log(server.logger.info(f'GAME ENDED: Winner is {[x['name'] for x in game_state['board']['snakes']]}'))
+    game_id = game_state['game']['id']
+    game_board = server.game_runtime.running_games.get(game_id)
+    if game_board is not None:
+      game_board.read_game_data(game_state)
+      game_board.end_game(game_state)
     await server.game_runtime.delete_game_board(game_state)
-    await server.metrics_collector.record_game_end(game_state)
+    server.game_lifecycle.submit(
+      game_id, lambda: server.game_lifecycle.ended(game_state, game_board),
+    )
     return 'ok'
 
   # @blueprint.get('/cleanup')

server/services/game_lifecycle.py

@@ -0,0 +1,75 @@
+"""Ordered, application-managed game bookkeeping outside HTTP response budgets."""
+
+import asyncio
+from collections.abc import Awaitable, Callable
+import os
+
+from quart_common.web.logger import await_log
+from server.database import StorageLoader
+
+class GameLifecycleService:
+  def __init__(self, server):
+    self.server = server
+    self.pending: dict[str, asyncio.Future] = {}
+
+  def submit(self, game_id:str, operation:Callable[[], Awaitable[None]]) -> None:
+    previous = self.pending.get(game_id)
+    completed = asyncio.get_running_loop().create_future()
+    self.pending[game_id] = completed
+
+    async def run():
+      try:
+        if previous is not None:
+          await asyncio.shield(previous)
+        await operation()
+      except Exception:
+        self.server.app.logger.exception('Game bookkeeping failed for %s', game_id)
+      finally:
+        if not completed.done():
+          completed.set_result(None)
+        if self.pending.get(game_id) is completed:
+          self.pending.pop(game_id, None)
+
+    self.server.app.add_background_task(run)
+
+  async def drain(self) -> None:
+    while self.pending:
+      await asyncio.gather(*(asyncio.shield(task) for task in list(self.pending.values())))
+
+  async def started(self, game_state:dict, game_board, active_count:int) -> None:
+    server = self.server
+    await server.metrics_collector.record_game_started(active_count)
+    await server.gameplay_tracking.record_gameplay_start(game_state, game_board)
+    await server.dashboard_query.push_dashboard_games_update(game_state, trigger='game_started')
+    await await_log(server.logger.info(f'GAME START: {game_state["game"]}'))
+    await server.game_runtime.prune_stale_games()
+
+  async def ended(self, game_state:dict, game_board) -> None:
+    server = self.server
+    try:
+      if server.store_game_state and game_board is not None:
+        kwargs = {
+          'file_path': os.path.join(server.data_path, 'data'),
+          'database': os.getenv('EDGEDB_DATABASE'),
+        }
+        if server.check_tls_security:
+          kwargs['tls_security'] = None
+        # Legacy exporters include synchronous serialization and database calls.
+        # Keep those off the request event loop as well as off the response path.
+        def export():
+          asyncio.run(game_board.save(StorageLoader.build(server.storage_type), **kwargs))
+
+        await asyncio.to_thread(export)
+    finally:
+      # A history export failure must not suppress the game result or metrics.
+      await server.gameplay_tracking.record_gameplay_end(game_state)
+      await server.metrics_collector.record_game_end(game_state)
+      await server.dashboard_query.push_dashboard_games_update(game_state)
+      await await_log(server.logger.info(
+        f'GAME ENDED: Winner is {[snake["name"] for snake in game_state["board"]["snakes"]]}'
+      ))
+
+  async def turn(self, game_state:dict, move:str|None, thinking:dict|None) -> None:
+    server = self.server
+    await server.gameplay_tracking.record_gameplay_turn_snapshot(game_state, move, thinking)
+    await server.dashboard_query.push_dashboard_game_replay_update(game_state['game']['id'])

server/services/game_runtime.py

@@ -18,7 +18,7 @@ class GameRuntimeService:
   def attach_metrics_collector(self, metrics_collector:MetricsCollector) -> None:
     self.metrics_collector = metrics_collector
 
-  async def create_game_board(self, game_state:dict) -> GameBoard:
+  async def create_game_board(self, game_state:dict, publish_metrics:bool=True) -> GameBoard:
     game_id = game_state['game']['id']
     new_game_board = GameBoard(
       game_id=game_id,
@@ -34,7 +34,7 @@ class GameRuntimeService:
     self.running_games[game_id] = new_game_board
     self.game_move_counts[game_id] = 0
     self.game_last_seen_unix[game_id] = int(time.time())
-    if self.metrics_collector is not None:
+    if publish_metrics and self.metrics_collector is not None:
       await self.metrics_collector.record_game_started(len(self.game_last_seen_unix))
     return new_game_board
 

server/services/gameplay_tracking.py

@@ -23,10 +23,13 @@ class GameplayTrackingService:
       await await_log(self.logger.warning(f"Gameplay DB start record failed:{error}"))
 
   async def record_gameplay_turn(self, game_state:dict, my_move:str, game_board:GameBoard) -> None:
+    thinking = self._extract_latest_snake_thinking(game_board)
+    await self.record_gameplay_turn_snapshot(game_state, my_move, thinking)
+
+  async def record_gameplay_turn_snapshot(self, game_state:dict, my_move:str|None, thinking:dict|None) -> None:
     if self.gameplay_database is None:
       return
     try:
-      thinking = self._extract_latest_snake_thinking(game_board)
       await self.gameplay_database.record_turn(game_state, my_move, thinking)
     except Exception as error:
       await await_log(self.logger.warning(f"Gameplay DB turn record failed:{error}"))