perf(battlesnake): defer ordered lifecycle bookkeeping

- Acknowledge start and end without waiting for persistence or dashboards. - Keep runtime creation and finished-game removal on the response path. - Serialize start, turn, and end bookkeeping per game in Quart tasks. - Capture turn thinking before scheduling deferred persistence. - Defer start metrics, logging, and stale-game pruning. - Run legacy history exports in a worker thread. - Record results and end metrics even when history export fails. - Drain pending bookkeeping before closing storage during shutdown. - Test sub-500 ms responses with slow backends and real SQLite. - Cover event ordering, export failures, and rejected user agents.
2026-11-03 10:38:20 +01:00
parent d4d6375e01
commit 4403978537
6 changed files with 242 additions and 33 deletions
@@ -0,0 +1,141 @@
+import asyncio
+from copy import deepcopy
+import os
+import tempfile
+from time import perf_counter
+import unittest
+from unittest.mock import AsyncMock, Mock, patch
+
+from server.Server import Server
+
+class TestBattlesnakeLifecycle(unittest.IsolatedAsyncioTestCase):
+  def setUp(self):
+    self.server = Server(
+      data_path=os.path.dirname(os.path.dirname(__file__)),
+      snake_type='TemplateSnake', storage_type='LocalStorage',
+      gameplay_db_enabled=False,
+    )
+    snake = {
+      'id': 'snake-1', 'name': 'test', 'health': 100, 'length': 3,
+      'head': {'x': 1, 'y': 1},
+      'body': [{'x': 1, 'y': 1}] * 3,
+    }
+    self.state = {
+      'game': {
+        'id': 'game-1', 'source': 'test', 'map': 'standard', 'timeout': 500,
+        'ruleset': {'name': 'standard', 'version': 'v1'},
+      },
+      'turn': 0, 'you': snake,
+      'board': {'width': 11, 'height': 11, 'food': [], 'hazards': [], 'snakes': [snake]},
+    }
+    self.headers = {'User-Agent': 'BattlesnakeArena'}
+
+  async def test_slow_io_does_not_delay_acknowledgements_or_reorder_events(self):
+    events = []
+    gate = asyncio.Event()
+
+    async def slow_start(*args):
+      await gate.wait()
+      await asyncio.sleep(0.55)
+      events.append('start')
+
+    async def slow_end(*args):
+      await asyncio.sleep(0.55)
+      events.append('end')
+
+    async def record_turn(*args):
+      events.append('turn')
+
+    server = self.server
+    server.logger = Mock()
+    server.logger.info.return_value = None
+    server.gameplay_tracking.record_gameplay_start = slow_start
+    server.gameplay_tracking.record_gameplay_end = slow_end
+    server.gameplay_tracking.record_gameplay_turn_snapshot = record_turn
+    server.dashboard_query.push_dashboard_games_update = AsyncMock()
+    server.dashboard_query.push_dashboard_game_replay_update = AsyncMock()
+    server.metrics_collector.record_game_started = AsyncMock()
+    server.metrics_collector.record_game_end = AsyncMock()
+    client = server.app.test_client()
+
+    started = perf_counter()
+    response = await client.post('/start', json=self.state, headers=self.headers)
+    start_ms = (perf_counter() - started) * 1000
+    self.assertEqual(response.status_code, 200)
+    self.assertEqual(await response.get_data(), b'ok')
+    self.assertLess(start_ms, 500)
+    self.assertIn('game-1', server.game_runtime.running_games)
+    self.assertEqual(events, [])
+
+    # A real move request must not overtake the pending start DB write.
+    response = await client.post('/move', json=self.state, headers=self.headers)
+    self.assertEqual(response.status_code, 200)
+    self.assertIn((await response.get_json())['move'], ('up', 'down', 'left', 'right'))
+    end_state = deepcopy(self.state)
+    end_state['turn'] = 1
+    started = perf_counter()
+    response = await client.post('/end', json=end_state, headers=self.headers)
+    end_ms = (perf_counter() - started) * 1000
+    self.assertEqual(response.status_code, 200)
+    self.assertLess(end_ms, 500)
+    self.assertNotIn('game-1', server.game_runtime.running_games)
+    self.assertEqual(events, [])
+
+    gate.set()
+    await server.game_lifecycle.drain()
+    self.assertEqual(events, ['start', 'turn', 'end'])
+    self.assertEqual(server.game_lifecycle.pending, {})
+    server.metrics_collector.record_game_started.assert_awaited_once_with(1)
+    server.metrics_collector.record_game_end.assert_awaited_once_with(end_state)
+    print(f'Lifecycle HTTP timings with >500 ms I/O: start={start_ms:.1f} ms, end={end_ms:.1f} ms')
+
+  async def test_export_failure_still_records_result_and_cleans_runtime(self):
+    server = self.server
+    server.logger = Mock()
+    server.logger.info.return_value = None
+    server.gameplay_tracking.record_gameplay_start = AsyncMock()
+    server.gameplay_tracking.record_gameplay_end = AsyncMock()
+    server.dashboard_query.push_dashboard_games_update = AsyncMock()
+    server.metrics_collector.record_game_end = AsyncMock()
+    client = server.app.test_client()
+    await client.post('/start', json=self.state, headers=self.headers)
+    await server.game_lifecycle.drain()
+    board = server.game_runtime.running_games['game-1']
+    board.save = AsyncMock(side_effect=OSError('export failed'))
+    server.store_game_state = True
+    with patch('server.services.game_lifecycle.StorageLoader.build', return_value=object):
+      response = await client.post('/end', json=self.state, headers=self.headers)
+      await server.game_lifecycle.drain()
+    self.assertEqual(response.status_code, 200)
+    self.assertNotIn('game-1', server.game_runtime.running_games)
+    board.save.assert_awaited_once()
+    server.gameplay_tracking.record_gameplay_end.assert_awaited_once()
+    server.metrics_collector.record_game_end.assert_awaited_once()
+
+  async def test_real_sqlite_lifecycle_persists_after_acknowledgements(self):
+    with tempfile.TemporaryDirectory() as folder:
+      server = Server(
+        data_path=folder, snake_type='TemplateSnake', storage_type='LocalStorage',
+        gameplay_db_path=os.path.join(folder, 'gameplay.sqlite3'),
+      )
+      server.logger = Mock()
+      server.logger.info.return_value = None
+      async with server.app.test_app():
+        client = server.app.test_client()
+        timings = {}
+        for endpoint in ('start', 'end'):
+          started = perf_counter()
+          response = await client.post('/' + endpoint, json=self.state, headers=self.headers)
+          timings[endpoint] = (perf_counter() - started) * 1000
+          self.assertEqual(response.status_code, 200)
+          self.assertLess(timings[endpoint], 500)
+        await server.game_lifecycle.drain()
+        replay = await server.gameplay_database.get_game_replay('game-1')
+        self.assertIsNotNone(replay)
+        self.assertNotEqual(replay['game']['status'], 'running')
+        print(f'Real SQLite HTTP timings: {timings}')
+
+  async def test_invalid_agent_does_not_schedule_work(self):
+    response = await self.server.app.test_client().post('/start', json=self.state)
+    self.assertEqual(response.status_code, 404)
+    self.assertEqual(self.server.game_lifecycle.pending, {})