allow to send metrics to memory or redis

server/Server.py

@@ -1,15 +1,18 @@
 from server.Files import read_file
+
 from server.game_board_stats import GameBoardStoreBuilder
 from server.GameBoard import GameBoard
+
 from snakes import SnakeBuilder
 from quart_common.web.logger import await_log
 from quart_common.web.logger import build_logger
-from typing import cast
 
+from server.metrics.MetricsManager import MetricsManager
 from server.storage.StorageLoader import StorageLoader
 
 from quart import Quart, request, jsonify
 import logging, json, os, re, time
+from typing import cast
 
 class Server:
   default_snake_config = {
@@ -21,7 +24,7 @@ class Server:
     'version': '1.0.0',
   }
 
-  def __init__(self, data_path:str, snake_type:str, storage_type:str, debug:bool=False, check_tls_security:bool=False, game_state_backend:str='memory', game_state_redis_url:str='redis://localhost:6379/0', game_state_ttl_sec:int=900, game_state_local_cache:bool=True):
+  def __init__(self, data_path:str, snake_type:str, storage_type:str, debug:bool=False, check_tls_security:bool=False, game_state_backend:str='memory', game_state_redis_url:str='redis://localhost:6379/0', game_state_ttl_sec:int=900, game_state_local_cache:bool=True, metrics_backend:str='memory', metrics_redis_url:str='redis://localhost:6379/0', metrics_ttl_sec:int=None):
     self.debug = debug
     self.snake_type = snake_type
     self.storage_type = storage_type
@@ -38,6 +41,13 @@ class Server:
       redis_url=game_state_redis_url,
       ttl_seconds=game_state_ttl_sec,
     )
+    self.metrics_backend = (metrics_backend or 'memory').strip().lower()
+    self.metrics_manager = MetricsManager(
+      backend=self.metrics_backend,
+      redis_url=metrics_redis_url,
+      ttl_seconds=metrics_ttl_sec,
+      key_prefix=os.environ.get('METRICS_REDIS_KEY_PREFIX', 'snake:metrics:worker'),
+    )
 
     self.running_games:dict[str, GameBoard] = {}
     self.game_move_counts:dict[str, int] = {}
@@ -74,6 +84,7 @@ class Server:
       'last_move_unix': 0,
       'games_stuck_removed': 0,
       'game_state_local_cache_enabled': bool(self.game_state_local_cache),
+      'metrics_backend': self.metrics_backend,
     }
     self.logger = build_logger('Battlesnake', debug_env_var='DEBUG_SERVER')
     self.snake_version = self._get_snake_version()
@@ -163,6 +174,7 @@ class Server:
     @self.app.after_serving
     async def shutdown_state_storage():
       await self.game_state_store.close()
+      await self._close_metrics_store()
 
     @self.app.get('/cleanup')
     async def cleanup():
@@ -171,12 +183,13 @@ class Server:
 
     @self.app.get('/metrics')
     async def metrics():
-      return jsonify(self._build_metrics())
+      return jsonify(await self._build_metrics())
 
     @self.app.get('/metrics/prometheus')
     async def metrics_prometheus():
+      snapshot = await self._build_metrics()
       return (
-        self._build_prometheus_metrics(),
+        self._build_prometheus_metrics(snapshot),
         200,
         {'Content-Type': 'text/plain; version=0.0.4; charset=utf-8'},
       )
@@ -338,12 +351,12 @@ class Server:
     else:
       self.metrics['losses'] += 1
 
-  def _build_metrics(self) -> dict:
+  def _build_local_metrics(self) -> dict:
     games_ended = self.metrics['games_ended']
     total_moves = self.metrics['total_moves']
     avg_turns = self.metrics['total_turns'] / games_ended if games_ended else 0.0
     win_rate = self.metrics['wins'] / games_ended if games_ended else 0.0
-    avg_move_ms = self.metrics['move_response_time_ms_total'] / total_moves if total_moves else 0.0
+    avg_move_ms = (self.metrics['move_response_time_ms_total'] / total_moves if total_moves else 0.0)
 
     now = int(time.time())
     oldest_active_age = 0
@@ -369,13 +382,16 @@ class Server:
       'active_games_stale': stale_candidates,
     }
 
-  def _record_http_request(self, endpoint:str):
+  def _record_http_request(self, endpoint: str):
     self.metrics['http_requests_total'] += 1
     endpoint_counts = self.metrics['http_requests_by_endpoint']
     endpoint_counts[endpoint] = endpoint_counts.get(endpoint, 0) + 1
 
-  def _build_prometheus_metrics(self) -> str:
-    snapshot = self._build_metrics()
+  async def _build_metrics(self) -> dict:
+    local_snapshot = self._build_local_metrics()
+    return await self.metrics_manager.snapshot(local_snapshot)
+
+  def _build_prometheus_metrics(self, snapshot: dict) -> str:
     lines = [
       '# HELP snake_games_started_total Total games started by snake server.',
       '# TYPE snake_games_started_total counter',
@@ -466,3 +482,6 @@ class Server:
       lines.append(f'snake_moves_by_direction_total{{direction="{direction}"}} {count}')
 
     return '\n'.join(lines) + '\n'
+
+  async def _close_metrics_store(self) -> None:
+    await self.metrics_manager.close()