diff --git a/grafana/snake-metrics-dashboard.json b/grafana/snake-metrics-dashboard.json new file mode 100644 index 0000000..2c8fe15 --- /dev/null +++ b/grafana/snake-metrics-dashboard.json @@ -0,0 +1,917 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "snake_win_rate", + "legendFormat": "Win Rate", + "range": true, + "refId": "A" + } + ], + "title": "Win Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "snake_active_games", + "legendFormat": "Active Games", + "range": true, + "refId": "A" + } + ], + "title": "Active Games", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "snake_avg_turns_per_game", + "legendFormat": "Avg Turns", + "range": true, + "refId": "A" + } + ], + "title": "Avg Turns / Game", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "snake_max_turn", + "legendFormat": "Max Turn", + "range": true, + "refId": "A" + } + ], + "title": "Max Turn", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "increase(snake_wins_total[$__range])", + "legendFormat": "Wins", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "increase(snake_losses_total[$__range])", + "legendFormat": "Losses", + "range": true, + "refId": "B" + } + ], + "title": "Wins vs Losses (Range)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "increase(snake_moves_total[$__rate_interval])", + "legendFormat": "Moves", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "increase(snake_games_started_total[$__rate_interval])", + "legendFormat": "Games Started", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "expr": "increase(snake_games_ended_total[$__rate_interval])", + "legendFormat": "Games Ended", + "range": true, + "refId": "C" + } + ], + "title": "Activity (Interval Increases)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 12 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "snake_avg_move_response_ms", + "legendFormat": "Avg Move ms", + "range": true, + "refId": "A" + } + ], + "title": "Avg Move Response", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 12 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "snake_games_autocreated_total", + "legendFormat": "Auto-created", + "range": true, + "refId": "A" + } + ], + "title": "Auto-created Games", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 12 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "snake_active_games_peak", + "legendFormat": "Active Peak", + "range": true, + "refId": "A" + } + ], + "title": "Active Games Peak", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 12 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "increase(snake_http_requests_total[$__range])", + "legendFormat": "HTTP req", + "range": true, + "refId": "A" + } + ], + "title": "HTTP Requests (Range)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "sum by (endpoint) (rate(snake_http_requests_by_endpoint_total[$__rate_interval]))", + "legendFormat": "{{endpoint}}", + "range": true, + "refId": "A" + } + ], + "title": "HTTP Requests by Endpoint (Rate)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "editorMode": "code", + "expr": "sum by (direction) (rate(snake_moves_by_direction_total[$__rate_interval]))", + "legendFormat": "{{direction}}", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "snake_avg_move_response_ms", + "legendFormat": "avg move ms", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "expr": "snake_move_response_ms_max", + "legendFormat": "max move ms", + "range": true, + "refId": "C" + } + ], + "title": "Move Directions + Move Latency", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "battlesnake", + "snake", + "prometheus" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Snake Performance", + "uid": "snake-performance", + "version": 2, + "weekStart": "" +} diff --git a/server/Server.py b/server/Server.py index 1377563..d72dc59 100644 --- a/server/Server.py +++ b/server/Server.py @@ -8,7 +8,7 @@ from typing import cast from server.storage.StorageLoader import StorageLoader from quart import Quart, request, jsonify -import logging, json, os, re +import logging, json, os, re, time class Server: default_snake_config = { @@ -41,6 +41,30 @@ class Server: 'total_moves': 0, 'total_turns': 0, 'max_turn': 0, + 'active_games_peak': 0, + 'games_autocreated': 0, + 'http_requests_total': 0, + 'http_requests_by_endpoint': { + 'info': 0, + 'start': 0, + 'move': 0, + 'end': 0, + 'cleanup': 0, + 'metrics': 0, + 'metrics_prometheus': 0, + }, + 'move_direction_counts': { + 'up': 0, + 'down': 0, + 'left': 0, + 'right': 0, + 'unknown': 0, + }, + 'move_response_time_ms_total': 0.0, + 'move_response_time_ms_max': 0.0, + 'last_game_start_unix': 0, + 'last_game_end_unix': 0, + 'last_move_unix': 0, } self.logger = build_logger('Battlesnake', debug_env_var='DEBUG_SERVER') self.snake_version = self._get_snake_version() @@ -52,6 +76,7 @@ class Server: # TIP: If you open your Battlesnake URL in a browser you should see this data @self.app.get('/') async def on_info(): + self._record_http_request('info') snake_config = await self._read_json_config_or_create() await await_log(self.logger.info(f'INFO Snake: {snake_config}')) @@ -60,6 +85,7 @@ class Server: # start is called when your Battlesnake begins a game @self.app.post('/start') async def on_start(): + self._record_http_request('start') game_state = await request.get_json() await self._create_game_board(game_state) await await_log(self.logger.info(f'GAME START: {game_state['game']}')) @@ -68,9 +94,24 @@ class Server: # move is called when your Battlesnake game is running game @self.app.post('/move') async def on_move(): + self._record_http_request('move') game_state = await request.get_json() + move_started = time.perf_counter() game_board = await self._get_game_board(game_state) next_move = game_board.snake_neat_make_a_move() + elapsed_ms = (time.perf_counter() - move_started) * 1000.0 + self.metrics['move_response_time_ms_total'] += elapsed_ms + self.metrics['move_response_time_ms_max'] = max( + self.metrics['move_response_time_ms_max'], + elapsed_ms, + ) + + move_counts = self.metrics['move_direction_counts'] + if next_move in move_counts: + move_counts[next_move] += 1 + else: + move_counts['unknown'] += 1 + self.metrics['last_move_unix'] = int(time.time()) if self.debug: await await_log(self.logger.debug(f'TURN: {game_state['turn']:3}, MOVE: {next_move:5}')) @@ -80,6 +121,7 @@ class Server: # end is called when your Battlesnake finishes a game @self.app.post('/end') async def on_end(): + self._record_http_request('end') game_state = await request.get_json() if self.store_game_state: game_board = await self._get_game_board(game_state, end=True) @@ -182,6 +224,11 @@ class Server: self.running_games[game_id] = new_game_board self.game_move_counts[game_id] = 0 self.metrics['games_started'] += 1 + self.metrics['active_games_peak'] = max( + self.metrics['active_games_peak'], + len(self.running_games), + ) + self.metrics['last_game_start_unix'] = int(time.time()) return new_game_board def _delete_game_board(self, game_state:dict): @@ -195,6 +242,7 @@ class Server: game_board = self.running_games[game_id] except KeyError: game_board = await self._create_game_board(game_state) + self.metrics['games_autocreated'] += 1 if not end: self.metrics['total_moves'] += 1 @@ -216,6 +264,7 @@ class Server: def _record_game_end(self, game_state: dict): self.metrics['games_ended'] += 1 + self.metrics['last_game_end_unix'] = int(time.time()) final_turn = int(game_state.get('turn', 0)) self.metrics['total_turns'] += final_turn @@ -232,8 +281,10 @@ class Server: def _build_metrics(self) -> dict: games_ended = self.metrics['games_ended'] + total_moves = self.metrics['total_moves'] avg_turns = self.metrics['total_turns'] / games_ended if games_ended else 0.0 win_rate = self.metrics['wins'] / games_ended if games_ended else 0.0 + avg_move_ms = self.metrics['move_response_time_ms_total'] / total_moves if total_moves else 0.0 return { **self.metrics, @@ -241,8 +292,16 @@ class Server: 'tracked_games': len(self.game_move_counts), 'avg_turns_per_game': round(avg_turns, 2), 'win_rate': round(win_rate, 4), + 'avg_move_response_ms': round(avg_move_ms, 2), + 'http_requests_by_endpoint': dict(self.metrics['http_requests_by_endpoint']), + 'move_direction_counts': dict(self.metrics['move_direction_counts']), } + def _record_http_request(self, endpoint:str): + self.metrics['http_requests_total'] += 1 + endpoint_counts = self.metrics['http_requests_by_endpoint'] + endpoint_counts[endpoint] = endpoint_counts.get(endpoint, 0) + 1 + def _build_prometheus_metrics(self) -> str: snapshot = self._build_metrics() lines = [ @@ -273,11 +332,57 @@ class Server: '# HELP snake_max_turn Highest final turn seen in an ended game.', '# TYPE snake_max_turn gauge', f'snake_max_turn {snapshot['max_turn']}', + '# HELP snake_active_games_peak Highest active game count observed.', + '# TYPE snake_active_games_peak gauge', + f'snake_active_games_peak {snapshot['active_games_peak']}', + '# HELP snake_games_autocreated_total Games created on /move or /end due to missing /start.', + '# TYPE snake_games_autocreated_total counter', + f'snake_games_autocreated_total {snapshot['games_autocreated']}', + '# HELP snake_http_requests_total Total HTTP requests handled by this process.', + '# TYPE snake_http_requests_total counter', + f'snake_http_requests_total {snapshot['http_requests_total']}', + '# HELP snake_move_response_ms_total Total move endpoint compute time in milliseconds.', + '# TYPE snake_move_response_ms_total counter', + f"snake_move_response_ms_total {round(snapshot['move_response_time_ms_total'], 3)}", + '# HELP snake_move_response_ms_max Maximum move endpoint compute time in milliseconds.', + '# TYPE snake_move_response_ms_max gauge', + f"snake_move_response_ms_max {round(snapshot['move_response_time_ms_max'], 3)}", '# HELP snake_avg_turns_per_game Average final turn per ended game.', '# TYPE snake_avg_turns_per_game gauge', f'snake_avg_turns_per_game {snapshot['avg_turns_per_game']}', + '# HELP snake_avg_move_response_ms Average move endpoint compute time in milliseconds.', + '# TYPE snake_avg_move_response_ms gauge', + f'snake_avg_move_response_ms {snapshot['avg_move_response_ms']}', '# HELP snake_win_rate Win ratio from ended games (0.0 - 1.0).', '# TYPE snake_win_rate gauge', f'snake_win_rate {snapshot['win_rate']}', + '# HELP snake_last_game_start_unix Unix timestamp of most recent /start request.', + '# TYPE snake_last_game_start_unix gauge', + f'snake_last_game_start_unix {snapshot['last_game_start_unix']}', + '# HELP snake_last_game_end_unix Unix timestamp of most recent /end request.', + '# TYPE snake_last_game_end_unix gauge', + f'snake_last_game_end_unix {snapshot['last_game_end_unix']}', + '# HELP snake_last_move_unix Unix timestamp of most recent /move response.', + '# TYPE snake_last_move_unix gauge', + f'snake_last_move_unix {snapshot['last_move_unix']}', ] + + lines.extend([ + '# HELP snake_http_requests_by_endpoint_total Requests served grouped by endpoint.', + '# TYPE snake_http_requests_by_endpoint_total counter', + ]) + for endpoint, count in snapshot['http_requests_by_endpoint'].items(): + lines.append( + f'snake_http_requests_by_endpoint_total{{endpoint="{endpoint}"}} {count}' + ) + + lines.extend([ + '# HELP snake_moves_by_direction_total Move responses grouped by direction.', + '# TYPE snake_moves_by_direction_total counter', + ]) + for direction, count in snapshot['move_direction_counts'].items(): + lines.append( + f'snake_moves_by_direction_total{{direction="{direction}"}} {count}' + ) + return '\n'.join(lines) + '\n'