add Training for AI and AI Model and allow to collect rl data from BestBattleSnake

snakes/BestBattleSnake.py

@@ -1,10 +1,12 @@
 from collections.abc import Iterator
 from collections import deque
 from typing import Any, cast
-import random, os
 from time import perf_counter
+from pathlib import Path
+import random, json, os
 
 from snakes.TemplateSnake import TemplateSnake
+from server.GameBoard import GameBoard
 
 class BestBattleSnake(TemplateSnake):
   VERSION = "2.6.0"
@@ -38,6 +40,11 @@ class BestBattleSnake(TemplateSnake):
     self.previous_hazards = set()
     self.duel_style = self._get_duel_style()
     self.timeout_buffer_ms = self._get_timeout_buffer_ms()
+    self.rl_bootstrap_enabled = self._env_bool("RL_BOOTSTRAP_ENABLED", default=False)
+    self.rl_min_base_rows = self._env_int("RL_MIN_BASE_ROWS", default=5000)
+    self.rl_base_dataset_path = Path(os.getenv("RL_BASE_DATASET", "data/dataset/best_moves.jsonl"))
+    self.rl_bootstrap_path = Path(os.getenv("RL_BOOTSTRAP_OUTPUT", "data/dataset/rl_bootstrap.jsonl"))
+    self.rl_needs_more_data = False
 
   def _get_duel_style(self) -> str:
     """Resolve duel tuning style from `BATTLE_SNAKE_DUEL_STYLE` or `DUEL_STYLE`."""
@@ -78,7 +85,62 @@ class BestBattleSnake(TemplateSnake):
     except ValueError:
       return 120
 
-  def choose_move(self, game_data:dict) -> str:
+  def _env_bool(self, name:str, default:bool=False) -> bool:
+    value = os.getenv(name)
+    if value is None:
+      return default
+    return value.lower() in {'1', 'true', 'yes', 'on'}
+
+  def _env_int(self, name:str, default:int) -> int:
+    value = os.getenv(name)
+    if value is None:
+      return default
+    try:
+      return int(value)
+    except ValueError:
+      return default
+
+  def _count_jsonl_rows(self, path:Path) -> int:
+    if not path.exists() or not path.is_file():
+      return 0
+    count = 0
+    with path.open("r", encoding="utf-8") as handle:
+      for line in handle:
+        if line.strip():
+          count += 1
+    return count
+
+  def _refresh_rl_bootstrap_state(self):
+    if not self.rl_bootstrap_enabled:
+      self.rl_needs_more_data = False
+      return
+    base_rows = self._count_jsonl_rows(self.rl_base_dataset_path)
+    self.rl_needs_more_data = base_rows < self.rl_min_base_rows
+
+  def _record_rl_bootstrap_sample(self, game_data:GameBoard, move:str, safe_moves:MoveMap, reason:str, scores:dict[str, float]|None=None):
+    if not self.rl_bootstrap_enabled or not self.rl_needs_more_data:
+      return
+
+    try:
+      self.rl_bootstrap_path.parent.mkdir(parents=True, exist_ok=True)
+      row = {
+        "source": "best_battlesnake_bootstrap",
+        "game_id": getattr(game_data, "id", None),
+        "turn": game_data.get_turn(),
+        "move": move,
+        "safe_moves": list(safe_moves.keys()),
+        "reason": reason,
+        "game_board": game_data.get_game_board_as_dict(),
+      }
+      if scores:
+        row["scores"] = {k: round(v, 5) for k, v in scores.items()}
+
+      with self.rl_bootstrap_path.open("a", encoding="utf-8") as handle:
+        handle.write(json.dumps(row, ensure_ascii=False) + "\n")
+    except Exception:
+      return
+
+  def choose_move(self, game_data:GameBoard) -> str:
     """Pick the next move from a Battlesnake move request.
 
     Docs: https://docs.battlesnake.com/api/example-move
@@ -97,6 +159,7 @@ class BestBattleSnake(TemplateSnake):
       self.last_move = None
       self.previous_hazards = set()
       self.last_game_id = game_id
+      self._refresh_rl_bootstrap_state()
 
     my_snake = cast(dict[str, Any], game_data.get_my_snake())
     my_head = my_snake["head"]
@@ -149,6 +212,7 @@ class BestBattleSnake(TemplateSnake):
           "reason": "no_safe_moves",
         }
       )
+      self._record_rl_bootstrap_sample(game_data, fallback, safe_moves, "no_safe_moves")
       self.previous_hazards = set(hazard_set)
       return fallback
 
@@ -179,6 +243,7 @@ class BestBattleSnake(TemplateSnake):
       self.recent_heads.append(current_head_point)
       self.last_move = best_move
       self.add_to_history({"turn": turn, "move": best_move, "scores": scores})
+      self._record_rl_bootstrap_sample(game_data, best_move, safe_moves, "constrictor", scores)
       self.previous_hazards = set(hazard_set)
       return best_move
 
@@ -202,6 +267,7 @@ class BestBattleSnake(TemplateSnake):
       self.recent_heads.append(current_head_point)
       self.last_move = best_move
       self.add_to_history({"turn": turn, "move": best_move, "scores": scores})
+      self._record_rl_bootstrap_sample(game_data, best_move, safe_moves, "duel", scores)
       self.previous_hazards = set(hazard_set)
       return best_move
 
@@ -333,6 +399,7 @@ class BestBattleSnake(TemplateSnake):
       self.recent_heads.append(current_head_point)
       self.last_move = quick_move
       self.add_to_history({"turn": turn, "move": quick_move, "reason": "timeout_budget"})
+      self._record_rl_bootstrap_sample(game_data, quick_move, safe_moves, "timeout_budget")
       self.previous_hazards = set(hazard_set)
       return quick_move
 
@@ -367,6 +434,7 @@ class BestBattleSnake(TemplateSnake):
     self.recent_heads.append(current_head_point)
     self.last_move = best_move
     self.add_to_history({"turn": turn, "move": best_move, "scores": scores})
+    self._record_rl_bootstrap_sample(game_data, best_move, safe_moves, "multi", scores)
     self.previous_hazards = set(hazard_set)
     return best_move
 
@@ -807,7 +875,7 @@ class BestBattleSnake(TemplateSnake):
         return True
     return False
 
-  def _hazard_damage_per_turn(self, game_data:dict) -> int:
+  def _hazard_damage_per_turn(self, game_data:GameBoard) -> int:
     """Read royale hazard damage from ruleset settings.
 
     Docs: https://docs.battlesnake.com/maps/royale