update to store and curate data correctly

server/DatasetCurator.py

@@ -0,0 +1,144 @@
+import argparse
+import hashlib
+import json
+from pathlib import Path
+
+class DatasetCurator:
+  def __init__(self, input_file: str, output_file: str, min_turn: int = 6, late_turn: int = 20, max_safe_options: int = 2, min_score: int = 3,):
+    self.input_file = Path(input_file)
+    self.output_file = Path(output_file)
+    self.min_turn = min_turn
+    self.late_turn = late_turn
+    self.max_safe_options = max_safe_options
+    self.min_score = min_score
+
+  def _safe_options_count(self, row: dict):
+    history = row.get("history", {})
+    for item in history.get("data", []):
+      if item.get("function") == "get_possible_moves":
+        return len(item.get("safe_positions", {}))
+    return None
+
+  def _state_hash(self, row: dict):
+    board = row.get("game_board", {})
+    snakes = board.get("snakes", [])
+
+    snakes_key = []
+    for snake in snakes:
+      snakes_key.append(
+        (
+          snake.get("id"),
+          snake.get("health"),
+          tuple(
+            (seg.get("x"), seg.get("y")) for seg in snake.get("body", [])
+          ),
+        )
+      )
+
+    key = {
+      "width": board.get("width"),
+      "height": board.get("height"),
+      "snakes": sorted(snakes_key),
+      "food": sorted((f.get("x"), f.get("y")) for f in board.get("food", [])),
+      "hazards": sorted(
+        (h.get("x"), h.get("y")) for h in board.get("hazards", [])
+      ),
+    }
+    raw = json.dumps(key, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha1(raw.encode("utf-8")).hexdigest()
+
+  def _score(self, row: dict):
+    score = 0
+    turn = int(row.get("turn", 0))
+    safe_options = self._safe_options_count(row)
+    snakes = row.get("game_board", {}).get("snakes", [])
+    opponents = max(0, len(snakes) - 1)
+
+    if turn >= self.late_turn:
+      score += 2
+    if safe_options is not None and safe_options <= self.max_safe_options:
+      score += 3
+    if opponents >= 1:
+      score += 1
+
+    return score, safe_options
+
+  def curate(self):
+    self.output_file.parent.mkdir(parents=True, exist_ok=True)
+
+    total = 0
+    kept = 0
+    skipped_turn = 0
+    skipped_quality = 0
+    skipped_duplicate = 0
+    seen_states = set()
+
+    with self.input_file.open("r", encoding="utf-8") as src:
+      with self.output_file.open("w", encoding="utf-8") as dst:
+        for line in src:
+          if not line.strip():
+            continue
+
+          total += 1
+          row = json.loads(line)
+
+          if not row.get("is_good_move", False):
+            skipped_quality += 1
+            continue
+
+          if int(row.get("turn", 0)) < self.min_turn:
+            skipped_turn += 1
+            continue
+
+          quality_score, safe_options = self._score(row)
+          if quality_score < self.min_score:
+            skipped_quality += 1
+            continue
+
+          state_key = self._state_hash(row)
+          dedupe_key = (state_key, row.get("move"))
+          if dedupe_key in seen_states:
+            skipped_duplicate += 1
+            continue
+          seen_states.add(dedupe_key)
+
+          compact_row = {
+            "game_id": row.get("game_id"),
+            "turn": row.get("turn"),
+            "move": row.get("move"),
+            "game_type": row.get("game_type"),
+            "quality_score": quality_score,
+            "safe_options": safe_options,
+            "game_board": row.get("game_board"),
+          }
+          dst.write(json.dumps(compact_row, ensure_ascii=False) + "\n")
+          kept += 1
+
+    return {
+      "total_rows": total,
+      "kept_rows": kept,
+      "skipped_turn": skipped_turn,
+      "skipped_quality": skipped_quality,
+      "skipped_duplicate": skipped_duplicate,
+      "output_file": str(self.output_file),
+    }
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="Create curated best-moves dataset")
+  parser.add_argument("--input", required=True, help="Input JSONL file")
+  parser.add_argument("--output", required=True, help="Output JSONL file")
+  parser.add_argument("--min-turn", type=int, default=6)
+  parser.add_argument("--late-turn", type=int, default=20)
+  parser.add_argument("--max-safe-options", type=int, default=2)
+  parser.add_argument("--min-score", type=int, default=3)
+  args = parser.parse_args()
+
+  report = DatasetCurator(
+    input_file=args.input,
+    output_file=args.output,
+    min_turn=args.min_turn,
+    late_turn=args.late_turn,
+    max_safe_options=args.max_safe_options,
+    min_score=args.min_score,
+  ).curate()
+  print(json.dumps(report, indent=2))