update to store and curate data correctly
This commit is contained in:
@@ -8,3 +8,4 @@ __pycache__/
|
||||
data/
|
||||
.env
|
||||
dbschema/migrations/
|
||||
*.jsonl
|
||||
|
||||
@@ -90,6 +90,49 @@ Or with `just`:
|
||||
just export-dataset
|
||||
```
|
||||
|
||||
Curate a high-quality training subset (single file):
|
||||
|
||||
```sh
|
||||
python -m server.DatasetCurator --input good_moves-2026-04-03.jsonl --output data/dataset/best_moves.jsonl
|
||||
```
|
||||
|
||||
Curate from multiple JSONL sources (repeat `--input`):
|
||||
|
||||
```sh
|
||||
python -m server.DatasetCurator \
|
||||
--input good_moves-2026-04-03.jsonl \
|
||||
--input good_moves-2026-04-04.jsonl \
|
||||
--output data/dataset/best_moves.jsonl
|
||||
```
|
||||
|
||||
Curate from folder or glob:
|
||||
|
||||
```sh
|
||||
python -m server.DatasetCurator --input data/dataset --output data/dataset/best_moves.jsonl
|
||||
python -m server.DatasetCurator --input "good_moves-*.jsonl" --output data/dataset/best_moves.jsonl
|
||||
```
|
||||
|
||||
Append mode (keeps existing curated rows and deduplicates against them):
|
||||
|
||||
```sh
|
||||
python -m server.DatasetCurator --input "good_moves-*.jsonl" --output data/dataset/best_moves.jsonl --append
|
||||
```
|
||||
|
||||
Archive processed input files after curation:
|
||||
|
||||
```sh
|
||||
python -m server.DatasetCurator --input "good_moves-*.jsonl" --output data/dataset/best_moves.jsonl --append --archive-input
|
||||
python -m server.DatasetCurator --input "good_moves-*.jsonl" --output data/dataset/best_moves.jsonl --append --archive-input --archive-dir data/dataset/archive
|
||||
```
|
||||
|
||||
Or with `just`:
|
||||
|
||||
```sh
|
||||
just curate-dataset
|
||||
just curate-dataset append=true
|
||||
just curate-dataset append=true archive=true archive_dir=data/dataset/archive
|
||||
```
|
||||
|
||||
To store compact dataset-only records (JSONL) and skip full per-game JSON files:
|
||||
|
||||
```sh
|
||||
|
||||
@@ -5,7 +5,6 @@ server = Server(
|
||||
data_path=os.path.dirname(__file__),
|
||||
snake_type=os.environ.get("SNAKE", "BestBattleSnake"),
|
||||
storage_type=os.environ.get("STORAGE", "LocalStorage"),
|
||||
store_game_when_win_and_moves_are_bigger_as=int(os.environ.get("STORE_IF_WIN_AND_MOVES_ARE_BIGGER_AS", 10)),
|
||||
debug=os.environ.get("DEBUG_SERVER", False),
|
||||
check_tls_security=False,
|
||||
)
|
||||
|
||||
@@ -52,3 +52,6 @@ test-seed:
|
||||
|
||||
export-dataset input="data" output="data/dataset/good_moves.jsonl":
|
||||
python -m server.DatasetExporter --input "{{input}}" --output "{{output}}"
|
||||
|
||||
curate-dataset input="good_moves-*.jsonl" output="data/dataset/best_moves.jsonl" min_turn="6" late_turn="20" max_safe_options="2" min_score="3" append="false" archive="false" archive_dir="":
|
||||
FLAGS=""; if [ "{{append}}" = "true" ]; then FLAGS="$FLAGS --append"; fi; if [ "{{archive}}" = "true" ]; then FLAGS="$FLAGS --archive-input"; fi; if [ -n "{{archive_dir}}" ]; then FLAGS="$FLAGS --archive-dir {{archive_dir}}"; fi; python -m server.DatasetCurator --input "{{input}}" --output "{{output}}" --min-turn "{{min_turn}}" --late-turn "{{late_turn}}" --max-safe-options "{{max_safe_options}}" --min-score "{{min_score}}" $FLAGS
|
||||
|
||||
@@ -24,14 +24,12 @@ if __name__ == "__main__":
|
||||
"STORE_GAME_HISTORY": True,
|
||||
"DEBUG": True,
|
||||
"SNAKE": "TemplateSnake",
|
||||
"STORE_IF_WIN_AND_MOVES_ARE_BIGGER_AS": 10,
|
||||
})
|
||||
|
||||
server = Server(
|
||||
data_path=os.path.dirname(__file__),
|
||||
snake_type=os.environ.get("SNAKE", "TemplateSnake"),
|
||||
storage_type=os.environ.get("STORAGE", "LocalStorage"),
|
||||
store_game_when_win_and_moves_are_bigger_as=int(os.environ.get("STORE_IF_WIN_AND_MOVES_ARE_BIGGER_AS", 10)),
|
||||
debug=os.environ.get("DEBUG_SERVER", False),
|
||||
check_tls_security=False,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
class DatasetCurator:
|
||||
def __init__(self, input_file: str, output_file: str, min_turn: int = 6, late_turn: int = 20, max_safe_options: int = 2, min_score: int = 3,):
|
||||
self.input_file = Path(input_file)
|
||||
self.output_file = Path(output_file)
|
||||
self.min_turn = min_turn
|
||||
self.late_turn = late_turn
|
||||
self.max_safe_options = max_safe_options
|
||||
self.min_score = min_score
|
||||
|
||||
def _safe_options_count(self, row: dict):
|
||||
history = row.get("history", {})
|
||||
for item in history.get("data", []):
|
||||
if item.get("function") == "get_possible_moves":
|
||||
return len(item.get("safe_positions", {}))
|
||||
return None
|
||||
|
||||
def _state_hash(self, row: dict):
|
||||
board = row.get("game_board", {})
|
||||
snakes = board.get("snakes", [])
|
||||
|
||||
snakes_key = []
|
||||
for snake in snakes:
|
||||
snakes_key.append(
|
||||
(
|
||||
snake.get("id"),
|
||||
snake.get("health"),
|
||||
tuple(
|
||||
(seg.get("x"), seg.get("y")) for seg in snake.get("body", [])
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
key = {
|
||||
"width": board.get("width"),
|
||||
"height": board.get("height"),
|
||||
"snakes": sorted(snakes_key),
|
||||
"food": sorted((f.get("x"), f.get("y")) for f in board.get("food", [])),
|
||||
"hazards": sorted(
|
||||
(h.get("x"), h.get("y")) for h in board.get("hazards", [])
|
||||
),
|
||||
}
|
||||
raw = json.dumps(key, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha1(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
def _score(self, row: dict):
|
||||
score = 0
|
||||
turn = int(row.get("turn", 0))
|
||||
safe_options = self._safe_options_count(row)
|
||||
snakes = row.get("game_board", {}).get("snakes", [])
|
||||
opponents = max(0, len(snakes) - 1)
|
||||
|
||||
if turn >= self.late_turn:
|
||||
score += 2
|
||||
if safe_options is not None and safe_options <= self.max_safe_options:
|
||||
score += 3
|
||||
if opponents >= 1:
|
||||
score += 1
|
||||
|
||||
return score, safe_options
|
||||
|
||||
def curate(self):
|
||||
self.output_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
total = 0
|
||||
kept = 0
|
||||
skipped_turn = 0
|
||||
skipped_quality = 0
|
||||
skipped_duplicate = 0
|
||||
seen_states = set()
|
||||
|
||||
with self.input_file.open("r", encoding="utf-8") as src:
|
||||
with self.output_file.open("w", encoding="utf-8") as dst:
|
||||
for line in src:
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
total += 1
|
||||
row = json.loads(line)
|
||||
|
||||
if not row.get("is_good_move", False):
|
||||
skipped_quality += 1
|
||||
continue
|
||||
|
||||
if int(row.get("turn", 0)) < self.min_turn:
|
||||
skipped_turn += 1
|
||||
continue
|
||||
|
||||
quality_score, safe_options = self._score(row)
|
||||
if quality_score < self.min_score:
|
||||
skipped_quality += 1
|
||||
continue
|
||||
|
||||
state_key = self._state_hash(row)
|
||||
dedupe_key = (state_key, row.get("move"))
|
||||
if dedupe_key in seen_states:
|
||||
skipped_duplicate += 1
|
||||
continue
|
||||
seen_states.add(dedupe_key)
|
||||
|
||||
compact_row = {
|
||||
"game_id": row.get("game_id"),
|
||||
"turn": row.get("turn"),
|
||||
"move": row.get("move"),
|
||||
"game_type": row.get("game_type"),
|
||||
"quality_score": quality_score,
|
||||
"safe_options": safe_options,
|
||||
"game_board": row.get("game_board"),
|
||||
}
|
||||
dst.write(json.dumps(compact_row, ensure_ascii=False) + "\n")
|
||||
kept += 1
|
||||
|
||||
return {
|
||||
"total_rows": total,
|
||||
"kept_rows": kept,
|
||||
"skipped_turn": skipped_turn,
|
||||
"skipped_quality": skipped_quality,
|
||||
"skipped_duplicate": skipped_duplicate,
|
||||
"output_file": str(self.output_file),
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Create curated best-moves dataset")
|
||||
parser.add_argument("--input", required=True, help="Input JSONL file")
|
||||
parser.add_argument("--output", required=True, help="Output JSONL file")
|
||||
parser.add_argument("--min-turn", type=int, default=6)
|
||||
parser.add_argument("--late-turn", type=int, default=20)
|
||||
parser.add_argument("--max-safe-options", type=int, default=2)
|
||||
parser.add_argument("--min-score", type=int, default=3)
|
||||
args = parser.parse_args()
|
||||
|
||||
report = DatasetCurator(
|
||||
input_file=args.input,
|
||||
output_file=args.output,
|
||||
min_turn=args.min_turn,
|
||||
late_turn=args.late_turn,
|
||||
max_safe_options=args.max_safe_options,
|
||||
min_score=args.min_score,
|
||||
).curate()
|
||||
print(json.dumps(report, indent=2))
|
||||
+8
-4
@@ -8,9 +8,15 @@ from quart import Quart, request, jsonify
|
||||
import logging, json, os, re
|
||||
|
||||
class Server:
|
||||
default_snake_config = {"apiversion":"1","author":"","color":"#888888","head":"default","tail":"default"}
|
||||
default_snake_config = {
|
||||
"apiversion": "1",
|
||||
"author": "",
|
||||
"color": "#888888",
|
||||
"head": "default",
|
||||
"tail": "default",
|
||||
}
|
||||
|
||||
def __init__(self, data_path:str, snake_type:str, storage_type:str, debug:bool=False, store_game_when_win_and_moves_are_bigger_as:int=10, check_tls_security:bool=False):
|
||||
def __init__(self, data_path:str, snake_type:str, storage_type:str, debug:bool=False, check_tls_security:bool=False):
|
||||
self.debug = debug
|
||||
self.snake_type = snake_type
|
||||
self.storage_type = storage_type
|
||||
@@ -20,7 +26,6 @@ class Server:
|
||||
self.check_tls_security = check_tls_security
|
||||
|
||||
self.store_game_state = False
|
||||
self.store_game_when_win_and_moves_are_bigger_as = store_game_when_win_and_moves_are_bigger_as
|
||||
|
||||
self.running_games:dict[str, GameBoard] = {}
|
||||
|
||||
@@ -62,7 +67,6 @@ class Server:
|
||||
game_state = await request.get_json()
|
||||
if self.store_game_state:
|
||||
game_board = await self._get_game_board(game_state, end=True)
|
||||
#if not game_board.get_winner() == "me" and not game_board.get_turn() <= self.store_game_when_win_and_moves_are_bigger_as:
|
||||
if self.check_tls_security:
|
||||
await game_board.save(
|
||||
StorageLoader.build(self.storage_type),
|
||||
|
||||
Reference in New Issue
Block a user