From eb290dd63478e4be5ce40060f91afdc53a4daa61 Mon Sep 17 00:00:00 2001 From: Daniel Dolezal Date: Sat, 4 Apr 2026 23:01:34 +0200 Subject: [PATCH] rework folder structure complitly --- .gitignore | 2 +- server/Server.py | 6 +-- server/{ => dataset}/Dataset.py | 6 +-- server/{ => dataset}/DatasetCurator.py | 46 +++++-------------- server/{ => dataset}/DatasetExporter.py | 12 +++-- server/{ => dataset}/DatasetStats.py | 16 +++---- server/dataset/__init__.py | 4 ++ .../MemoryGameBoardStore.py | 0 .../RedisGameBoardStore.py | 3 +- .../__init__.py | 6 +-- server/storage/EdgeDB.py | 6 +-- server/storage/LocalStorage.py | 5 +- server/storage/StorageLoader.py | 7 --- server/storage/__init__.py | 6 +++ tests/test_Dataset.py | 2 +- tests/test_DatasetExporter.py | 2 +- tests/test_GameStateStore.py | 10 ++-- 17 files changed, 57 insertions(+), 82 deletions(-) rename server/{ => dataset}/Dataset.py (91%) rename server/{ => dataset}/DatasetCurator.py (89%) rename server/{ => dataset}/DatasetExporter.py (89%) rename server/{ => dataset}/DatasetStats.py (96%) create mode 100644 server/dataset/__init__.py rename server/{game_board_stats => game_state_store}/MemoryGameBoardStore.py (100%) rename server/{game_board_stats => game_state_store}/RedisGameBoardStore.py (98%) rename server/{game_board_stats => game_state_store}/__init__.py (60%) delete mode 100644 server/storage/StorageLoader.py create mode 100644 server/storage/__init__.py diff --git a/.gitignore b/.gitignore index 5fec197..9930b5f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,5 +12,5 @@ data/ dbschema/migrations/ *.jsonl -dataset/ +/dataset/ models/ diff --git a/server/Server.py b/server/Server.py index e8c7909..b77dc53 100644 --- a/server/Server.py +++ b/server/Server.py @@ -1,12 +1,12 @@ from quart_common.web.logger import build_logger, await_log from server.Files import read_file -from server.game_board_stats import GameBoardStoreBuilder +from server.game_state_store import GameStateStoreBuilder from server.GameBoard import GameBoard from snakes import SnakeBuilder -from server.storage.StorageLoader import StorageLoader +from server.storage import StorageLoader from server.metrics import ( MetricsStoreBuilder, @@ -39,7 +39,7 @@ class Server: self.store_game_state = False normalized_backend = (game_state_backend or 'memory').strip().lower() self.game_state_local_cache = (game_state_local_cache and normalized_backend != 'memory') - self.game_state_store = GameBoardStoreBuilder.build( + self.game_state_store = GameStateStoreBuilder.build( backend=game_state_backend, redis_url=game_state_redis_url, ttl_seconds=game_state_ttl_sec, diff --git a/server/Dataset.py b/server/dataset/Dataset.py similarity index 91% rename from server/Dataset.py rename to server/dataset/Dataset.py index 02be287..11dd089 100644 --- a/server/Dataset.py +++ b/server/dataset/Dataset.py @@ -3,17 +3,17 @@ from server.GameBoard import GameBoard class Dataset: VALID_MOVES = {"up", "down", "left", "right"} - def __init__(self, game_board: GameBoard): + def __init__(self, game_board:GameBoard): self.game_board = game_board def _did_we_win(self): winners = self.game_board.winner_snake_names or [] return "me" in winners - def _is_good_move(self, move: str): + def _is_good_move(self, move:str): return move in self.VALID_MOVES - def build(self, only_good_moves: bool = True): + def build(self, only_good_moves:bool=True): game_type = self.game_board.get_type_of_game() did_win = self._did_we_win() diff --git a/server/DatasetCurator.py b/server/dataset/DatasetCurator.py similarity index 89% rename from server/DatasetCurator.py rename to server/dataset/DatasetCurator.py index c431757..a6a0f35 100644 --- a/server/DatasetCurator.py +++ b/server/dataset/DatasetCurator.py @@ -1,23 +1,8 @@ -import argparse -import glob -import hashlib -import json -import shutil +import argparse, hashlib, shutil, glob, json from pathlib import Path class DatasetCurator: - def __init__( - self, - input_files: list[str], - output_file: str, - min_turn: int = 6, - late_turn: int = 20, - max_safe_options: int = 2, - min_score: int = 3, - append: bool = False, - archive_input: bool = False, - archive_dir: str | None = None, - ): + def __init__(self, input_files:list[str], output_file:str, min_turn:int=6, late_turn:int=20, max_safe_options:int=2, min_score:int=3, append:bool=False, archive_input:bool=False, archive_dir:str|None=None): self.input_files = input_files self.output_file = Path(output_file) self.min_turn = min_turn @@ -66,42 +51,36 @@ class DatasetCurator: return resolved - def _safe_options_count(self, row: dict): + def _safe_options_count(self, row:dict): history = row.get("history", {}) for item in history.get("data", []): if item.get("function") == "get_possible_moves": return len(item.get("safe_positions", {})) return None - def _state_hash(self, row: dict): + def _state_hash(self, row:dict): board = row.get("game_board", {}) snakes = board.get("snakes", []) snakes_key = [] for snake in snakes: - snakes_key.append( - ( - snake.get("id"), - snake.get("health"), - tuple( - (seg.get("x"), seg.get("y")) for seg in snake.get("body", []) - ), - ) - ) + snakes_key.append(( + snake.get("id"), + snake.get("health"), + tuple((seg.get("x"), seg.get("y")) for seg in snake.get("body", [])), + )) key = { "width": board.get("width"), "height": board.get("height"), "snakes": sorted(snakes_key), "food": sorted((f.get("x"), f.get("y")) for f in board.get("food", [])), - "hazards": sorted( - (h.get("x"), h.get("y")) for h in board.get("hazards", []) - ), + "hazards": sorted((h.get("x"), h.get("y")) for h in board.get("hazards", [])), } raw = json.dumps(key, sort_keys=True, separators=(",", ":")) return hashlib.sha1(raw.encode("utf-8")).hexdigest() - def _score(self, row: dict): + def _score(self, row:dict): score = 0 turn = int(row.get("turn", 0)) safe_options = self._safe_options_count(row) @@ -197,7 +176,7 @@ class DatasetCurator: "output_file": str(self.output_file), } - def _archive_processed_files(self, input_paths: list[Path]): + def _archive_processed_files(self, input_paths:list[Path]): self.archive_dir.mkdir(parents=True, exist_ok=True) archived = [] @@ -235,7 +214,6 @@ class DatasetCurator: return archived - if __name__ == "__main__": parser = argparse.ArgumentParser(description="Create curated best-moves dataset") parser.add_argument( diff --git a/server/DatasetExporter.py b/server/dataset/DatasetExporter.py similarity index 89% rename from server/DatasetExporter.py rename to server/dataset/DatasetExporter.py index b936a0a..c9aee09 100644 --- a/server/DatasetExporter.py +++ b/server/dataset/DatasetExporter.py @@ -1,7 +1,5 @@ -import argparse -import json from pathlib import Path - +import argparse, json class DatasetExporter: def __init__(self, input_dir:str, output_file:str): @@ -56,8 +54,12 @@ class DatasetExporter: if __name__ == "__main__": parser = argparse.ArgumentParser(description="Export Battlesnake dataset to JSONL") - parser.add_argument("--input", default="data", help="Input directory with stored game JSON files") - parser.add_argument("--output", default="data/dataset/good_moves.jsonl", help="Output JSONL file") + parser.add_argument( + "--input", default="data", help="Input directory with stored game JSON files" + ) + parser.add_argument( + "--output", default="data/dataset/good_moves.jsonl", help="Output JSONL file" + ) args = parser.parse_args() report = DatasetExporter(args.input, args.output).export_jsonl() diff --git a/server/DatasetStats.py b/server/dataset/DatasetStats.py similarity index 96% rename from server/DatasetStats.py rename to server/dataset/DatasetStats.py index 6b8b709..db3d747 100644 --- a/server/DatasetStats.py +++ b/server/dataset/DatasetStats.py @@ -1,16 +1,12 @@ -import argparse -import glob -import json -import re from collections import Counter, defaultdict +import argparse, glob, json, re from datetime import datetime from pathlib import Path - class DatasetStats: DAY_PATTERN = re.compile(r"(\d{4}-\d{2}-\d{2})") - def __init__(self, input_files: list[str]): + def __init__(self, input_files:list[str]): self.input_files = input_files def _resolve_input_files(self): @@ -49,20 +45,20 @@ class DatasetStats: return resolved - def _infer_day(self, file_path: Path): + def _infer_day(self, file_path:Path): match = self.DAY_PATTERN.search(file_path.name) if match: return match.group(1) return datetime.fromtimestamp(file_path.stat().st_mtime).strftime("%Y-%m-%d") - def _game_score(self, game: dict): + def _game_score(self, game:dict): max_turn = game["max_turn"] rows = game["rows"] avg_safe = game["avg_safe_options"] pressure_bonus = 0 if avg_safe is None else max(0.0, 4.0 - avg_safe) return round(max_turn * 2.0 + rows + pressure_bonus, 3) - def _pressure_score(self, game: dict): + def _pressure_score(self, game:dict): max_turn = game["max_turn"] rows = max(1, game["rows"]) pressure_turns = game["pressure_turns"] @@ -72,7 +68,7 @@ class DatasetStats: safe_tightness = 0.0 if avg_safe is None else max(0.0, 3.0 - avg_safe) return round(max_turn * 1.2 + pressure_ratio * 120.0 + safe_tightness * 20.0, 3) - def _extract_safe_options(self, row: dict): + def _extract_safe_options(self, row:dict): top_level = row.get("safe_options") if isinstance(top_level, int): return top_level diff --git a/server/dataset/__init__.py b/server/dataset/__init__.py new file mode 100644 index 0000000..56431c5 --- /dev/null +++ b/server/dataset/__init__.py @@ -0,0 +1,4 @@ +from .Dataset import Dataset +from .DatasetExporter import DatasetExporter +from .DatasetCurator import DatasetCurator +from .DatasetStats import DatasetStats diff --git a/server/game_board_stats/MemoryGameBoardStore.py b/server/game_state_store/MemoryGameBoardStore.py similarity index 100% rename from server/game_board_stats/MemoryGameBoardStore.py rename to server/game_state_store/MemoryGameBoardStore.py diff --git a/server/game_board_stats/RedisGameBoardStore.py b/server/game_state_store/RedisGameBoardStore.py similarity index 98% rename from server/game_board_stats/RedisGameBoardStore.py rename to server/game_state_store/RedisGameBoardStore.py index 9207898..afabb88 100644 --- a/server/game_board_stats/RedisGameBoardStore.py +++ b/server/game_state_store/RedisGameBoardStore.py @@ -1,6 +1,5 @@ from server.GameBoard import GameBoard -import inspect -import pickle +import inspect, pickle class RedisGameBoardStore: def __init__(self, redis_url:str="redis://localhost:6379/0", key_prefix:str="snake:gameboard", ttl_seconds:int=900, **kwargs): diff --git a/server/game_board_stats/__init__.py b/server/game_state_store/__init__.py similarity index 60% rename from server/game_board_stats/__init__.py rename to server/game_state_store/__init__.py index 517fde9..b2390c9 100644 --- a/server/game_board_stats/__init__.py +++ b/server/game_state_store/__init__.py @@ -1,7 +1,7 @@ -from server.game_board_stats.MemoryGameBoardStore import MemoryGameBoardStore -from server.game_board_stats.RedisGameBoardStore import RedisGameBoardStore +from .MemoryGameBoardStore import MemoryGameBoardStore +from .RedisGameBoardStore import RedisGameBoardStore -class GameBoardStoreBuilder: +class GameStateStoreBuilder: @classmethod def build(self, backend:str="memory", **kwargs) -> MemoryGameBoardStore|RedisGameBoardStore: selected = (backend or "memory").strip().lower() diff --git a/server/storage/EdgeDB.py b/server/storage/EdgeDB.py index 18f156b..11493c0 100644 --- a/server/storage/EdgeDB.py +++ b/server/storage/EdgeDB.py @@ -1,5 +1,5 @@ from server.GameBoard import GameBoard -from server.Dataset import Dataset +from server.dataset.Dataset import Dataset from datetime import datetime import json, time @@ -55,8 +55,8 @@ class EdgeDB: calculations = snake_calulations[i] if i < len(snake_calulations) else [] calculations.append({ "dataset": { - "is_good_move": labels_by_turn.get(moves[i]["turn"], False) - } + "is_good_move": labels_by_turn.get(moves[i]["turn"], False) + } }) data.append({ "turn": moves[i]["turn"], diff --git a/server/storage/LocalStorage.py b/server/storage/LocalStorage.py index 2151433..e2b6098 100644 --- a/server/storage/LocalStorage.py +++ b/server/storage/LocalStorage.py @@ -1,11 +1,10 @@ +from server.dataset.Dataset import Dataset from server.GameBoard import GameBoard -from server.Dataset import Dataset from server.Files import save_file import aiofiles import aiofiles.os -import gzip -import json, os +import gzip, json, os class LocalStorage: def __init__(self, file_path:str, **kwargs): diff --git a/server/storage/StorageLoader.py b/server/storage/StorageLoader.py deleted file mode 100644 index c31f962..0000000 --- a/server/storage/StorageLoader.py +++ /dev/null @@ -1,7 +0,0 @@ - -class StorageLoader: - @classmethod - def build(self, selected_storage:str): - storage_module = __import__(f'server.storage.{selected_storage}', fromlist=[selected_storage]) - storage_class = getattr(storage_module, selected_storage) - return storage_class diff --git a/server/storage/__init__.py b/server/storage/__init__.py new file mode 100644 index 0000000..0ac5113 --- /dev/null +++ b/server/storage/__init__.py @@ -0,0 +1,6 @@ +class StorageLoader: + @classmethod + def build(self, selected_storage: str): + storage_module = __import__(f"server.storage.{selected_storage}", fromlist=[selected_storage]) + storage_class = getattr(storage_module, selected_storage) + return storage_class diff --git a/tests/test_Dataset.py b/tests/test_Dataset.py index 65e4a97..657dbf4 100644 --- a/tests/test_Dataset.py +++ b/tests/test_Dataset.py @@ -1,7 +1,7 @@ import unittest from typing import cast -from server.Dataset import Dataset +from server.dataset.Dataset import Dataset from server.GameBoard import GameBoard class DummySnake: diff --git a/tests/test_DatasetExporter.py b/tests/test_DatasetExporter.py index 56bf361..2f307de 100644 --- a/tests/test_DatasetExporter.py +++ b/tests/test_DatasetExporter.py @@ -3,7 +3,7 @@ import tempfile import unittest from pathlib import Path -from server.DatasetExporter import DatasetExporter +from server.dataset.DatasetExporter import DatasetExporter class TestDatasetExporter(unittest.TestCase): def test_export_jsonl(self): diff --git a/tests/test_GameStateStore.py b/tests/test_GameStateStore.py index 6c237a3..f826ffd 100644 --- a/tests/test_GameStateStore.py +++ b/tests/test_GameStateStore.py @@ -2,9 +2,7 @@ import unittest from typing import Any, cast from server.GameBoard import GameBoard -from server.game_board_stats import GameBoardStoreBuilder -from server.game_board_stats.MemoryGameBoardStore import MemoryGameBoardStore -from server.game_board_stats.RedisGameBoardStore import RedisGameBoardStore +from server.game_state_store import GameStateStoreBuilder, MemoryGameBoardStore, RedisGameBoardStore from snakes.TemplateSnake import TemplateSnake class _FakeRedis: @@ -73,9 +71,9 @@ class TestGameStateStore(unittest.IsolatedAsyncioTestCase): return board def test_builder_selects_store_backend(self): - memory_store = GameBoardStoreBuilder.build(backend="memory") - redis_store = GameBoardStoreBuilder.build(backend="redis") - default_store = GameBoardStoreBuilder.build(backend="unknown") + memory_store = GameStateStoreBuilder.build(backend="memory") + redis_store = GameStateStoreBuilder.build(backend="redis") + default_store = GameStateStoreBuilder.build(backend="unknown") self.assertIsInstance(memory_store, MemoryGameBoardStore) self.assertIsInstance(redis_store, RedisGameBoardStore)