add compression to GameplayDatabase and test if compression works with the sqlite_zstd extension

server/database/GameplayDatabase.py

@@ -1,7 +1,9 @@
 from datetime import datetime, timezone
-import asyncio, sqlite3, json
+import asyncio, sqlite3, json, os
 from pathlib import Path
 
+_ZSTD_EXT = Path(os.environ.get("SQLITE_ZSTD_EXT", "/usr/local/lib/libsqlite_zstd.so")).expanduser().resolve()
+
 class GameplayDatabase:
   def __init__(self, db_path:str, busy_timeout_ms:int=5000):
     self.db_path = db_path
@@ -14,6 +16,12 @@ class GameplayDatabase:
       timeout=max(1, self.busy_timeout_ms // 1000),
       isolation_level=None,
     )
+
+    if Path(_ZSTD_EXT).exists():
+      connection.enable_load_extension(True)
+      connection.load_extension(str(_ZSTD_EXT))
+      connection.enable_load_extension(False)
+
     connection.row_factory = sqlite3.Row
     connection.execute("PRAGMA foreign_keys = ON")
     connection.execute("PRAGMA journal_mode = WAL")
@@ -26,7 +34,10 @@ class GameplayDatabase:
   def _initialize_database(self) -> None:
     Path(self.db_path).parent.mkdir(parents=True, exist_ok=True)
     with self._connect() as connection:
-      connection.execute("PRAGMA auto_vacuum = INCREMENTAL")
+      current_vacuum = connection.execute("PRAGMA auto_vacuum").fetchone()[0]
+      if current_vacuum != 1:
+        connection.execute("PRAGMA auto_vacuum = FULL")
+        connection.execute("VACUUM")
       connection.executescript("""
         CREATE TABLE IF NOT EXISTS games (
           game_id TEXT PRIMARY KEY,
@@ -81,17 +92,31 @@ class GameplayDatabase:
           FOREIGN KEY (game_id) REFERENCES games(game_id) ON DELETE CASCADE
         );
 
-        CREATE INDEX IF NOT EXISTS idx_turns_game_turn ON turns(game_id, turn);
-        CREATE INDEX IF NOT EXISTS idx_games_status ON games(status);
-        CREATE INDEX IF NOT EXISTS idx_snake_turns_game_turn ON snake_turns(game_id, turn);
         """)
+      self._create_indexes_if_tables(connection)
       self._ensure_column_exists(connection, "turns", "my_thinking_json", "TEXT")
       self._ensure_column_exists(connection, "games", "your_snake_type", "TEXT")
       self._ensure_column_exists(connection, "games", "your_snake_version", "TEXT")
       self._ensure_column_exists(connection, "games", "game_type", "TEXT")
       self._ensure_column_exists(connection, "snake_turns", "latency", "TEXT")
+      self._enable_zstd_compression(connection)
       connection.execute("PRAGMA optimize")
 
+  def _create_indexes_if_tables(self, connection: sqlite3.Connection) -> None:
+    real_tables = {
+      row[0] for row in connection.execute(
+        "SELECT name FROM sqlite_master WHERE type='table'"
+      ).fetchall()
+    }
+    indexes = [
+      ("idx_turns_game_turn",       "turns",       "game_id, turn"),
+      ("idx_games_status",          "games",       "status"),
+      ("idx_snake_turns_game_turn", "snake_turns", "game_id, turn"),
+    ]
+    for idx_name, table, cols in indexes:
+      if table in real_tables:
+        connection.execute(f"CREATE INDEX IF NOT EXISTS {idx_name} ON {table}({cols})")
+
   def _ensure_column_exists(self, connection:sqlite3.Connection, table_name:str, column_name:str, column_type:str) -> None:
     existing = connection.execute(f"PRAGMA table_info({table_name})").fetchall()
     if any(row["name"] == column_name for row in existing):
@@ -99,6 +124,26 @@ class GameplayDatabase:
 
     connection.execute(f"ALTER TABLE {table_name} ADD COLUMN {column_name} {column_type}")
 
+  def _enable_zstd_compression(self, connection: sqlite3.Connection) -> None:
+    compressed_columns = [
+      ("turns",       "board_state_json"),
+      ("turns",       "snakes_json"),
+      ("turns",       "you_json"),
+      ("turns",       "food_json"),
+      ("turns",       "hazards_json"),
+      ("snake_turns", "body_json"),
+    ]
+    for table, column in compressed_columns:
+      try:
+        connection.execute(
+          "SELECT zstd_enable_transparent(?)",
+          [json.dumps({"table": table, "column": column, "compression_level": 6, "dict_chooser": "'a'"})],
+        )
+      except sqlite3.OperationalError:
+        pass  # already enabled
+
+    connection.execute("SELECT zstd_incremental_maintenance(null, 1)")
+
   def _utc_now(self) -> str:
     return datetime.now(timezone.utc).isoformat()
 
@@ -200,7 +245,6 @@ class GameplayDatabase:
         ),
       )
       connection.execute("PRAGMA wal_checkpoint(PASSIVE)")
-      connection.execute("PRAGMA incremental_vacuum(200)")
       connection.execute("PRAGMA optimize")
 
   def _record_turn_sync(self, game_state:dict, my_move:str|None, my_thinking:dict|None) -> None: