add script to analyse dataset

2026-04-03 15:45:54 +02:00
parent 49f2e0b008
commit 8e733dfe39
3 changed files with 268 additions and 0 deletions
@@ -55,3 +55,6 @@ export-dataset input="data" output="data/dataset/good_moves.jsonl":

 curate-dataset input="good_moves-*.jsonl" output="data/dataset/best_moves.jsonl" min_turn="6" late_turn="20" max_safe_options="2" min_score="3" append="false" archive="false" archive_dir="":
  FLAGS=""; if [ "{{append}}" = "true" ]; then FLAGS="$FLAGS --append"; fi; if [ "{{archive}}" = "true" ]; then FLAGS="$FLAGS --archive-input"; fi; if [ -n "{{archive_dir}}" ]; then FLAGS="$FLAGS --archive-dir {{archive_dir}}"; fi; python -m server.DatasetCurator --input "{{input}}" --output "{{output}}" --min-turn "{{min_turn}}" --late-turn "{{late_turn}}" --max-safe-options "{{max_safe_options}}" --min-score "{{min_score}}" $FLAGS
+
+analyze-dataset input="good_moves-*.jsonl" output="":
+  if [ -n "{{output}}" ]; then python -m server.DatasetStats --input "{{input}}" --output "{{output}}"; else python -m server.DatasetStats --input "{{input}}"; fi