{
  "instances": 69,
  "families": 14,
  "published": 30,
  "configurations": 17,
  "rows": [
    {
      "id": "opus55_tools",
      "model": "Claude Opus 5.5",
      "effort": "high",
      "track": "tool-assisted",
      "score": 180.05879794613446,
      "hfr": 0.9850625231363075,
      "valid": 100.0,
      "output_tokens": 1702589,
      "mean_output_tokens": 24675.202898550724,
      "token_usage_is_lower_bound": true,
      "token_usage_known_instances": 69,
      "token_usage_complete_instances": 68
    },
    {
      "id": "astra_tools",
      "model": "GPT-6 Astra",
      "effort": "high",
      "track": "tool-assisted",
      "score": 143.15755303222824,
      "hfr": 0.9983527131782945,
      "valid": 100.0,
      "output_tokens": 445123,
      "mean_output_tokens": 6451.057971014493,
      "token_usage_is_lower_bound": false,
      "token_usage_known_instances": 69,
      "token_usage_complete_instances": 69
    },
    {
      "id": "luna_tools",
      "model": "GPT-5.6 Luna",
      "effort": "high",
      "track": "tool-assisted",
      "score": 119.35416818356748,
      "hfr": 0.9609113359497304,
      "valid": 95.65217391304348,
      "output_tokens": 866731,
      "mean_output_tokens": 12561.31884057971,
      "token_usage_is_lower_bound": false,
      "token_usage_known_instances": 69,
      "token_usage_complete_instances": 69
    },
    {
      "id": "astra_high",
      "model": "GPT-6 Astra",
      "effort": "high",
      "track": "tool-free",
      "score": 91.89683210349455,
      "hfr": 0.7624057710284657,
      "valid": 100.0,
      "output_tokens": 1016189,
      "mean_output_tokens": 14727.376811594202,
      "token_usage_is_lower_bound": true,
      "token_usage_known_instances": 69,
      "token_usage_complete_instances": 68
    },
    {
      "id": "astra_medium",
      "model": "GPT-6 Astra",
      "effort": "medium",
      "track": "tool-free",
      "score": 75.33972146300668,
      "hfr": 0.6561895846265097,
      "valid": 98.55072463768117,
      "output_tokens": 329309,
      "mean_output_tokens": 4772.594202898551,
      "token_usage_is_lower_bound": false,
      "token_usage_known_instances": 69,
      "token_usage_complete_instances": 69
    },
    {
      "id": "fable",
      "model": "Claude Fable 5.1",
      "effort": "medium",
      "track": "tool-free",
      "score": 71.89512750577023,
      "hfr": 0.5901052098702387,
      "valid": 85.5072463768116,
      "output_tokens": 2655322,
      "mean_output_tokens": 38482.92753623189,
      "token_usage_is_lower_bound": false,
      "token_usage_known_instances": 69,
      "token_usage_complete_instances": 69
    },
    {
      "id": "opus55_high",
      "model": "Claude Opus 5.5",
      "effort": "high",
      "track": "tool-free",
      "score": 69.81016939692064,
      "hfr": 0.5738437106351996,
      "valid": 82.6086956521739,
      "output_tokens": 2843896,
      "mean_output_tokens": 41215.884057971016,
      "token_usage_is_lower_bound": false,
      "token_usage_known_instances": 69,
      "token_usage_complete_instances": 69
    },
    {
      "id": "opus55_medium",
      "model": "Claude Opus 5.5",
      "effort": "medium",
      "track": "tool-free",
      "score": 66.69944099739205,
      "hfr": 0.5785533665514241,
      "valid": 88.40579710144928,
      "output_tokens": 1103648,
      "mean_output_tokens": 15994.898550724638,
      "token_usage_is_lower_bound": false,
      "token_usage_known_instances": 69,
      "token_usage_complete_instances": 69
    },
    {
      "id": "deepseek_high",
      "model": "DeepSeek V4.1 Flash",
      "effort": "high",
      "track": "tool-free",
      "score": 44.89806890990687,
      "hfr": 0.3785294554561002,
      "valid": 75.36231884057972,
      "output_tokens": 5062981,
      "mean_output_tokens": 73376.53623188406,
      "token_usage_is_lower_bound": false,
      "token_usage_known_instances": 69,
      "token_usage_complete_instances": 69
    },
    {
      "id": "fable_high",
      "model": "Claude Fable 5.1",
      "effort": "high",
      "track": "tool-free",
      "score": 43.733216554432424,
      "hfr": 0.5398261924645702,
      "valid": 52.17391304347826,
      "output_tokens": 5388481,
      "mean_output_tokens": 78093.92753623189,
      "token_usage_is_lower_bound": true,
      "token_usage_known_instances": 69,
      "token_usage_complete_instances": 68
    },
    {
      "id": "qwen38",
      "model": "Qwen3.8-27B",
      "effort": "high",
      "track": "tool-free",
      "score": 38.06654380503357,
      "hfr": 0.30959200535079906,
      "valid": 62.31884057971014,
      "output_tokens": 6426008,
      "mean_output_tokens": 93130.55072463768,
      "token_usage_is_lower_bound": false,
      "token_usage_known_instances": 69,
      "token_usage_complete_instances": 69
    },
    {
      "id": "deepseek_low",
      "model": "DeepSeek V4.1 Flash",
      "effort": "low",
      "track": "tool-free",
      "score": 37.10558480494949,
      "hfr": 0.3909162323098981,
      "valid": 78.26086956521739,
      "output_tokens": 3484759,
      "mean_output_tokens": 50503.75362318841,
      "token_usage_is_lower_bound": false,
      "token_usage_known_instances": 69,
      "token_usage_complete_instances": 69
    },
    {
      "id": "luna_high",
      "model": "GPT-5.6 Luna",
      "effort": "high",
      "track": "tool-free",
      "score": 34.85434046942786,
      "hfr": 0.25225582230527727,
      "valid": 68.11594202898551,
      "output_tokens": 1075115,
      "mean_output_tokens": 15581.376811594202,
      "token_usage_is_lower_bound": true,
      "token_usage_known_instances": 68,
      "token_usage_complete_instances": 68
    },
    {
      "id": "luna_medium",
      "model": "GPT-5.6 Luna",
      "effort": "medium",
      "track": "tool-free",
      "score": 16.695906429695206,
      "hfr": 0.16379044431922102,
      "valid": 56.52173913043478,
      "output_tokens": 280150,
      "mean_output_tokens": 4060.144927536232,
      "token_usage_is_lower_bound": true,
      "token_usage_known_instances": 68,
      "token_usage_complete_instances": 68
    },
    {
      "id": "qwen35",
      "model": "Qwen3.5-27B",
      "effort": "high",
      "track": "tool-free",
      "score": 16.17731377372567,
      "hfr": 0.1317399843944729,
      "valid": 52.17391304347826,
      "output_tokens": 3906549,
      "mean_output_tokens": 56616.65217391304,
      "token_usage_is_lower_bound": false,
      "token_usage_known_instances": 69,
      "token_usage_complete_instances": 69
    },
    {
      "id": "qwen9",
      "model": "Qwen3.5-9B",
      "effort": "high",
      "track": "tool-free",
      "score": 14.516174608015614,
      "hfr": 0.11798935765066647,
      "valid": 43.47826086956522,
      "output_tokens": 3977429,
      "mean_output_tokens": 57643.89855072464,
      "token_usage_is_lower_bound": false,
      "token_usage_known_instances": 69,
      "token_usage_complete_instances": 69
    },
    {
      "id": "qwen4",
      "model": "Qwen3.5-4B",
      "effort": "high",
      "track": "tool-free",
      "score": 7.551596769783502,
      "hfr": 0.0673626583634123,
      "valid": 37.68115942028986,
      "output_tokens": 3653968,
      "mean_output_tokens": 52956.057971014496,
      "token_usage_is_lower_bound": false,
      "token_usage_known_instances": 69,
      "token_usage_complete_instances": 69
    }
  ],
  "sources": [
    "bench/paper/tables/publication_data.json",
    "bench/paper/tables/tool_publication_data.json",
    "output/figures/score-vs-tokens/data.json"
  ],
  "token_accounting": {
    "definition": "Reported output tokens, including reasoning, for the 69 selected responses. Input tokens and earlier failed/superseded submissions are excluded. For tools, count all model responses in the selected trajectory. Unknown or interrupted usage makes the reported sum a lower bound.",
    "mean_definition": "Reported total divided by all 69 evaluated instances; incomplete usage remains a lower bound, not a known-only mean."
  },
  "tracks": {
    "tool-free": {
      "label": "Without tools",
      "configurations": 14,
      "protocol": "One response per instance \u00b7 128k output tokens, including reasoning."
    },
    "tool-assisted": {
      "label": "Code + web",
      "configurations": 3,
      "protocol": "Per instance: 4 CPU threads \u00b7 16 GiB memory \u00b7 2 hours \u00b7 no cumulative output-token cap."
    }
  }
}