[
  {
    "key": "jev",
    "name": "Jev",
    "model": "~typesafe/jev-latest",
    "n": 77,
    "correct": 58,
    "errors": 19,
    "accuracy": 0.7532467532467533,
    "accuracy_ci95": [
      0.6464643724401389,
      0.8359605177154248
    ],
    "mean_confidence": 0.8742857142857143,
    "confidence_gap": 0.12103896103896106,
    "brier": 0.17065974025974026,
    "ece10": 0.13844155844155842,
    "auroc": 0.8107985480943738,
    "coverage_at_5pct_observed_error": 0.5454545454545454,
    "threshold_at_5pct_observed_error": 0.97,
    "observed_error_at_threshold": 0.047619047619047616,
    "cost_77": 0.005518338,
    "cost_per_1000": 0.07166672727272727,
    "input_tokens": 131389,
    "output_tokens": 63789,
    "reasoning_tokens": 0,
    "median_latency_s": 0.38717197200094233,
    "multiclass_brier": 0.39156493506493506
  },
  {
    "key": "luna",
    "name": "GPT-5.6 Luna",
    "model": "openai/gpt-5.6-luna",
    "n": 77,
    "correct": 64,
    "errors": 13,
    "accuracy": 0.8311688311688312,
    "accuracy_ci95": [
      0.7322690482024222,
      0.8985942696933148
    ],
    "mean_confidence": 0.964025974025974,
    "confidence_gap": 0.13285714285714278,
    "brier": 0.13973376623376624,
    "ece10": 0.1359740259740259,
    "auroc": 0.7542067307692307,
    "coverage_at_5pct_observed_error": 0.07792207792207792,
    "threshold_at_5pct_observed_error": 1.0,
    "observed_error_at_threshold": 0.0,
    "cost_77": 0.00886545,
    "cost_per_1000": 0.11513571428571429,
    "input_tokens": 37365,
    "output_tokens": 1235,
    "reasoning_tokens": 0,
    "median_latency_s": 0.9702107589982916
  },
  {
    "key": "sol",
    "name": "GPT-5.6 Sol",
    "model": "openai/gpt-5.6-sol",
    "n": 77,
    "correct": 64,
    "errors": 13,
    "accuracy": 0.8311688311688312,
    "accuracy_ci95": [
      0.7322690482024222,
      0.8985942696933148
    ],
    "mean_confidence": 0.938051948051948,
    "confidence_gap": 0.1068831168831168,
    "brier": 0.1392038961038961,
    "ece10": 0.1266233766233768,
    "auroc": 0.7325721153846154,
    "coverage_at_5pct_observed_error": 0.14285714285714285,
    "threshold_at_5pct_observed_error": 1.0,
    "observed_error_at_threshold": 0.0,
    "cost_77": 0.0862092,
    "cost_per_1000": 1.1196,
    "input_tokens": 37365,
    "output_tokens": 1235,
    "reasoning_tokens": 0,
    "median_latency_s": 1.448459234998154
  },
  {
    "key": "astra",
    "name": "GPT-6 Astra",
    "model": "openai/gpt-6-astra",
    "n": 77,
    "correct": 68,
    "errors": 9,
    "accuracy": 0.8831168831168831,
    "accuracy_ci95": [
      0.7925471803683823,
      0.9372750893541368
    ],
    "mean_confidence": 0.9397402597402598,
    "confidence_gap": 0.05662337662337669,
    "brier": 0.08801298701298702,
    "ece10": 0.0574025974025974,
    "auroc": 0.8398692810457516,
    "coverage_at_5pct_observed_error": 0.7922077922077922,
    "threshold_at_5pct_observed_error": 0.91,
    "observed_error_at_threshold": 0.04918032786885246,
    "cost_77": 0.483912,
    "cost_per_1000": 6.284571428571429,
    "input_tokens": 37365,
    "output_tokens": 2303,
    "reasoning_tokens": 1022,
    "median_latency_s": 1.7235131970010116
  },
  {
    "key": "opus",
    "name": "Claude Opus 5",
    "model": "anthropic/claude-opus-5",
    "n": 77,
    "correct": 61,
    "errors": 16,
    "accuracy": 0.7922077922077922,
    "accuracy_ci95": [
      0.6887937633995327,
      0.8678503406261175
    ],
    "mean_confidence": 0.7990909090909091,
    "confidence_gap": 0.006883116883116824,
    "brier": 0.12378831168831168,
    "ece10": 0.044545454545454444,
    "auroc": 0.8427254098360656,
    "coverage_at_5pct_observed_error": 0.5974025974025974,
    "threshold_at_5pct_observed_error": 0.76,
    "observed_error_at_threshold": 0.043478260869565216,
    "cost_77": 0.37442295,
    "cost_per_1000": 4.862635714285713,
    "input_tokens": 67091,
    "output_tokens": 1710,
    "reasoning_tokens": 0,
    "median_latency_s": 2.3109009920008248
  },
  {
    "key": "fable",
    "name": "Claude Fable 5.1",
    "model": "anthropic/claude-fable-5.1",
    "n": 77,
    "correct": 64,
    "errors": 13,
    "accuracy": 0.8311688311688312,
    "accuracy_ci95": [
      0.7322690482024222,
      0.8985942696933148
    ],
    "mean_confidence": 0.8355844155844155,
    "confidence_gap": 0.004415584415584317,
    "brier": 0.11974545454545454,
    "ece10": 0.07558441558441553,
    "auroc": 0.7860576923076923,
    "coverage_at_5pct_observed_error": 0.6233766233766234,
    "threshold_at_5pct_observed_error": 0.82,
    "observed_error_at_threshold": 0.020833333333333332,
    "cost_77": 0.7613595,
    "cost_per_1000": 9.887785714285714,
    "input_tokens": 67245,
    "output_tokens": 1932,
    "reasoning_tokens": 0,
    "median_latency_s": 3.332588430999749
  },
  {
    "key": "gemini",
    "name": "Gemini 3.8 Flash",
    "model": "google/gemini-3.8-flash",
    "n": 77,
    "correct": 67,
    "errors": 10,
    "accuracy": 0.8701298701298701,
    "accuracy_ci95": [
      0.7771800466752254,
      0.9279024850905981
    ],
    "mean_confidence": 0.9397402597402598,
    "confidence_gap": 0.06961038961038968,
    "brier": 0.10820779220779221,
    "ece10": 0.08909090909090911,
    "auroc": 0.7664179104477612,
    "coverage_at_5pct_observed_error": 0.37662337662337664,
    "threshold_at_5pct_observed_error": 0.98,
    "observed_error_at_threshold": 0.034482758620689655,
    "cost_77": 0.091792305,
    "cost_per_1000": 1.1921078571428572,
    "input_tokens": 47471,
    "output_tokens": 15231,
    "reasoning_tokens": 13020,
    "median_latency_s": 2.8198920680006268
  },
  {
    "key": "sonnet",
    "name": "Claude Sonnet 5",
    "model": "anthropic/claude-sonnet-5",
    "n": 77,
    "correct": 60,
    "errors": 17,
    "accuracy": 0.7792207792207793,
    "accuracy_ci95": [
      0.6745672277887955,
      0.8573371382801591
    ],
    "mean_confidence": 0.8155844155844156,
    "confidence_gap": 0.036363636363636376,
    "brier": 0.1523844155844156,
    "ece10": 0.1306493506493507,
    "auroc": 0.7475490196078431,
    "coverage_at_5pct_observed_error": 0.1038961038961039,
    "threshold_at_5pct_observed_error": 0.97,
    "observed_error_at_threshold": 0.0,
    "cost_77": 0.15048198,
    "cost_per_1000": 1.9543114285714287,
    "input_tokens": 67091,
    "output_tokens": 1782,
    "reasoning_tokens": 0,
    "median_latency_s": 2.016437911999674
  },
  {
    "key": "deepseek",
    "name": "DeepSeek V4.1 Flash",
    "model": "deepseek/deepseek-v4.1-flash",
    "n": 77,
    "correct": 61,
    "errors": 16,
    "accuracy": 0.7922077922077922,
    "accuracy_ci95": [
      0.6887937633995327,
      0.8678503406261175
    ],
    "mean_confidence": 0.8445454545454545,
    "confidence_gap": 0.05233766233766224,
    "brier": 0.15785324675324675,
    "ece10": 0.09467532467532455,
    "auroc": 0.6659836065573771,
    "coverage_at_5pct_observed_error": 0.03896103896103896,
    "threshold_at_5pct_observed_error": 0.98,
    "observed_error_at_threshold": 0.0,
    "cost_77": 0.00653570478,
    "cost_per_1000": 0.08487928285714286,
    "input_tokens": 42846,
    "output_tokens": 1309,
    "reasoning_tokens": 0,
    "median_latency_s": 0.48846277999837184
  }
]