{
  "published_at": "2026-09-20T10:16:45.261618+00:00",
  "source_sha256": "e9899119a5a33099fa1ba7f4b5cb2aa6a507cef01a05d94a36a42bf3af50903f",
  "manifest": {
    "campaign": "speed-cost-20260920",
    "models": [
      "anthropic:claude-opus-4-6",
      "anthropic:claude-sonnet-4-5-20250929",
      "anthropic:claude-haiku-4-5-20251001",
      "openai:gpt-5.5",
      "openai:gpt-4o",
      "openai:gpt-4o-mini",
      "gemini:gemini-3.1-flash-lite",
      "gemini:gemini-3.5-flash-lite",
      "gemini:gemini-3.8-flash"
    ],
    "trials": 3,
    "concurrency": 1,
    "judge": false,
    "cache_policy": "provider cache as observed; cold cache not guaranteed",
    "tests": [
      {
        "id": "perf.simple-turn-budget",
        "title": "Simple no-tool turn within 045 budget",
        "sha256": "121e978efb7b8ea12859cd8249ef77e2f3b069f52b4c63fd9ca91b88ad8f43f3"
      },
      {
        "id": "perf.one-tool-turn-budget",
        "title": "One tool call turn under 20s",
        "sha256": "9ac387d01197212589f6f877eb2075751dd8fb2dad127653afa50939ecf7718b"
      },
      {
        "id": "perf.second-turn-cache",
        "title": "Second turn in a thread \u2192 cache hit, fast",
        "sha256": "54b7d3b9aaeb5e3b6c408e386e3aa09dbd262f6dbc8342158627aee90fbe3f68"
      },
      {
        "id": "perf.loaded-history-simple",
        "title": "Simple question stays fast on a loaded history",
        "sha256": "a8e1aef69d76d7ca2517354e53e9bf570b2bc9e878809175d9ae8bb1f1e44450"
      }
    ]
  },
  "completed_cells": 108,
  "expected_cells": 108,
  "rows": [
    {
      "model": "anthropic:claude-opus-4-6",
      "test_id": "perf.simple-turn-budget",
      "title": "Simple no-tool turn within 045 budget",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.46900125,
      "known_cost_usd": 0.46900125,
      "usd_per_attempt": 0.15633375,
      "usd_per_quality_success": 0.15633375,
      "trial_s": {
        "n": 3,
        "mean": 4.419366666666667,
        "median": 3.8873,
        "p95": 6.0487
      },
      "turn_s": {
        "n": 3,
        "mean": 4.419366666666667,
        "median": 3.8873,
        "p95": 6.0487
      },
      "first_activity_s": {
        "n": 3,
        "mean": 4.1541999999999994,
        "median": 3.7695,
        "p95": 5.5782
      },
      "first_text_s": {
        "n": 3,
        "mean": 4.1541999999999994,
        "median": 3.7695,
        "p95": 5.5782
      },
      "input_tokens": 74934,
      "output_tokens": 27,
      "cache_read_tokens": 0
    },
    {
      "model": "anthropic:claude-opus-4-6",
      "test_id": "perf.one-tool-turn-budget",
      "title": "One tool call turn under 20s",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.5855954999999999,
      "known_cost_usd": 0.5855954999999999,
      "usd_per_attempt": 0.19519849999999997,
      "usd_per_quality_success": 0.19519849999999997,
      "trial_s": {
        "n": 3,
        "mean": 11.457566666666667,
        "median": 9.627600000000001,
        "p95": 15.3635
      },
      "turn_s": {
        "n": 3,
        "mean": 11.457566666666667,
        "median": 9.627600000000001,
        "p95": 15.3635
      },
      "first_activity_s": {
        "n": 3,
        "mean": 5.566766666666667,
        "median": 3.8038000000000003,
        "p95": 9.3474
      },
      "first_text_s": {
        "n": 3,
        "mean": 11.1942,
        "median": 9.343200000000001,
        "p95": 15.023299999999999
      },
      "input_tokens": 232524,
      "output_tokens": 378,
      "cache_read_tokens": 152541
    },
    {
      "model": "anthropic:claude-opus-4-6",
      "test_id": "perf.second-turn-cache",
      "title": "Second turn in a thread \u2192 cache hit, fast",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.08421725,
      "known_cost_usd": 0.08421725,
      "usd_per_attempt": 0.028072416666666666,
      "usd_per_quality_success": 0.028072416666666666,
      "trial_s": {
        "n": 3,
        "mean": 7.4017,
        "median": 7.835299999999999,
        "p95": 8.6509
      },
      "turn_s": {
        "n": 6,
        "mean": 3.70085,
        "median": 3.28055,
        "p95": 5.4062
      },
      "first_activity_s": {
        "n": 6,
        "mean": 3.605183333333333,
        "median": 3.1624,
        "p95": 5.2906
      },
      "first_text_s": {
        "n": 6,
        "mean": 3.6052,
        "median": 3.1624,
        "p95": 5.2907
      },
      "input_tokens": 149897,
      "output_tokens": 129,
      "cache_read_tokens": 148842
    },
    {
      "model": "anthropic:claude-opus-4-6",
      "test_id": "perf.loaded-history-simple",
      "title": "Simple question stays fast on a loaded history",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.91899875,
      "known_cost_usd": 0.91899875,
      "usd_per_attempt": 0.3063329166666667,
      "usd_per_quality_success": 0.3063329166666667,
      "trial_s": {
        "n": 3,
        "mean": 17.7962,
        "median": 18.929299999999998,
        "p95": 19.6049
      },
      "turn_s": {
        "n": 12,
        "mean": 4.44905,
        "median": 4.711550000000001,
        "p95": 6.02
      },
      "first_activity_s": {
        "n": 12,
        "mean": 3.477675,
        "median": 3.4606000000000003,
        "p95": 4.5324
      },
      "first_text_s": {
        "n": 12,
        "mean": 3.4777,
        "median": 3.4606000000000003,
        "p95": 4.5324
      },
      "input_tokens": 328256,
      "output_tokens": 856,
      "cache_read_tokens": 200690
    },
    {
      "model": "anthropic:claude-opus-4-6",
      "test_id": "ALL",
      "title": "All selected tests",
      "expected": 12,
      "attempted": 12,
      "executed": 12,
      "passed": 12,
      "quality_passed": 12,
      "metered": 12,
      "cost_usd": 2.05781275,
      "known_cost_usd": 2.05781275,
      "usd_per_attempt": 0.17148439583333333,
      "usd_per_quality_success": 0.17148439583333333,
      "trial_s": {
        "n": 12,
        "mean": 10.268708333333333,
        "median": 9.01625,
        "p95": 18.929299999999998
      },
      "turn_s": {
        "n": 24,
        "mean": 5.134354166666666,
        "median": 4.55775,
        "p95": 9.627600000000001
      },
      "first_activity_s": {
        "n": 24,
        "mean": 3.8552541666666666,
        "median": 3.5134499999999997,
        "p95": 5.5782
      },
      "first_text_s": {
        "n": 24,
        "mean": 4.5587,
        "median": 3.5491,
        "p95": 9.343200000000001
      },
      "input_tokens": 785611,
      "output_tokens": 1390,
      "cache_read_tokens": 502073
    },
    {
      "model": "anthropic:claude-sonnet-4-5-20250929",
      "test_id": "perf.simple-turn-budget",
      "title": "Simple no-tool turn within 045 budget",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.2813595,
      "known_cost_usd": 0.2813595,
      "usd_per_attempt": 0.0937865,
      "usd_per_quality_success": 0.0937865,
      "trial_s": {
        "n": 3,
        "mean": 2.3390666666666666,
        "median": 2.3205999999999998,
        "p95": 2.3789000000000002
      },
      "turn_s": {
        "n": 3,
        "mean": 2.3390666666666666,
        "median": 2.3205999999999998,
        "p95": 2.3789000000000002
      },
      "first_activity_s": {
        "n": 3,
        "mean": 2.1632333333333333,
        "median": 2.2081999999999997,
        "p95": 2.2251
      },
      "first_text_s": {
        "n": 3,
        "mean": 2.1632333333333333,
        "median": 2.2081999999999997,
        "p95": 2.2251
      },
      "input_tokens": 74931,
      "output_tokens": 25,
      "cache_read_tokens": 0
    },
    {
      "model": "anthropic:claude-sonnet-4-5-20250929",
      "test_id": "perf.one-tool-turn-budget",
      "title": "One tool call turn under 20s",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.3909834,
      "known_cost_usd": 0.3909834,
      "usd_per_attempt": 0.1303278,
      "usd_per_quality_success": 0.1303278,
      "trial_s": {
        "n": 3,
        "mean": 8.3071,
        "median": 8.4931,
        "p95": 8.8175
      },
      "turn_s": {
        "n": 3,
        "mean": 8.3071,
        "median": 8.4931,
        "p95": 8.8175
      },
      "first_activity_s": {
        "n": 3,
        "mean": 1.9669333333333332,
        "median": 1.9078,
        "p95": 2.2771999999999997
      },
      "first_text_s": {
        "n": 3,
        "mean": 1.9669333333333332,
        "median": 1.9078,
        "p95": 2.2771999999999997
      },
      "input_tokens": 312736,
      "output_tokens": 607,
      "cache_read_tokens": 229228
    },
    {
      "model": "anthropic:claude-sonnet-4-5-20250929",
      "test_id": "perf.second-turn-cache",
      "title": "Second turn in a thread \u2192 cache hit, fast",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.04907355,
      "known_cost_usd": 0.04907355,
      "usd_per_attempt": 0.01635785,
      "usd_per_quality_success": 0.01635785,
      "trial_s": {
        "n": 3,
        "mean": 3.5615666666666668,
        "median": 3.5266,
        "p95": 3.7819000000000003
      },
      "turn_s": {
        "n": 6,
        "mean": 1.7807833333333334,
        "median": 1.8765,
        "p95": 2.1264000000000003
      },
      "first_activity_s": {
        "n": 6,
        "mean": 1.6806,
        "median": 1.77785,
        "p95": 2.001
      },
      "first_text_s": {
        "n": 6,
        "mean": 1.6806,
        "median": 1.77785,
        "p95": 2.001
      },
      "input_tokens": 149887,
      "output_tokens": 33,
      "cache_read_tokens": 148836
    },
    {
      "model": "anthropic:claude-sonnet-4-5-20250929",
      "test_id": "perf.loaded-history-simple",
      "title": "Simple question stays fast on a loaded history",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.5444997,
      "known_cost_usd": 0.5444997,
      "usd_per_attempt": 0.18149990000000002,
      "usd_per_quality_success": 0.18149990000000002,
      "trial_s": {
        "n": 3,
        "mean": 11.046966666666668,
        "median": 10.750200000000001,
        "p95": 12.3327
      },
      "turn_s": {
        "n": 12,
        "mean": 2.7617416666666665,
        "median": 2.96095,
        "p95": 3.7546
      },
      "first_activity_s": {
        "n": 12,
        "mean": 1.7414166666666666,
        "median": 1.70015,
        "p95": 2.4642
      },
      "first_text_s": {
        "n": 12,
        "mean": 1.7414166666666666,
        "median": 1.70015,
        "p95": 2.4642
      },
      "input_tokens": 328033,
      "output_tokens": 432,
      "cache_read_tokens": 200604
    },
    {
      "model": "anthropic:claude-sonnet-4-5-20250929",
      "test_id": "ALL",
      "title": "All selected tests",
      "expected": 12,
      "attempted": 12,
      "executed": 12,
      "passed": 12,
      "quality_passed": 12,
      "metered": 12,
      "cost_usd": 1.26591615,
      "known_cost_usd": 1.26591615,
      "usd_per_attempt": 0.1054930125,
      "usd_per_quality_success": 0.1054930125,
      "trial_s": {
        "n": 12,
        "mean": 6.313675,
        "median": 5.6963,
        "p95": 10.750200000000001
      },
      "turn_s": {
        "n": 24,
        "mean": 3.1568375,
        "median": 2.3771000000000004,
        "p95": 8.4931
      },
      "first_activity_s": {
        "n": 24,
        "mean": 1.8071291666666667,
        "median": 1.8023500000000001,
        "p95": 2.4642
      },
      "first_text_s": {
        "n": 24,
        "mean": 1.8071291666666667,
        "median": 1.8023500000000001,
        "p95": 2.4642
      },
      "input_tokens": 865587,
      "output_tokens": 1097,
      "cache_read_tokens": 578668
    },
    {
      "model": "anthropic:claude-haiku-4-5-20251001",
      "test_id": "perf.simple-turn-budget",
      "title": "Simple no-tool turn within 045 budget",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.0650714,
      "known_cost_usd": 0.0650714,
      "usd_per_attempt": 0.02169046666666667,
      "usd_per_quality_success": 0.02169046666666667,
      "trial_s": {
        "n": 3,
        "mean": 1.3456000000000001,
        "median": 1.2878,
        "p95": 1.6059
      },
      "turn_s": {
        "n": 3,
        "mean": 1.3456000000000001,
        "median": 1.2878,
        "p95": 1.6059
      },
      "first_activity_s": {
        "n": 3,
        "mean": 1.2872000000000001,
        "median": 1.2654,
        "p95": 1.512
      },
      "first_text_s": {
        "n": 3,
        "mean": 1.2872000000000001,
        "median": 1.2654,
        "p95": 1.512
      },
      "input_tokens": 74931,
      "output_tokens": 26,
      "cache_read_tokens": 24974
    },
    {
      "model": "anthropic:claude-haiku-4-5-20251001",
      "test_id": "perf.one-tool-turn-budget",
      "title": "One tool call turn under 20s",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 1,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.13350585,
      "known_cost_usd": 0.13350585,
      "usd_per_attempt": 0.044501950000000005,
      "usd_per_quality_success": 0.044501950000000005,
      "trial_s": {
        "n": 3,
        "mean": 7.561,
        "median": 7.515899999999999,
        "p95": 8.2326
      },
      "turn_s": {
        "n": 3,
        "mean": 7.561,
        "median": 7.515899999999999,
        "p95": 8.2326
      },
      "first_activity_s": {
        "n": 3,
        "mean": 2.0312333333333332,
        "median": 1.9404000000000001,
        "p95": 2.2291999999999996
      },
      "first_text_s": {
        "n": 3,
        "mean": 2.9987999999999997,
        "median": 3.0862,
        "p95": 3.1266
      },
      "input_tokens": 360181,
      "output_tokens": 726,
      "cache_read_tokens": 278551
    },
    {
      "model": "anthropic:claude-haiku-4-5-20251001",
      "test_id": "perf.second-turn-cache",
      "title": "Second turn in a thread \u2192 cache hit, fast",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.016487850000000002,
      "known_cost_usd": 0.016487850000000002,
      "usd_per_attempt": 0.00549595,
      "usd_per_quality_success": 0.00549595,
      "trial_s": {
        "n": 3,
        "mean": 2.9341666666666666,
        "median": 2.7709,
        "p95": 3.7196
      },
      "turn_s": {
        "n": 6,
        "mean": 1.4670833333333333,
        "median": 1.25175,
        "p95": 2.6881
      },
      "first_activity_s": {
        "n": 6,
        "mean": 1.4179833333333334,
        "median": 1.20485,
        "p95": 2.6443000000000003
      },
      "first_text_s": {
        "n": 6,
        "mean": 1.4179833333333334,
        "median": 1.20485,
        "p95": 2.6443000000000003
      },
      "input_tokens": 149907,
      "output_tokens": 54,
      "cache_read_tokens": 148836
    },
    {
      "model": "anthropic:claude-haiku-4-5-20251001",
      "test_id": "perf.loaded-history-simple",
      "title": "Simple question stays fast on a loaded history",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.18181015,
      "known_cost_usd": 0.18181015,
      "usd_per_attempt": 0.06060338333333334,
      "usd_per_quality_success": 0.06060338333333334,
      "trial_s": {
        "n": 3,
        "mean": 7.5323,
        "median": 7.596,
        "p95": 7.693
      },
      "turn_s": {
        "n": 12,
        "mean": 1.883075,
        "median": 1.8293000000000001,
        "p95": 2.2379000000000002
      },
      "first_activity_s": {
        "n": 12,
        "mean": 1.42615,
        "median": 1.44505,
        "p95": 1.8169000000000002
      },
      "first_text_s": {
        "n": 12,
        "mean": 1.4261666666666666,
        "median": 1.44505,
        "p95": 1.8169000000000002
      },
      "input_tokens": 328150,
      "output_tokens": 474,
      "cache_read_tokens": 200644
    },
    {
      "model": "anthropic:claude-haiku-4-5-20251001",
      "test_id": "ALL",
      "title": "All selected tests",
      "expected": 12,
      "attempted": 12,
      "executed": 12,
      "passed": 10,
      "quality_passed": 12,
      "metered": 12,
      "cost_usd": 0.39687525,
      "known_cost_usd": 0.39687525,
      "usd_per_attempt": 0.0330729375,
      "usd_per_quality_success": 0.0330729375,
      "trial_s": {
        "n": 12,
        "mean": 4.843266666666667,
        "median": 5.32705,
        "p95": 7.693
      },
      "turn_s": {
        "n": 24,
        "mean": 2.4216333333333333,
        "median": 1.7403499999999998,
        "p95": 7.515899999999999
      },
      "first_activity_s": {
        "n": 24,
        "mean": 1.482375,
        "median": 1.4403,
        "p95": 2.2291999999999996
      },
      "first_text_s": {
        "n": 24,
        "mean": 1.6033291666666667,
        "median": 1.4403,
        "p95": 3.0862
      },
      "input_tokens": 913169,
      "output_tokens": 1280,
      "cache_read_tokens": 653005
    },
    {
      "model": "openai:gpt-5.5",
      "test_id": "perf.simple-turn-budget",
      "title": "Simple no-tool turn within 045 budget",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.29319,
      "known_cost_usd": 0.29319,
      "usd_per_attempt": 0.09773,
      "usd_per_quality_success": 0.09773,
      "trial_s": {
        "n": 3,
        "mean": 2.8386,
        "median": 2.5098000000000003,
        "p95": 3.5439000000000003
      },
      "turn_s": {
        "n": 3,
        "mean": 2.8386,
        "median": 2.5098000000000003,
        "p95": 3.5439000000000003
      },
      "first_activity_s": {
        "n": 3,
        "mean": 2.723333333333333,
        "median": 2.3886999999999996,
        "p95": 3.4206999999999996
      },
      "first_text_s": {
        "n": 3,
        "mean": 2.723333333333333,
        "median": 2.3886999999999996,
        "p95": 3.4206999999999996
      },
      "input_tokens": 57828,
      "output_tokens": 135,
      "cache_read_tokens": 0
    },
    {
      "model": "openai:gpt-5.5",
      "test_id": "perf.one-tool-turn-budget",
      "title": "One tool call turn under 20s",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.257352,
      "known_cost_usd": 0.257352,
      "usd_per_attempt": 0.08578400000000001,
      "usd_per_quality_success": 0.08578400000000001,
      "trial_s": {
        "n": 3,
        "mean": 6.834433333333333,
        "median": 6.4958,
        "p95": 7.9817
      },
      "turn_s": {
        "n": 3,
        "mean": 6.834433333333333,
        "median": 6.4958,
        "p95": 7.9817
      },
      "first_activity_s": {
        "n": 3,
        "mean": 3.2857000000000003,
        "median": 3.1458000000000004,
        "p95": 4.0703000000000005
      },
      "first_text_s": {
        "n": 3,
        "mean": 6.762733333333333,
        "median": 6.4364,
        "p95": 7.913
      },
      "input_tokens": 178308,
      "output_tokens": 518,
      "cache_read_tokens": 144384
    },
    {
      "model": "openai:gpt-5.5",
      "test_id": "perf.second-turn-cache",
      "title": "Second turn in a thread \u2192 cache hit, fast",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.15356,
      "known_cost_usd": 0.15356,
      "usd_per_attempt": 0.051186666666666665,
      "usd_per_quality_success": 0.051186666666666665,
      "trial_s": {
        "n": 3,
        "mean": 4.386966666666666,
        "median": 4.4961,
        "p95": 4.9418999999999995
      },
      "turn_s": {
        "n": 6,
        "mean": 2.1934833333333335,
        "median": 2.17785,
        "p95": 2.7164
      },
      "first_activity_s": {
        "n": 6,
        "mean": 2.1473,
        "median": 2.16435,
        "p95": 2.6579
      },
      "first_text_s": {
        "n": 6,
        "mean": 2.1473,
        "median": 2.16435,
        "p95": 2.6579
      },
      "input_tokens": 115696,
      "output_tokens": 140,
      "cache_read_tokens": 95360
    },
    {
      "model": "openai:gpt-5.5",
      "test_id": "perf.loaded-history-simple",
      "title": "Simple question stays fast on a loaded history",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.490901,
      "known_cost_usd": 0.490901,
      "usd_per_attempt": 0.16363366666666665,
      "usd_per_quality_success": 0.16363366666666665,
      "trial_s": {
        "n": 3,
        "mean": 8.689833333333333,
        "median": 8.786200000000001,
        "p95": 8.8452
      },
      "turn_s": {
        "n": 12,
        "mean": 2.172458333333333,
        "median": 2.0818,
        "p95": 2.6477
      },
      "first_activity_s": {
        "n": 12,
        "mean": 1.8101666666666667,
        "median": 1.6908,
        "p95": 2.2464
      },
      "first_text_s": {
        "n": 12,
        "mean": 1.8101666666666667,
        "median": 1.6908,
        "p95": 2.2464
      },
      "input_tokens": 252547,
      "output_tokens": 365,
      "cache_read_tokens": 173952
    },
    {
      "model": "openai:gpt-5.5",
      "test_id": "ALL",
      "title": "All selected tests",
      "expected": 12,
      "attempted": 12,
      "executed": 12,
      "passed": 12,
      "quality_passed": 12,
      "metered": 12,
      "cost_usd": 1.195003,
      "known_cost_usd": 1.195003,
      "usd_per_attempt": 0.09958358333333334,
      "usd_per_quality_success": 0.09958358333333334,
      "trial_s": {
        "n": 12,
        "mean": 5.687458333333334,
        "median": 5.48385,
        "p95": 8.786200000000001
      },
      "turn_s": {
        "n": 24,
        "mean": 2.843729166666667,
        "median": 2.24185,
        "p95": 6.4958
      },
      "first_activity_s": {
        "n": 24,
        "mean": 2.1930375,
        "median": 2.14115,
        "p95": 3.4206999999999996
      },
      "first_text_s": {
        "n": 24,
        "mean": 2.627666666666667,
        "median": 2.14115,
        "p95": 6.4364
      },
      "input_tokens": 604379,
      "output_tokens": 1158,
      "cache_read_tokens": 413696
    },
    {
      "model": "openai:gpt-4o",
      "test_id": "perf.simple-turn-budget",
      "title": "Simple no-tool turn within 045 budget",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.120245,
      "known_cost_usd": 0.120245,
      "usd_per_attempt": 0.04008166666666667,
      "usd_per_quality_success": 0.04008166666666667,
      "trial_s": {
        "n": 3,
        "mean": 2.226166666666667,
        "median": 2.1412,
        "p95": 2.9414000000000002
      },
      "turn_s": {
        "n": 3,
        "mean": 2.226166666666667,
        "median": 2.1412,
        "p95": 2.9414000000000002
      },
      "first_activity_s": {
        "n": 3,
        "mean": 2.0554,
        "median": 2.0292,
        "p95": 2.6052
      },
      "first_text_s": {
        "n": 3,
        "mean": 2.0554,
        "median": 2.0292,
        "p95": 2.6052
      },
      "input_tokens": 57558,
      "output_tokens": 19,
      "cache_read_tokens": 19072
    },
    {
      "model": "openai:gpt-4o",
      "test_id": "perf.one-tool-turn-budget",
      "title": "One tool call turn under 20s",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 0,
      "quality_passed": 0,
      "metered": 3,
      "cost_usd": 0.20832499999999998,
      "known_cost_usd": 0.20832499999999998,
      "usd_per_attempt": 0.06944166666666667,
      "usd_per_quality_success": null,
      "trial_s": {
        "n": 3,
        "mean": 4.061366666666666,
        "median": 4.1008000000000004,
        "p95": 4.7245
      },
      "turn_s": {
        "n": 3,
        "mean": 4.061366666666666,
        "median": 4.1008000000000004,
        "p95": 4.7245
      },
      "first_activity_s": {
        "n": 3,
        "mean": 2.4305,
        "median": 2.2046,
        "p95": 3.0541
      },
      "first_text_s": {
        "n": 3,
        "mean": 3.8185,
        "median": 3.9178,
        "p95": 4.4475
      },
      "input_tokens": 115302,
      "output_tokens": 151,
      "cache_read_tokens": 65152
    },
    {
      "model": "openai:gpt-4o",
      "test_id": "perf.second-turn-cache",
      "title": "Second turn in a thread \u2192 cache hit, fast",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.1829575,
      "known_cost_usd": 0.1829575,
      "usd_per_attempt": 0.06098583333333333,
      "usd_per_quality_success": 0.06098583333333333,
      "trial_s": {
        "n": 3,
        "mean": 3.2812666666666668,
        "median": 3.3026,
        "p95": 3.3118999999999996
      },
      "turn_s": {
        "n": 6,
        "mean": 1.6406333333333334,
        "median": 1.6513,
        "p95": 2.0179
      },
      "first_activity_s": {
        "n": 6,
        "mean": 1.5499333333333334,
        "median": 1.55395,
        "p95": 1.9304000000000001
      },
      "first_text_s": {
        "n": 6,
        "mean": 1.5499333333333334,
        "median": 1.55395,
        "p95": 1.9304000000000001
      },
      "input_tokens": 115159,
      "output_tokens": 34,
      "cache_read_tokens": 84224
    },
    {
      "model": "openai:gpt-4o",
      "test_id": "perf.loaded-history-simple",
      "title": "Simple question stays fast on a loaded history",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.4343725,
      "known_cost_usd": 0.4343725,
      "usd_per_attempt": 0.14479083333333334,
      "usd_per_quality_success": 0.14479083333333334,
      "trial_s": {
        "n": 3,
        "mean": 7.362133333333333,
        "median": 7.0663,
        "p95": 8.1709
      },
      "turn_s": {
        "n": 12,
        "mean": 1.8405333333333334,
        "median": 1.8806,
        "p95": 2.3045
      },
      "first_activity_s": {
        "n": 12,
        "mean": 1.6163833333333333,
        "median": 1.61735,
        "p95": 2.0202
      },
      "first_text_s": {
        "n": 12,
        "mean": 1.6163916666666667,
        "median": 1.6174,
        "p95": 2.0202
      },
      "input_tokens": 251401,
      "output_tokens": 315,
      "cache_read_tokens": 157824
    },
    {
      "model": "openai:gpt-4o",
      "test_id": "ALL",
      "title": "All selected tests",
      "expected": 12,
      "attempted": 12,
      "executed": 12,
      "passed": 9,
      "quality_passed": 9,
      "metered": 12,
      "cost_usd": 0.9459,
      "known_cost_usd": 0.9459,
      "usd_per_attempt": 0.07882499999999999,
      "usd_per_quality_success": 0.1051,
      "trial_s": {
        "n": 12,
        "mean": 4.232733333333333,
        "median": 3.33535,
        "p95": 7.0663
      },
      "turn_s": {
        "n": 24,
        "mean": 2.1163666666666665,
        "median": 1.8806,
        "p95": 4.1008000000000004
      },
      "first_activity_s": {
        "n": 24,
        "mean": 1.7564125,
        "median": 1.6777000000000002,
        "p95": 2.6052
      },
      "first_text_s": {
        "n": 24,
        "mean": 1.9299166666666667,
        "median": 1.67775,
        "p95": 3.9178
      },
      "input_tokens": 539420,
      "output_tokens": 519,
      "cache_read_tokens": 326272
    },
    {
      "model": "openai:gpt-4o-mini",
      "test_id": "perf.simple-turn-budget",
      "title": "Simple no-tool turn within 045 budget",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.0072141,
      "known_cost_usd": 0.0072141,
      "usd_per_attempt": 0.0024047,
      "usd_per_quality_success": 0.0024047,
      "trial_s": {
        "n": 3,
        "mean": 1.7491333333333334,
        "median": 1.7227999999999999,
        "p95": 2.0199000000000003
      },
      "turn_s": {
        "n": 3,
        "mean": 1.7491333333333334,
        "median": 1.7227999999999999,
        "p95": 2.0199000000000003
      },
      "first_activity_s": {
        "n": 3,
        "mean": 1.6555,
        "median": 1.6938,
        "p95": 1.9187
      },
      "first_text_s": {
        "n": 3,
        "mean": 1.6555,
        "median": 1.6938,
        "p95": 1.9187
      },
      "input_tokens": 57558,
      "output_tokens": 18,
      "cache_read_tokens": 19072
    },
    {
      "model": "openai:gpt-4o-mini",
      "test_id": "perf.one-tool-turn-budget",
      "title": "One tool call turn under 20s",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 0,
      "quality_passed": 0,
      "metered": 3,
      "cost_usd": 0.0220185,
      "known_cost_usd": 0.0220185,
      "usd_per_attempt": 0.0073395,
      "usd_per_quality_success": null,
      "trial_s": {
        "n": 3,
        "mean": 5.4401,
        "median": 4.9203,
        "p95": 7.0062
      },
      "turn_s": {
        "n": 3,
        "mean": 5.4401,
        "median": 4.9203,
        "p95": 7.0062
      },
      "first_activity_s": {
        "n": 3,
        "mean": 1.9474666666666667,
        "median": 1.961,
        "p95": 2.0965
      },
      "first_text_s": {
        "n": 3,
        "mean": 5.2091666666666665,
        "median": 4.7035,
        "p95": 6.7405
      },
      "input_tokens": 232290,
      "output_tokens": 289,
      "cache_read_tokens": 173312
    },
    {
      "model": "openai:gpt-4o-mini",
      "test_id": "perf.second-turn-cache",
      "title": "Second turn in a thread \u2192 cache hit, fast",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.01299765,
      "known_cost_usd": 0.01299765,
      "usd_per_attempt": 0.00433255,
      "usd_per_quality_success": 0.00433255,
      "trial_s": {
        "n": 3,
        "mean": 3.262433333333333,
        "median": 3.2481999999999998,
        "p95": 3.4568999999999996
      },
      "turn_s": {
        "n": 6,
        "mean": 1.6312166666666665,
        "median": 1.7095,
        "p95": 1.886
      },
      "first_activity_s": {
        "n": 6,
        "mean": 1.59045,
        "median": 1.6806,
        "p95": 1.8675
      },
      "first_text_s": {
        "n": 6,
        "mean": 1.59045,
        "median": 1.6806,
        "p95": 1.8675
      },
      "input_tokens": 115155,
      "output_tokens": 26,
      "cache_read_tokens": 57216
    },
    {
      "model": "openai:gpt-4o-mini",
      "test_id": "perf.loaded-history-simple",
      "title": "Simple question stays fast on a loaded history",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.02612055,
      "known_cost_usd": 0.02612055,
      "usd_per_attempt": 0.00870685,
      "usd_per_quality_success": 0.00870685,
      "trial_s": {
        "n": 3,
        "mean": 7.990066666666667,
        "median": 7.5967,
        "p95": 9.9007
      },
      "turn_s": {
        "n": 12,
        "mean": 1.9975166666666666,
        "median": 1.86795,
        "p95": 2.9795
      },
      "first_activity_s": {
        "n": 12,
        "mean": 1.7010583333333333,
        "median": 1.62765,
        "p95": 2.47
      },
      "first_text_s": {
        "n": 12,
        "mean": 1.7010666666666667,
        "median": 1.62765,
        "p95": 2.4701
      },
      "input_tokens": 251653,
      "output_tokens": 429,
      "cache_read_tokens": 158464
    },
    {
      "model": "openai:gpt-4o-mini",
      "test_id": "ALL",
      "title": "All selected tests",
      "expected": 12,
      "attempted": 12,
      "executed": 12,
      "passed": 9,
      "quality_passed": 9,
      "metered": 12,
      "cost_usd": 0.0683508,
      "known_cost_usd": 0.0683508,
      "usd_per_attempt": 0.0056959,
      "usd_per_quality_success": 0.007594533333333334,
      "trial_s": {
        "n": 12,
        "mean": 4.610433333333334,
        "median": 3.92535,
        "p95": 7.5967
      },
      "turn_s": {
        "n": 24,
        "mean": 2.3052166666666665,
        "median": 1.7638500000000001,
        "p95": 4.9203
      },
      "first_activity_s": {
        "n": 24,
        "mean": 1.6985125,
        "median": 1.71625,
        "p95": 2.47
      },
      "first_text_s": {
        "n": 24,
        "mean": 2.1062291666666666,
        "median": 1.71625,
        "p95": 4.7035
      },
      "input_tokens": 656656,
      "output_tokens": 762,
      "cache_read_tokens": 408064
    },
    {
      "model": "gemini:gemini-3.1-flash-lite",
      "test_id": "perf.simple-turn-budget",
      "title": "Simple no-tool turn within 045 budget",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.017103,
      "known_cost_usd": 0.017103,
      "usd_per_attempt": 0.005701,
      "usd_per_quality_success": 0.005701,
      "trial_s": {
        "n": 3,
        "mean": 1.8951333333333333,
        "median": 1.953,
        "p95": 2.0805
      },
      "turn_s": {
        "n": 3,
        "mean": 1.8951333333333333,
        "median": 1.953,
        "p95": 2.0805
      },
      "first_activity_s": {
        "n": 3,
        "mean": 1.8454000000000002,
        "median": 1.8657000000000001,
        "p95": 2.0465
      },
      "first_text_s": {
        "n": 3,
        "mean": 1.8454000000000002,
        "median": 1.8657000000000001,
        "p95": 2.0465
      },
      "input_tokens": 68322,
      "output_tokens": 15,
      "cache_read_tokens": 0
    },
    {
      "model": "gemini:gemini-3.1-flash-lite",
      "test_id": "perf.one-tool-turn-budget",
      "title": "One tool call turn under 20s",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 0,
      "quality_passed": 2,
      "metered": 3,
      "cost_usd": 0.051879800000000004,
      "known_cost_usd": 0.051879800000000004,
      "usd_per_attempt": 0.017293266666666668,
      "usd_per_quality_success": 0.025939900000000002,
      "trial_s": {
        "n": 3,
        "mean": 11.756866666666667,
        "median": 10.0526,
        "p95": 16.0395
      },
      "turn_s": {
        "n": 3,
        "mean": 11.756866666666667,
        "median": 10.0526,
        "p95": 16.0395
      },
      "first_activity_s": {
        "n": 3,
        "mean": 1.7729333333333335,
        "median": 1.6947,
        "p95": 1.9634
      },
      "first_text_s": {
        "n": 3,
        "mean": 11.671766666666667,
        "median": 9.9952,
        "p95": 15.9137
      },
      "input_tokens": 687179,
      "output_tokens": 580,
      "cache_read_tokens": 536822
    },
    {
      "model": "gemini:gemini-3.1-flash-lite",
      "test_id": "perf.second-turn-cache",
      "title": "Second turn in a thread \u2192 cache hit, fast",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.0069134750000000005,
      "known_cost_usd": 0.0069134750000000005,
      "usd_per_attempt": 0.002304491666666667,
      "usd_per_quality_success": 0.002304491666666667,
      "trial_s": {
        "n": 3,
        "mean": 3.2421,
        "median": 3.2183,
        "p95": 3.4389000000000003
      },
      "turn_s": {
        "n": 6,
        "mean": 1.62105,
        "median": 1.6350500000000001,
        "p95": 1.9195
      },
      "first_activity_s": {
        "n": 6,
        "mean": 1.5430333333333333,
        "median": 1.5788,
        "p95": 1.8031
      },
      "first_text_s": {
        "n": 6,
        "mean": 1.5430333333333333,
        "median": 1.5788,
        "p95": 1.8031
      },
      "input_tokens": 136658,
      "output_tokens": 23,
      "cache_read_tokens": 121269
    },
    {
      "model": "gemini:gemini-3.1-flash-lite",
      "test_id": "perf.loaded-history-simple",
      "title": "Simple question stays fast on a loaded history",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.028819525,
      "known_cost_usd": 0.028819525,
      "usd_per_attempt": 0.009606508333333333,
      "usd_per_quality_success": 0.009606508333333333,
      "trial_s": {
        "n": 3,
        "mean": 6.789133333333333,
        "median": 6.761,
        "p95": 6.964
      },
      "turn_s": {
        "n": 12,
        "mean": 1.6972833333333333,
        "median": 1.6076000000000001,
        "p95": 2.252
      },
      "first_activity_s": {
        "n": 12,
        "mean": 1.522,
        "median": 1.48175,
        "p95": 2.0009
      },
      "first_text_s": {
        "n": 12,
        "mean": 1.5220083333333334,
        "median": 1.48175,
        "p95": 2.0009
      },
      "input_tokens": 298507,
      "output_tokens": 359,
      "cache_read_tokens": 205981
    },
    {
      "model": "gemini:gemini-3.1-flash-lite",
      "test_id": "ALL",
      "title": "All selected tests",
      "expected": 12,
      "attempted": 12,
      "executed": 12,
      "passed": 9,
      "quality_passed": 11,
      "metered": 12,
      "cost_usd": 0.1047158,
      "known_cost_usd": 0.1047158,
      "usd_per_attempt": 0.008726316666666666,
      "usd_per_quality_success": 0.00951961818181818,
      "trial_s": {
        "n": 12,
        "mean": 5.9208083333333335,
        "median": 5.040649999999999,
        "p95": 10.0526
      },
      "turn_s": {
        "n": 24,
        "mean": 2.9604041666666667,
        "median": 1.71515,
        "p95": 10.0526
      },
      "first_activity_s": {
        "n": 24,
        "mean": 1.59905,
        "median": 1.64235,
        "p95": 2.0153
      },
      "first_text_s": {
        "n": 24,
        "mean": 2.8364083333333334,
        "median": 1.65375,
        "p95": 9.9952
      },
      "input_tokens": 1190666,
      "output_tokens": 977,
      "cache_read_tokens": 864072
    },
    {
      "model": "gemini:gemini-3.5-flash-lite",
      "test_id": "perf.simple-turn-budget",
      "title": "Simple no-tool turn within 045 budget",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.0205341,
      "known_cost_usd": 0.0205341,
      "usd_per_attempt": 0.0068447,
      "usd_per_quality_success": 0.0068447,
      "trial_s": {
        "n": 3,
        "mean": 1.893433333333333,
        "median": 1.8131,
        "p95": 2.2325999999999997
      },
      "turn_s": {
        "n": 3,
        "mean": 1.893433333333333,
        "median": 1.8131,
        "p95": 2.2325999999999997
      },
      "first_activity_s": {
        "n": 3,
        "mean": 1.8529000000000002,
        "median": 1.7584000000000002,
        "p95": 2.1969000000000003
      },
      "first_text_s": {
        "n": 3,
        "mean": 1.8529000000000002,
        "median": 1.7584000000000002,
        "p95": 2.1969000000000003
      },
      "input_tokens": 68322,
      "output_tokens": 15,
      "cache_read_tokens": 0
    },
    {
      "model": "gemini:gemini-3.5-flash-lite",
      "test_id": "perf.one-tool-turn-budget",
      "title": "One tool call turn under 20s",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.045561779999999996,
      "known_cost_usd": 0.045561779999999996,
      "usd_per_attempt": 0.01518726,
      "usd_per_quality_success": 0.01518726,
      "trial_s": {
        "n": 3,
        "mean": 5.499766666666667,
        "median": 5.4701,
        "p95": 5.9257
      },
      "turn_s": {
        "n": 3,
        "mean": 5.499766666666667,
        "median": 5.4701,
        "p95": 5.9257
      },
      "first_activity_s": {
        "n": 3,
        "mean": 1.7452666666666667,
        "median": 1.6834,
        "p95": 1.9644000000000001
      },
      "first_text_s": {
        "n": 3,
        "mean": 5.474566666666667,
        "median": 5.4489,
        "p95": 5.896199999999999
      },
      "input_tokens": 281349,
      "output_tokens": 171,
      "cache_read_tokens": 145446
    },
    {
      "model": "gemini:gemini-3.5-flash-lite",
      "test_id": "perf.second-turn-cache",
      "title": "Second turn in a thread \u2192 cache hit, fast",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.01484015,
      "known_cost_usd": 0.01484015,
      "usd_per_attempt": 0.004946716666666667,
      "usd_per_quality_success": 0.004946716666666667,
      "trial_s": {
        "n": 3,
        "mean": 3.5013666666666667,
        "median": 3.5722000000000005,
        "p95": 3.5951999999999997
      },
      "turn_s": {
        "n": 6,
        "mean": 1.7506833333333334,
        "median": 1.6683499999999998,
        "p95": 2.1724
      },
      "first_activity_s": {
        "n": 6,
        "mean": 1.7056666666666667,
        "median": 1.64815,
        "p95": 2.1001999999999996
      },
      "first_text_s": {
        "n": 6,
        "mean": 1.7056666666666667,
        "median": 1.64815,
        "p95": 2.1001999999999996
      },
      "input_tokens": 136655,
      "output_tokens": 14,
      "cache_read_tokens": 97005
    },
    {
      "model": "gemini:gemini-3.5-flash-lite",
      "test_id": "perf.loaded-history-simple",
      "title": "Simple question stays fast on a loaded history",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.05095481,
      "known_cost_usd": 0.05095481,
      "usd_per_attempt": 0.01698493666666667,
      "usd_per_quality_success": 0.01698493666666667,
      "trial_s": {
        "n": 3,
        "mean": 6.750266666666667,
        "median": 6.7123,
        "p95": 6.9765
      },
      "turn_s": {
        "n": 12,
        "mean": 1.6875666666666667,
        "median": 1.6536500000000003,
        "p95": 1.9888
      },
      "first_activity_s": {
        "n": 12,
        "mean": 1.56025,
        "median": 1.51715,
        "p95": 1.881
      },
      "first_text_s": {
        "n": 12,
        "mean": 1.5602583333333333,
        "median": 1.5171999999999999,
        "p95": 1.881
      },
      "input_tokens": 298322,
      "output_tokens": 284,
      "cache_read_tokens": 145377
    },
    {
      "model": "gemini:gemini-3.5-flash-lite",
      "test_id": "ALL",
      "title": "All selected tests",
      "expected": 12,
      "attempted": 12,
      "executed": 12,
      "passed": 12,
      "quality_passed": 12,
      "metered": 12,
      "cost_usd": 0.13189084,
      "known_cost_usd": 0.13189084,
      "usd_per_attempt": 0.010990903333333335,
      "usd_per_quality_success": 0.010990903333333335,
      "trial_s": {
        "n": 12,
        "mean": 4.411208333333334,
        "median": 4.34935,
        "p95": 6.7123
      },
      "turn_s": {
        "n": 24,
        "mean": 2.205604166666667,
        "median": 1.69015,
        "p95": 5.4701
      },
      "first_activity_s": {
        "n": 24,
        "mean": 1.6563125,
        "median": 1.5957000000000001,
        "p95": 2.1441999999999997
      },
      "first_text_s": {
        "n": 24,
        "mean": 2.1224791666666665,
        "median": 1.6035500000000003,
        "p95": 5.4489
      },
      "input_tokens": 784648,
      "output_tokens": 484,
      "cache_read_tokens": 387828
    },
    {
      "model": "gemini:gemini-3.8-flash",
      "test_id": "perf.simple-turn-budget",
      "title": "Simple no-tool turn within 045 budget",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.05355525,
      "known_cost_usd": 0.05355525,
      "usd_per_attempt": 0.01785175,
      "usd_per_quality_success": 0.01785175,
      "trial_s": {
        "n": 3,
        "mean": 2.916333333333333,
        "median": 2.9393000000000002,
        "p95": 3.0957
      },
      "turn_s": {
        "n": 3,
        "mean": 2.916333333333333,
        "median": 2.9393000000000002,
        "p95": 3.0957
      },
      "first_activity_s": {
        "n": 3,
        "mean": 2.891,
        "median": 2.9276999999999997,
        "p95": 3.0461
      },
      "first_text_s": {
        "n": 3,
        "mean": 2.891,
        "median": 2.9276999999999997,
        "p95": 3.0461
      },
      "input_tokens": 68322,
      "output_tokens": 617,
      "cache_read_tokens": 0
    },
    {
      "model": "gemini:gemini-3.8-flash",
      "test_id": "perf.one-tool-turn-budget",
      "title": "One tool call turn under 20s",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.086320275,
      "known_cost_usd": 0.086320275,
      "usd_per_attempt": 0.028773425000000002,
      "usd_per_quality_success": 0.028773425000000002,
      "trial_s": {
        "n": 3,
        "mean": 7.583933333333333,
        "median": 7.450699999999999,
        "p95": 9.0595
      },
      "turn_s": {
        "n": 3,
        "mean": 7.583933333333333,
        "median": 7.450699999999999,
        "p95": 9.0595
      },
      "first_activity_s": {
        "n": 3,
        "mean": 3.1589666666666667,
        "median": 2.9125,
        "p95": 3.8470999999999997
      },
      "first_text_s": {
        "n": 3,
        "mean": 7.560133333333333,
        "median": 7.4397,
        "p95": 9.025799999999998
      },
      "input_tokens": 214504,
      "output_tokens": 1937,
      "cache_read_tokens": 121217
    },
    {
      "model": "gemini:gemini-3.8-flash",
      "test_id": "perf.second-turn-cache",
      "title": "Second turn in a thread \u2192 cache hit, fast",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 3,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.023332275,
      "known_cost_usd": 0.023332275,
      "usd_per_attempt": 0.007777425,
      "usd_per_quality_success": 0.007777425,
      "trial_s": {
        "n": 3,
        "mean": 4.745166666666667,
        "median": 4.8424,
        "p95": 4.920700000000001
      },
      "turn_s": {
        "n": 6,
        "mean": 2.3725833333333335,
        "median": 2.3833,
        "p95": 2.6328
      },
      "first_activity_s": {
        "n": 6,
        "mean": 2.3481,
        "median": 2.35535,
        "p95": 2.6248
      },
      "first_text_s": {
        "n": 6,
        "mean": 2.348116666666667,
        "median": 2.3554000000000004,
        "p95": 2.6248
      },
      "input_tokens": 136656,
      "output_tokens": 717,
      "cache_read_tokens": 121257
    },
    {
      "model": "gemini:gemini-3.8-flash",
      "test_id": "perf.loaded-history-simple",
      "title": "Simple question stays fast on a loaded history",
      "expected": 3,
      "attempted": 3,
      "executed": 3,
      "passed": 2,
      "quality_passed": 3,
      "metered": 3,
      "cost_usd": 0.104796375,
      "known_cost_usd": 0.104796375,
      "usd_per_attempt": 0.034932125,
      "usd_per_quality_success": 0.034932125,
      "trial_s": {
        "n": 3,
        "mean": 12.9156,
        "median": 12.7645,
        "p95": 14.5583
      },
      "turn_s": {
        "n": 12,
        "mean": 3.2289,
        "median": 2.73335,
        "p95": 4.2695
      },
      "first_activity_s": {
        "n": 12,
        "mean": 3.118533333333333,
        "median": 2.6986999999999997,
        "p95": 4.0906
      },
      "first_text_s": {
        "n": 12,
        "mean": 3.118533333333333,
        "median": 2.6986999999999997,
        "p95": 4.0906
      },
      "input_tokens": 298506,
      "output_tokens": 955,
      "cache_read_tokens": 181725
    },
    {
      "model": "gemini:gemini-3.8-flash",
      "test_id": "ALL",
      "title": "All selected tests",
      "expected": 12,
      "attempted": 12,
      "executed": 12,
      "passed": 11,
      "quality_passed": 12,
      "metered": 12,
      "cost_usd": 0.268004175,
      "known_cost_usd": 0.268004175,
      "usd_per_attempt": 0.02233368125,
      "usd_per_quality_success": 0.02233368125,
      "trial_s": {
        "n": 12,
        "mean": 7.040258333333333,
        "median": 5.581150000000001,
        "p95": 12.7645
      },
      "turn_s": {
        "n": 24,
        "mean": 3.5201291666666665,
        "median": 2.6734,
        "p95": 7.450699999999999
      },
      "first_activity_s": {
        "n": 24,
        "mean": 2.9025375,
        "median": 2.662,
        "p95": 4.0906
      },
      "first_text_s": {
        "n": 24,
        "mean": 3.4526875,
        "median": 2.662,
        "p95": 7.4397
      },
      "input_tokens": 717988,
      "output_tokens": 4226,
      "cache_read_tokens": 424199
    }
  ],
  "pricing": {
    "date": "2026-09-20",
    "currency": "USD",
    "unit": "per million tokens",
    "notes": "Standard paid text inference; Anthropic 5m cache writes; Gemini promotional prices through 2026-12-31. Foreground only. No invoice reconciliation or free-tier assumption. Context >200k unknown.",
    "models": {
      "anthropic:claude-opus-4-6": {
        "input": 5,
        "output": 25,
        "cache_read": 0.5,
        "cache_write": 6.25,
        "source": "https://platform.claude.com/docs/en/about-claude/pricing"
      },
      "anthropic:claude-sonnet-4-5-20250929": {
        "input": 3,
        "output": 15,
        "cache_read": 0.30000000000000004,
        "cache_write": 3.75,
        "source": "https://platform.claude.com/docs/en/about-claude/pricing"
      },
      "anthropic:claude-haiku-4-5-20251001": {
        "input": 1,
        "output": 5,
        "cache_read": 0.1,
        "cache_write": 1.25,
        "source": "https://platform.claude.com/docs/en/about-claude/pricing"
      },
      "openai:gpt-5.5": {
        "input": 5,
        "output": 30,
        "cache_read": 0.5,
        "cache_write": 5,
        "source": "https://developers.openai.com/api/docs/models/gpt-5.5"
      },
      "openai:gpt-4o": {
        "input": 2.5,
        "output": 10,
        "cache_read": 1.25,
        "cache_write": 2.5,
        "source": "https://developers.openai.com/api/docs/models/gpt-4o"
      },
      "openai:gpt-4o-mini": {
        "input": 0.15,
        "output": 0.6,
        "cache_read": 0.075,
        "cache_write": 0.15,
        "source": "https://developers.openai.com/api/docs/models/gpt-4o-mini"
      },
      "gemini:gemini-3.1-flash-lite": {
        "input": 0.25,
        "output": 1.5,
        "cache_read": 0.025,
        "cache_write": 0.25,
        "source": "https://ai.google.dev/gemini-api/docs/pricing"
      },
      "gemini:gemini-3.5-flash-lite": {
        "input": 0.3,
        "output": 2.5,
        "cache_read": 0.03,
        "cache_write": 0.3,
        "source": "https://ai.google.dev/gemini-api/docs/pricing"
      },
      "gemini:gemini-3.8-flash": {
        "input": 0.75,
        "output": 3.75,
        "cache_read": 0.07500000000000001,
        "cache_write": 0.75,
        "source": "https://ai.google.dev/gemini-api/docs/pricing"
      }
    }
  },
  "excluded": [
    {
      "model": "xai:grok-4.5",
      "label": "Grok 4.5",
      "reason": "No configured local inference API credential"
    },
    {
      "model": "xai:grok-4.3",
      "label": "Grok 4.3",
      "reason": "No configured local inference API credential"
    },
    {
      "model": "xai:grok-build-0.1",
      "label": "Grok Build 0.1",
      "reason": "No configured local inference API credential"
    },
    {
      "model": "moonshot:kimi-k3",
      "label": "Kimi K3",
      "reason": "No configured local inference API credential"
    },
    {
      "model": "moonshot:kimi-k2.7-code",
      "label": "Kimi K2.7 Code",
      "reason": "No configured local inference API credential"
    },
    {
      "model": "xai:grok-4.6",
      "label": "Grok 4.6",
      "reason": "No configured local inference API credential"
    },
    {
      "model": "moonshot:kimi-k2.6",
      "label": "Kimi K2.6 Agent Ops",
      "reason": "No configured local inference API credential"
    }
  ]
}