{
 "analysis_method": "Every valid measured pair, no outlier dropped. % = mean(Context Mode - plain) / mean(plain), with the 95% t interval. Verdict: \"less\" when the whole interval is below zero, \"more\" when above, else \"same\". Gateway wins: pairs where Context Mode cost less.",
 "arms": {"gateway": "Claude Code talks to Anthropic through Context Mode.", "plain": "Claude Code talks to Anthropic direct."},
 "client": "Claude Code 2.1.283",
 "cost_basis": "Cost in US dollars at API list price, both arms: list price applied to the token counts Anthropic returned for each call. The runs used a claude.ai login, which has no per-token invoice. Plain: Claude Code's own total. Context Mode: the gateway's per-call record for the run's sessions, hidden calls included, each call at the list price of the model it ran on, checked per run against Claude Code's own figure.",
 "coverage": {"arms": 80, "arms_recomputed": 60, "arms_with_client_figure": 80, "calls_known_total": 123, "calls_with_tokens": 60},
 "date": "2026-09-29",
 "deployed_commits": ["105938c99653692afac606d0806f081cdb251865","83429d30dcbfff8c5b7708270875b05c812b3a80","cba7ce9872801b5193392fa3f14a651527faa87c"],
 "gateway_commit": "3bedf1ba",
 "harness": "unified-ab",
 "legend": {"col.cr": "cache read tokens", "col.cw1h": "cache write tokens, 1 hour lifetime", "col.cw5": "cache write tokens, 5 minute lifetime", "col.cwx": "cache write tokens whose lifetime was not saved: usd_recomputed prices them at the 1 hour rate (Claude Code's main thread), usd_recomputed_low at the 5 minute rate", "col.in": "input tokens not read from or written to the cache", "col.missing": "a token column left out of a block is all zeros; model indexes into models", "col.out": "output tokens; null = not saved per call", "from.client_totals": "Claude Code's own per-model totals for the session", "from.ledger": "the gateway ledger's rows for the run's sessions, one row per call", "from.none": "no per-call tokens were saved", "rest.per_model_total": "Claude Code's own token totals per model for the session", "rest.session_total": "the gateway ledger's token totals for the session", "why.edge": "The gateway's figure includes the edge-loop calls it makes for the client. Claude Code never sees those calls, so its own total is lower.", "why.no_tokens": "No tokens were saved for this arm. The run folders with each call's tokens were deleted after the run."},
 "model": "claude-opus-5-5",
 "prices": {
  "aliases": {"claude-haiku-4-5-20251001": "claude-haiku-4-5"},
  "as_of": "2026-09-29",
  "models": {
   "claude-haiku-4-5": {"cr": 0.1, "cw1h": 2, "cw5": 1.25, "in": 1, "out": 5},
   "claude-opus-5-5": {"cr": 0.2, "cw1h": 8, "cw5": 5, "in": 4, "out": 20}
  },
  "source_url": "https://www.anthropic.com/pricing",
  "unit": "USD per million tokens"
 },
 "schema": "context-mode.benchmark-evidence/2",
 "source_file": "final-opus55-tests-W3-minitest-W4.json",
 "warmup_pairs": 1,
 "workloads": [
  {
   "analysis": {"gateway_usd_mean": 0.06677, "gateway_wins": 11, "n": 12, "pct": -21.3, "pct_ci95": [-28.3,-14.2], "plain_usd_mean": 0.08481, "success_gateway": "12/12", "success_plain": "12/12", "verdict": "less"},
   "id": "tests",
   "invalid_attempts": 0,
   "label": "npm test fix loop, long",
   "pairs": [
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 4, "sum_complete": true},
      "rest": {"cr": [94610], "cwx": [5741], "in": [2598], "model": [0], "models": ["claude-opus-5-5"], "n": [4], "out": [715], "what": ["session_total"]},
      "success": true,
      "turns": 4,
      "usd": 0.089542,
      "usd_client": 0.082092,
      "usd_recomputed": 0.089542,
      "usd_recomputed_low": 0.072319,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 1,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0875318,
      "usd_client": 0.0875318,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [53821], "cwx": [5465], "in": [1075], "model": [0], "models": ["claude-opus-5-5"], "n": [3], "out": [311], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.065004,
      "usd_client": 0.0650042,
      "usd_recomputed": 0.065004,
      "usd_recomputed_low": 0.048609
     },
     "pair": 2,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0850486,
      "usd_client": 0.0850486,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [53925], "cwx": [5649], "in": [1019], "model": [0], "models": ["claude-opus-5-5"], "n": [3], "out": [411], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.068273,
      "usd_client": 0.068273,
      "usd_recomputed": 0.068273,
      "usd_recomputed_low": 0.051326
     },
     "pair": 3,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0857486,
      "usd_client": 0.0857486,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [53894], "cwx": [5618], "in": [1019], "model": [0], "models": ["claude-opus-5-5"], "n": [3], "out": [384], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.067479,
      "usd_client": 0.06747879999999999,
      "usd_recomputed": 0.067479,
      "usd_recomputed_low": 0.050625
     },
     "pair": 4,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.088922,
      "usd_client": 0.08892199999999999,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [53656], "cwx": [5306], "in": [1019], "model": [0], "models": ["claude-opus-5-5"], "n": [3], "out": [303], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.063315,
      "usd_client": 0.0633152,
      "usd_recomputed": 0.063315,
      "usd_recomputed_low": 0.047397
     },
     "pair": 5,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0836784,
      "usd_client": 0.0836784,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [53970], "cwx": [5832], "in": [1019], "model": [0], "models": ["claude-opus-5-5"], "n": [3], "out": [435], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.070226,
      "usd_client": 0.07022600000000001,
      "usd_recomputed": 0.070226,
      "usd_recomputed_low": 0.05273
     },
     "pair": 6,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0850046,
      "usd_client": 0.0850046,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [53824], "cwx": [5467], "in": [1019], "model": [0], "models": ["claude-opus-5-5"], "n": [3], "out": [286], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.064297,
      "usd_client": 0.0642968,
      "usd_recomputed": 0.064297,
      "usd_recomputed_low": 0.047896
     },
     "pair": 7,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0683108,
      "usd_client": 0.0683108,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [51645], "cwx": [3542], "in": [1017], "model": [0], "models": ["claude-opus-5-5"], "n": [3], "out": [458], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.051893,
      "usd_client": 0.051893,
      "usd_recomputed": 0.051893,
      "usd_recomputed_low": 0.041267
     },
     "pair": 8,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0858846,
      "usd_client": 0.0858846,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [53964], "cwx": [5706], "in": [1019], "model": [0], "models": ["claude-opus-5-5"], "n": [3], "out": [437], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.069257,
      "usd_client": 0.0692568,
      "usd_recomputed": 0.069257,
      "usd_recomputed_low": 0.052139
     },
     "pair": 9,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0858886,
      "usd_client": 0.0858886,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [53827], "cwx": [5481], "in": [1019], "model": [0], "models": ["claude-opus-5-5"], "n": [3], "out": [255], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.063789,
      "usd_client": 0.0637894,
      "usd_recomputed": 0.063789,
      "usd_recomputed_low": 0.047346
     },
     "pair": 10,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0873362,
      "usd_client": 0.08733619999999999,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [53825], "cwx": [5468], "in": [1019], "model": [0], "models": ["claude-opus-5-5"], "n": [3], "out": [270], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.063985,
      "usd_client": 0.063985,
      "usd_recomputed": 0.063985,
      "usd_recomputed_low": 0.047581
     },
     "pair": 11,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0871918,
      "usd_client": 0.0871918,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [53824], "cwx": [5467], "in": [1019], "model": [0], "models": ["claude-opus-5-5"], "n": [3], "out": [279], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.064157,
      "usd_client": 0.0641568,
      "usd_recomputed": 0.064157,
      "usd_recomputed_low": 0.047756
     },
     "pair": 12,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0872152,
      "usd_client": 0.08721519999999999,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    }
   ],
   "pairs_planned": 12
  },
  {
   "analysis": {"gateway_usd_mean": 0.20476, "gateway_wins": 17, "n": 20, "pct": -9.3, "pct_ci95": [-13.7,-4.8], "plain_usd_mean": 0.22563, "success_gateway": "20/20", "success_plain": "20/20", "verdict": "less"},
   "id": "W3",
   "invalid_attempts": 3,
   "label": "Large documents processed with a script",
   "note": "Replaced by the W3 row in v4-opus55-W1-W2-W3-W4.json (a later gateway commit).",
   "pairs": [
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [26642,29831,15543], "cwx": [2489,1255,4545], "in": [3134,4577,738], "model": [0,0,0], "models": ["claude-opus-5-5"], "out": [1061,3034,502]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.206451,
      "usd_client": 0.1560028,
      "usd_recomputed": 0.206451,
      "usd_recomputed_low": 0.181584,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 1,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [125064], "cwx": [15103], "in": [14], "model": [0], "models": ["claude-opus-5-5"], "n": [null], "out": [4472], "what": ["per_model_total"]},
      "success": true,
      "turns": 7,
      "usd": 0.2353328,
      "usd_client": 0.2353328,
      "usd_recomputed": 0.235333,
      "usd_recomputed_low": 0.190024
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [26815,31938,17477], "cwx": [2662,3016,2759], "in": [2784,4529,738], "model": [0,0,0], "models": ["claude-opus-5-5"], "out": [923,3180,599]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.208986,
      "usd_client": 0.160375,
      "usd_recomputed": 0.208986,
      "usd_recomputed_low": 0.183675,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 2,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [85379], "cwx": [14709], "in": [10], "model": [0], "models": ["claude-opus-5-5"], "n": [null], "out": [4471], "what": ["per_model_total"]},
      "success": true,
      "turns": 5,
      "usd": 0.2242078,
      "usd_client": 0.2242078,
      "usd_recomputed": 0.224208,
      "usd_recomputed_low": 0.180081
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [26816,31966,17504], "cwx": [2663,3042,2829], "in": [3243,4592,739], "model": [0,0,0], "models": ["claude-opus-5-5"], "out": [1159,3293,502]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.216905,
      "usd_client": 0.16280199999999997,
      "usd_recomputed": 0.216905,
      "usd_recomputed_low": 0.191303,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 3,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [109155], "cwx": [14020], "in": [12], "model": [0], "models": ["claude-opus-5-5"], "n": [null], "out": [3530], "what": ["per_model_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.204639,
      "usd_client": 0.20463900000000002,
      "usd_recomputed": 0.204639,
      "usd_recomputed_low": 0.162579
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [26815,31961,17500], "cwx": [2662,3039,2417], "in": [2819,4692,739], "model": [0,0,0], "models": ["claude-opus-5-5"], "out": [971,2858,361]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.196999,
      "usd_client": 0.14746019999999999,
      "usd_recomputed": 0.196999,
      "usd_recomputed_low": 0.172645,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 4,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [85424], "cwx": [14466], "in": [10], "model": [0], "models": ["claude-opus-5-5"], "n": [null], "out": [4036], "what": ["per_model_total"]},
      "success": true,
      "turns": 5,
      "usd": 0.2135728,
      "usd_client": 0.21357280000000003,
      "usd_recomputed": 0.213573,
      "usd_recomputed_low": 0.170175
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [26816,31934,17472], "cwx": [2663,3010,2620], "in": [2868,4782,739], "model": [0,0,0], "models": ["claude-opus-5-5"], "out": [889,3043,393]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.201644,
      "usd_client": 0.15354120000000002,
      "usd_recomputed": 0.201644,
      "usd_recomputed_low": 0.176765,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 5,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [129393], "cwx": [15113], "in": [14], "model": [0], "models": ["claude-opus-5-5"], "n": [null], "out": [4716], "what": ["per_model_total"]},
      "success": true,
      "turns": 7,
      "usd": 0.2411586,
      "usd_client": 0.24115860000000006,
      "usd_recomputed": 0.241159,
      "usd_recomputed_low": 0.19582
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [26816,31982,17520], "cwx": [2663,3058,2291], "in": [3236,4607,739], "model": [0,0,0], "models": ["claude-opus-5-5"], "out": [1328,2571,437]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.200408,
      "usd_client": 0.1499124,
      "usd_recomputed": 0.200408,
      "usd_recomputed_low": 0.176372,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 6,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [128762], "cwx": [14963], "in": [14], "model": [0], "models": ["claude-opus-5-5"], "n": [null], "out": [4503], "what": ["per_model_total"]},
      "success": true,
      "turns": 7,
      "usd": 0.2355724,
      "usd_client": 0.23557240000000002,
      "usd_recomputed": 0.235572,
      "usd_recomputed_low": 0.190683
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [26816,31968,17506], "cwx": [2663,3044,2645], "in": [2935,4509,744], "model": [0,0,0], "models": ["claude-opus-5-5"], "out": [938,3042,402]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.202466,
      "usd_client": 0.1542908,
      "usd_recomputed": 0.202466,
      "usd_recomputed_low": 0.17741,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 7,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [110554], "cwx": [15000], "in": [12], "model": [0], "models": ["claude-opus-5-5"], "n": [null], "out": [4478], "what": ["per_model_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.2317188,
      "usd_client": 0.2317188,
      "usd_recomputed": 0.231719,
      "usd_recomputed_low": 0.186719
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [26816,31933,17471], "cwx": [2663,3009,2539], "in": [3042,4746,744], "model": [0,0,0], "models": ["claude-opus-5-5"], "out": [1086,3001,455]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.2059,
      "usd_client": 0.1554448,
      "usd_recomputed": 0.2059,
      "usd_recomputed_low": 0.181267,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 8,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [129854], "cwx": [15141], "in": [14], "model": [0], "models": ["claude-opus-5-5"], "n": [null], "out": [4637], "what": ["per_model_total"]},
      "success": true,
      "turns": 7,
      "usd": 0.2398948,
      "usd_client": 0.2398948,
      "usd_recomputed": 0.239895,
      "usd_recomputed_low": 0.194472
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [26816,31915,17453], "cwx": [2663,2991,2387], "in": [3004,4480,744], "model": [0,0,0], "models": ["claude-opus-5-5"], "out": [942,2751,466]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.195657,
      "usd_client": 0.1469936,
      "usd_recomputed": 0.195657,
      "usd_recomputed_low": 0.171534,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 9,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [174104], "cwx": [16446], "in": [18], "model": [0], "models": ["claude-opus-5-5"], "n": [null], "out": [5228], "what": ["per_model_total"]},
      "success": true,
      "turns": 9,
      "usd": 0.2710208,
      "usd_client": 0.2710208,
      "usd_recomputed": 0.271021,
      "usd_recomputed_low": 0.221683
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [26815,31889,17428], "cwx": [2662,2967,2571], "in": [2890,4593,744], "model": [0,0,0], "models": ["claude-opus-5-5"], "out": [887,2970,584]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.202554,
      "usd_client": 0.15447539999999998,
      "usd_recomputed": 0.202554,
      "usd_recomputed_low": 0.177954,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 10,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [84907], "cwx": [13372], "in": [10], "model": [0], "models": ["claude-opus-5-5"], "n": [null], "out": [3172], "what": ["per_model_total"]},
      "success": true,
      "turns": 5,
      "usd": 0.1874374,
      "usd_client": 0.18743739999999998,
      "usd_recomputed": 0.187437,
      "usd_recomputed_low": 0.147321
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [26815,31937,17476], "cwx": [2662,3015,2793], "in": [3087,4523,739], "model": [0,0,0], "models": ["claude-opus-5-5"], "out": [1170,3238,619]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.216942,
      "usd_client": 0.1649826,
      "usd_recomputed": 0.216942,
      "usd_recomputed_low": 0.191532,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 11,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [80106], "cwx": [14198], "in": [10], "model": [0], "models": ["claude-opus-5-5"], "n": [null], "out": [3913], "what": ["per_model_total"]},
      "success": true,
      "turns": 5,
      "usd": 0.2079052,
      "usd_client": 0.20790519999999998,
      "usd_recomputed": 0.207905,
      "usd_recomputed_low": 0.165311
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [26816,31935,17473], "cwx": [2663,3011,2324], "in": [2850,4645,739], "model": [0,0,0], "models": ["claude-opus-5-5"], "out": [918,2762,372]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.193205,
      "usd_client": 0.1453176,
      "usd_recomputed": 0.193205,
      "usd_recomputed_low": 0.169211,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 12,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [130078], "cwx": [15327], "in": [14], "model": [0], "models": ["claude-opus-5-5"], "n": [null], "out": [4784], "what": ["per_model_total"]},
      "success": true,
      "turns": 7,
      "usd": 0.2443676,
      "usd_client": 0.2443676,
      "usd_recomputed": 0.244368,
      "usd_recomputed_low": 0.198387
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [26816,31981,17519], "cwx": [2663,3057,2791], "in": [3050,4645,739], "model": [0,0,0], "models": ["claude-opus-5-5"], "out": [1134,3322,412]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.214447,
      "usd_client": 0.160988,
      "usd_recomputed": 0.214447,
      "usd_recomputed_low": 0.188914,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 13,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [84975], "cwx": [14528], "in": [10], "model": [0], "models": ["claude-opus-5-5"], "n": [null], "out": [4276], "what": ["per_model_total"]},
      "success": true,
      "turns": 5,
      "usd": 0.218779,
      "usd_client": 0.218779,
      "usd_recomputed": 0.218779,
      "usd_recomputed_low": 0.175195
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [26815,31946,17485], "cwx": [2662,3024,2517], "in": [2919,4507,739], "model": [0,0,0], "models": ["claude-opus-5-5"], "out": [978,2928,509]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.201833,
      "usd_client": 0.1533342,
      "usd_recomputed": 0.201833,
      "usd_recomputed_low": 0.177224,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 14,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [85085], "cwx": [14392], "in": [10], "model": [0], "models": ["claude-opus-5-5"], "n": [null], "out": [4099], "what": ["per_model_total"]},
      "success": true,
      "turns": 5,
      "usd": 0.214173,
      "usd_client": 0.21417299999999997,
      "usd_recomputed": 0.214173,
      "usd_recomputed_low": 0.170997
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [26816,31949,17487], "cwx": [2663,3025,2637], "in": [2869,4565,739], "model": [0,0,0], "models": ["claude-opus-5-5"], "out": [950,3085,582]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.206882,
      "usd_client": 0.15817520000000002,
      "usd_recomputed": 0.206882,
      "usd_recomputed_low": 0.181907,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 15,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [129133], "cwx": [15052], "in": [14], "model": [0], "models": ["claude-opus-5-5"], "n": [null], "out": [4568], "what": ["per_model_total"]},
      "success": true,
      "turns": 7,
      "usd": 0.2376586,
      "usd_client": 0.23765860000000003,
      "usd_recomputed": 0.237659,
      "usd_recomputed_low": 0.192503
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [26816,31949,17487], "cwx": [2663,3025,2333], "in": [2980,4737,739], "model": [0,0,0], "models": ["claude-opus-5-5"], "out": [980,2820,485]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.198942,
      "usd_client": 0.1485512,
      "usd_recomputed": 0.198942,
      "usd_recomputed_low": 0.174879,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 16,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [129798], "cwx": [14558], "in": [14], "model": [0], "models": ["claude-opus-5-5"], "n": [null], "out": [3906], "what": ["per_model_total"]},
      "success": true,
      "turns": 7,
      "usd": 0.2205996,
      "usd_client": 0.22059959999999998,
      "usd_recomputed": 0.2206,
      "usd_recomputed_low": 0.176926
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [26817,31914,17451], "cwx": [2664,2988,2376], "in": [2972,4662,739], "model": [0,0,0], "models": ["claude-opus-5-5"], "out": [930,2945,383]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.198112,
      "usd_client": 0.146017,
      "usd_recomputed": 0.198112,
      "usd_recomputed_low": 0.174028,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 17,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [85328], "cwx": [14462], "in": [10], "model": [0], "models": ["claude-opus-5-5"], "n": [null], "out": [4280], "what": ["per_model_total"]},
      "success": true,
      "turns": 5,
      "usd": 0.2184016,
      "usd_client": 0.2184016,
      "usd_recomputed": 0.218402,
      "usd_recomputed_low": 0.175016
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [26814,31926,17466], "cwx": [2661,3006,2458], "in": [3026,4702,739], "model": [0,0,0], "models": ["claude-opus-5-5"], "out": [993,2827,666]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.203829,
      "usd_client": 0.15439039999999998,
      "usd_recomputed": 0.203829,
      "usd_recomputed_low": 0.179454,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 18,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [85507], "cwx": [14755], "in": [10], "model": [0], "models": ["claude-opus-5-5"], "n": [null], "out": [4489], "what": ["per_model_total"]},
      "success": true,
      "turns": 5,
      "usd": 0.2249614,
      "usd_client": 0.22496139999999998,
      "usd_recomputed": 0.224961,
      "usd_recomputed_low": 0.180696
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [26816,31958,17496], "cwx": [2663,3034,2704], "in": [3225,4525,739], "model": [0,0,0], "models": ["claude-opus-5-5"], "out": [1149,3186,520]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.213518,
      "usd_client": 0.1608628,
      "usd_recomputed": 0.213518,
      "usd_recomputed_low": 0.188315,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 19,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [104963], "cwx": [14575], "in": [12], "model": [0], "models": ["claude-opus-5-5"], "n": [null], "out": [4088], "what": ["per_model_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.2194006,
      "usd_client": 0.2194006,
      "usd_recomputed": 0.219401,
      "usd_recomputed_low": 0.175676
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [26816,31917,17455], "cwx": [2663,2993,2719], "in": [3205,4558,769], "model": [0,0,0], "models": ["claude-opus-5-5"], "out": [1199,3124,333]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.209486,
      "usd_client": 0.15771039999999997,
      "usd_recomputed": 0.209486,
      "usd_recomputed_low": 0.184361,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 20,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [134667], "cwx": [15137], "in": [16], "model": [0], "models": ["claude-opus-5-5"], "n": [null], "out": [3685], "what": ["per_model_total"]},
      "success": true,
      "turns": 8,
      "usd": 0.2217934,
      "usd_client": 0.2217934,
      "usd_recomputed": 0.221793,
      "usd_recomputed_low": 0.176382
     }
    }
   ],
   "pairs_planned": 20
  },
  {
   "analysis": {"gateway_usd_mean": 0.06101, "gateway_wins": 8, "n": 8, "pct": -61, "pct_ci95": [-69.9,-52.2], "plain_usd_mean": 0.15656, "success_gateway": "8/8", "success_plain": "8/8", "verdict": "less"},
   "id": "minitest",
   "invalid_attempts": 0,
   "label": "Test fix loop (minitest)",
   "pairs": [
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [53284], "cwx": [5220], "in": [940], "model": [0], "models": ["claude-opus-5-5"], "n": [3], "out": [448], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.065137,
      "usd_client": 0.0651368,
      "usd_recomputed": 0.065137,
      "usd_recomputed_low": 0.049477
     },
     "pair": 1,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.160973,
      "usd_client": 0.16097299999999998,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [53120], "cwx": [5083], "in": [932], "model": [0], "models": ["claude-opus-5-5"], "n": [3], "out": [335], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.061716,
      "usd_client": 0.061716,
      "usd_recomputed": 0.061716,
      "usd_recomputed_low": 0.046467
     },
     "pair": 2,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.1646268,
      "usd_client": 0.1646268,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [53186], "cwx": [5065], "in": [932], "model": [0], "models": ["claude-opus-5-5"], "n": [3], "out": [454], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.063965,
      "usd_client": 0.0639652,
      "usd_recomputed": 0.063965,
      "usd_recomputed_low": 0.04877
     },
     "pair": 3,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.1388592,
      "usd_client": 0.13885920000000002,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [53190], "cwx": [5068], "in": [932], "model": [0], "models": ["claude-opus-5-5"], "n": [3], "out": [452], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.06395,
      "usd_client": 0.06395,
      "usd_recomputed": 0.06395,
      "usd_recomputed_low": 0.048746
     },
     "pair": 4,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.160889,
      "usd_client": 0.160889,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [50893], "cwx": [2867], "in": [938], "model": [0], "models": ["claude-opus-5-5"], "n": [3], "out": [273], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.042327,
      "usd_client": 0.0423266,
      "usd_recomputed": 0.042327,
      "usd_recomputed_low": 0.033726
     },
     "pair": 5,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.1609406,
      "usd_client": 0.16094060000000002,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [51250], "cwx": [3128], "in": [938], "model": [0], "models": ["claude-opus-5-5"], "n": [3], "out": [412], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.047266,
      "usd_client": 0.047266,
      "usd_recomputed": 0.047266,
      "usd_recomputed_low": 0.037882
     },
     "pair": 6,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.1610206,
      "usd_client": 0.1610206,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 4, "sum_complete": true},
      "rest": {"cr": [73664], "cwx": [5218], "in": [942], "model": [0], "models": ["claude-opus-5-5"], "n": [4], "out": [333], "what": ["session_total"]},
      "success": true,
      "turns": 4,
      "usd": 0.066905,
      "usd_client": 0.06690480000000001,
      "usd_recomputed": 0.066905,
      "usd_recomputed_low": 0.051251
     },
     "pair": 7,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.1395088,
      "usd_client": 0.1395088,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 4, "sum_complete": true},
      "rest": {"cr": [73811], "cwx": [5669], "in": [942], "model": [0], "models": ["claude-opus-5-5"], "n": [4], "out": [647], "what": ["session_total"]},
      "success": true,
      "turns": 5,
      "usd": 0.076822,
      "usd_client": 0.0768222,
      "usd_recomputed": 0.076822,
      "usd_recomputed_low": 0.059815
     },
     "pair": 8,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.1656384,
      "usd_client": 0.16563840000000002,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    }
   ],
   "pairs_planned": 8
  }
 ]
}