{
 "analysis_method": "Every valid measured pair, no outlier dropped. % = mean(Context Mode - plain) / mean(plain), with the 95% t interval. Verdict: \"less\" when the whole interval is below zero, \"more\" when above, else \"same\". Gateway wins: pairs where Context Mode cost less.",
 "arms": {"gateway": "Claude Code talks to Anthropic through Context Mode.", "plain": "Claude Code talks to Anthropic direct."},
 "client": "Claude Code 2.1.283",
 "cost_basis": "Cost in US dollars at API list price, both arms: list price applied to the token counts Anthropic returned for each call. The runs used a claude.ai login, which has no per-token invoice. Plain: Claude Code's own total. Context Mode: the gateway's per-call record for the run's sessions, hidden calls included, each call at the list price of the model it ran on, checked per run against Claude Code's own figure.",
 "coverage": {"arms": 184, "arms_recomputed": 104, "arms_with_client_figure": 184, "calls_known_total": 350, "calls_with_tokens": 43},
 "date": "2026-09-29",
 "deployed_commits": ["83429d30dcbfff8c5b7708270875b05c812b3a80","cba7ce9872801b5193392fa3f14a651527faa87c"],
 "gateway_commit": "3bedf1ba29958a8c4512cc53de2fe6aac4eb045b",
 "harness": "unified-ab",
 "legend": {"col.cr": "cache read tokens", "col.cw1h": "cache write tokens, 1 hour lifetime", "col.cw5": "cache write tokens, 5 minute lifetime", "col.cwx": "cache write tokens whose lifetime was not saved: usd_recomputed prices them at the 1 hour rate (Claude Code's main thread), usd_recomputed_low at the 5 minute rate", "col.in": "input tokens not read from or written to the cache", "col.missing": "a token column left out of a block is all zeros; model indexes into models", "col.out": "output tokens; null = not saved per call", "from.client_totals": "Claude Code's own per-model totals for the session", "from.ledger": "the gateway ledger's rows for the run's sessions, one row per call", "from.none": "no per-call tokens were saved", "rest.minus_saved_rows": "the gateway ledger's session totals minus the rows in calls (the saved read returned at most 10 rows)", "rest.per_model_total": "Claude Code's own token totals per model for the session", "rest.session_total": "the gateway ledger's token totals for the session", "why.edge": "The gateway's figure includes the edge-loop calls it makes for the client. Claude Code never sees those calls, so its own total is lower.", "why.no_tokens": "No tokens were saved for this arm. The run folders with each call's tokens were deleted after the run."},
 "model": "claude-haiku-4-5-20251001",
 "prices": {
  "aliases": {"claude-haiku-4-5-20251001": "claude-haiku-4-5"},
  "as_of": "2026-09-29",
  "models": {
   "claude-haiku-4-5": {"cr": 0.1, "cw1h": 2, "cw5": 1.25, "in": 1, "out": 5},
   "claude-opus-5-5": {"cr": 0.2, "cw1h": 8, "cw5": 5, "in": 4, "out": 20}
  },
  "source_url": "https://www.anthropic.com/pricing",
  "unit": "USD per million tokens"
 },
 "schema": "context-mode.benchmark-evidence/2",
 "source_file": "final-haiku45.json",
 "warmup_pairs": 1,
 "workloads": [
  {
   "analysis": {"gateway_usd_mean": 0.03236, "gateway_wins": 12, "n": 12, "pct": -52.8, "pct_ci95": [-56.2,-49.4], "plain_usd_mean": 0.06859, "success_gateway": "12/12", "success_plain": "12/12", "verdict": "less"},
   "id": "tests",
   "invalid_attempts": 2,
   "label": "npm test fix loop",
   "pairs": [
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 6, "sum_complete": true},
      "rest": {"cr": [124914], "cwx": [8431], "in": [526], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [6], "out": [901], "what": ["session_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.034384,
      "usd_client": 0.0343844,
      "usd_recomputed": 0.034384,
      "usd_recomputed_low": 0.028061
     },
     "pair": 1,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.0710758,
      "usd_client": 0.0710758,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 6, "sum_complete": true},
      "rest": {"cr": [124666], "cwx": [8331], "in": [526], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [6], "out": [809], "what": ["session_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.0337,
      "usd_client": 0.033699599999999996,
      "usd_recomputed": 0.0337,
      "usd_recomputed_low": 0.027451
     },
     "pair": 2,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.0714815,
      "usd_client": 0.0714815,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 6, "sum_complete": true},
      "rest": {"cr": [124870], "cwx": [8440], "in": [583], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [6], "out": [909], "what": ["session_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.034495,
      "usd_client": 0.034495,
      "usd_recomputed": 0.034495,
      "usd_recomputed_low": 0.028165
     },
     "pair": 3,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.0631167,
      "usd_client": 0.0631167,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 6, "sum_complete": true},
      "rest": {"cr": [124929], "cwx": [8487], "in": [583], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [6], "out": [958], "what": ["session_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.03484,
      "usd_client": 0.0348399,
      "usd_recomputed": 0.03484,
      "usd_recomputed_low": 0.028475
     },
     "pair": 4,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.0711106,
      "usd_client": 0.07111060000000001,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 5, "sum_complete": true},
      "rest": {"cr": [102417], "cwx": [8227], "in": [578], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [5], "out": [751], "what": ["session_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.031029,
      "usd_client": 0.031028700000000003,
      "usd_recomputed": 0.031029,
      "usd_recomputed_low": 0.024858
     },
     "pair": 5,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.066447,
      "usd_client": 0.066447,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 6, "sum_complete": true},
      "rest": {"cr": [124745], "cwx": [8394], "in": [577], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [6], "out": [867], "what": ["session_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.034175,
      "usd_client": 0.034174500000000003,
      "usd_recomputed": 0.034174,
      "usd_recomputed_low": 0.027879
     },
     "pair": 6,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.0668361,
      "usd_client": 0.0668361,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 5, "sum_complete": true},
      "rest": {"cr": [96942], "cwx": [6453], "in": [572], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [5], "out": [756], "what": ["session_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.026952,
      "usd_client": 0.026952200000000003,
      "usd_recomputed": 0.026952,
      "usd_recomputed_low": 0.022112
     },
     "pair": 7,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.0677481,
      "usd_client": 0.0677481,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 6, "sum_complete": true},
      "rest": {"cr": [117744], "cwx": [6711], "in": [577], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [6], "out": [946], "what": ["session_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.030503,
      "usd_client": 0.030503399999999993,
      "usd_recomputed": 0.030503,
      "usd_recomputed_low": 0.02547
     },
     "pair": 8,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.0703051,
      "usd_client": 0.0703051,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 5, "sum_complete": true},
      "rest": {"cr": [102383], "cwx": [8237], "in": [572], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [5], "out": [768], "what": ["session_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.031124,
      "usd_client": 0.0311243,
      "usd_recomputed": 0.031124,
      "usd_recomputed_low": 0.024947
     },
     "pair": 9,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.0666629,
      "usd_client": 0.0666629,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 6, "sum_complete": true},
      "rest": {"cr": [124908], "cwx": [8435], "in": [577], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [6], "out": [899], "what": ["session_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.034433,
      "usd_client": 0.0344328,
      "usd_recomputed": 0.034433,
      "usd_recomputed_low": 0.028107
     },
     "pair": 10,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.0665904,
      "usd_client": 0.0665904,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 5, "sum_complete": true},
      "rest": {"cr": [102424], "cwx": [8267], "in": [572], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [5], "out": [814], "what": ["session_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.031418,
      "usd_client": 0.0314184,
      "usd_recomputed": 0.031418,
      "usd_recomputed_low": 0.025218
     },
     "pair": 11,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.0712212,
      "usd_client": 0.07122119999999998,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 5, "sum_complete": true},
      "rest": {"cr": [102388], "cwx": [8237], "in": [572], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [5], "out": [799], "what": ["session_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.03128,
      "usd_client": 0.0312798,
      "usd_recomputed": 0.03128,
      "usd_recomputed_low": 0.025102
     },
     "pair": 12,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.0705445,
      "usd_client": 0.07054450000000001,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    }
   ],
   "pairs_planned": 12
  },
  {
   "analysis": {"gateway_usd_mean": 0.03532, "gateway_wins": 8, "n": 8, "pct": -45.5, "pct_ci95": [-56.8,-34.2], "plain_usd_mean": 0.06481, "success_gateway": "8/8", "success_plain": "8/8", "verdict": "less"},
   "id": "minitest",
   "invalid_attempts": 2,
   "label": "Test fix loop (minitest)",
   "pairs": [
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 6, "sum_complete": true},
      "rest": {"cr": [118818], "cwx": [6924], "in": [496], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [6], "out": [1286], "what": ["session_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.032656,
      "usd_client": 0.0326558,
      "usd_recomputed": 0.032656,
      "usd_recomputed_low": 0.027463
     },
     "pair": 1,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.0690476,
      "usd_client": 0.06904759999999999,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 6, "sum_complete": true},
      "rest": {"cr": [126364], "cwx": [8819], "in": [496], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [6], "out": [1380], "what": ["session_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.03767,
      "usd_client": 0.0376704,
      "usd_recomputed": 0.03767,
      "usd_recomputed_low": 0.031056
     },
     "pair": 2,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.0533572,
      "usd_client": 0.0533572,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 6, "sum_complete": true},
      "rest": {"cr": [118279], "cwx": [6703], "in": [496], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [6], "out": [1217], "what": ["session_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.031815,
      "usd_client": 0.03181489999999999,
      "usd_recomputed": 0.031815,
      "usd_recomputed_low": 0.026788
     },
     "pair": 3,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.0675048,
      "usd_client": 0.0675048,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 5, "sum_complete": true},
      "rest": {"cr": [102804], "cwx": [8348], "in": [596], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [5], "out": [1145], "what": ["session_total"]},
      "success": true,
      "turns": 5,
      "usd": 0.033297,
      "usd_client": 0.0332974,
      "usd_recomputed": 0.033297,
      "usd_recomputed_low": 0.027036
     },
     "pair": 4,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.0683062,
      "usd_client": 0.0683062,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 6, "sum_complete": true},
      "rest": {"cr": [126100], "cwx": [8657], "in": [600], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [6], "out": [1205], "what": ["session_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.036549,
      "usd_client": 0.036549,
      "usd_recomputed": 0.036549,
      "usd_recomputed_low": 0.030056
     },
     "pair": 5,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.0691958,
      "usd_client": 0.0691958,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 7, "sum_complete": true},
      "rest": {"cr": [140259], "cwx": [7210], "in": [608], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [7], "out": [1517], "what": ["session_total"]},
      "success": true,
      "turns": 7,
      "usd": 0.036639,
      "usd_client": 0.0366389,
      "usd_recomputed": 0.036639,
      "usd_recomputed_low": 0.031231
     },
     "pair": 6,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.0682921,
      "usd_client": 0.0682921,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 6, "sum_complete": true},
      "rest": {"cr": [126269], "cwx": [8797], "in": [601], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [6], "out": [1389], "what": ["session_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.037767,
      "usd_client": 0.0377669,
      "usd_recomputed": 0.037767,
      "usd_recomputed_low": 0.031169
     },
     "pair": 7,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.0530823,
      "usd_client": 0.0530823,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 6, "sum_complete": true},
      "rest": {"cr": [125895], "cwx": [8569], "in": [613], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [6], "out": [1161], "what": ["session_total"]},
      "success": true,
      "turns": 6,
      "usd": 0.036146,
      "usd_client": 0.0361455,
      "usd_recomputed": 0.036145,
      "usd_recomputed_low": 0.029719
     },
     "pair": 8,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.0697,
      "usd_client": 0.06969999999999998,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    }
   ],
   "pairs_planned": 8
  },
  {
   "analysis": {"gateway_usd_mean": 0.06238, "gateway_wins": 12, "n": 12, "pct": -64.4, "pct_ci95": [-76.5,-52.3], "plain_usd_mean": 0.17522, "success_gateway": "11/12", "success_plain": "11/12", "verdict": "less"},
   "id": "W3",
   "invalid_attempts": 3,
   "label": "Large documents processed with a script",
   "pairs": [
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [44496,35300,18775], "cwx": [5079,2250,1834], "in": [4547,3702,613], "model": [0,0,0], "models": ["claude-haiku-4-5-20251001"], "out": [1878,2516,536]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.061695,
      "usd_client": 0.043023500000000006,
      "usd_recomputed": 0.061695,
      "usd_recomputed_low": 0.054823,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 1,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [279927], "cwx": [18878], "in": [87857], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [null], "out": [6462], "what": ["per_model_total"]},
      "success": true,
      "turns": 12,
      "usd": 0.1859157,
      "usd_client": 0.18591570000000002,
      "usd_recomputed": 0.185916,
      "usd_recomputed_low": 0.171757
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [44494,35264,18740], "cwx": [5078,2216,1832], "in": [4221,4671,613], "model": [0,0,0], "models": ["claude-haiku-4-5-20251001"], "out": [1506,2450,476]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.059767,
      "usd_client": 0.0438104,
      "usd_recomputed": 0.059767,
      "usd_recomputed_low": 0.052922,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 2,
     "plain": {
      "answer_checks": {"react_version_count_exact": false},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [241433], "cwx": [19375], "in": [133712], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [null], "out": [6233], "what": ["per_model_total"]},
      "success": true,
      "turns": 10,
      "usd": 0.2277703,
      "usd_client": 0.22777029999999998,
      "usd_recomputed": 0.22777,
      "usd_recomputed_low": 0.213239
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [27971,35130,18605], "cwx": [5079,2080,1674], "in": [1941,4379,611], "model": [0,0,0], "models": ["claude-haiku-4-5-20251001"], "out": [894,2215,759]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.052108,
      "usd_client": 0.0424075,
      "usd_recomputed": 0.052108,
      "usd_recomputed_low": 0.045483,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 3,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [268235], "cwx": [18660], "in": [87861], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [null], "out": [6433], "what": ["per_model_total"]},
      "success": true,
      "turns": 11,
      "usd": 0.1841695,
      "usd_client": 0.1841695,
      "usd_recomputed": 0.18417,
      "usd_recomputed_low": 0.170175
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [44496,35226,18701], "cwx": [5079,2176,2523], "in": [4474,4383,612], "model": [0,0,0], "models": ["claude-haiku-4-5-20251001"], "out": [1606,3078,404]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.064307,
      "usd_client": 0.04786570000000001,
      "usd_recomputed": 0.064307,
      "usd_recomputed_low": 0.056974,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 4,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [205130], "cwx": [17730], "in": [87884], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [null], "out": [5652], "what": ["per_model_total"]},
      "success": false,
      "turns": 9,
      "usd": 0.172117,
      "usd_client": 0.172117,
      "usd_recomputed": 0.172117,
      "usd_recomputed_low": 0.15882
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [27972,35172,18646], "cwx": [5080,2120,1808], "in": [2136,3781,612], "model": [0,0,0], "models": ["claude-haiku-4-5-20251001"], "out": [907,2436,427]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.051574,
      "usd_client": 0.040904800000000005,
      "usd_recomputed": 0.051574,
      "usd_recomputed_low": 0.044818,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 5,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [346535], "cwx": [20430], "in": [87938], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [null], "out": [7746], "what": ["per_model_total"]},
      "success": true,
      "turns": 14,
      "usd": 0.2021815,
      "usd_client": 0.2021815,
      "usd_recomputed": 0.202181,
      "usd_recomputed_low": 0.186859
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [27972,53892,18683], "cwx": [5080,2157,1720], "in": [2205,5456,611], "model": [0,0,0], "models": ["claude-haiku-4-5-20251001"], "out": [1128,2844,496]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.058581,
      "usd_client": 0.0415472,
      "usd_recomputed": 0.058581,
      "usd_recomputed_low": 0.051863,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 6,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [313860], "cwx": [18108], "in": [87858], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [null], "out": [5751], "what": ["per_model_total"]},
      "success": true,
      "turns": 13,
      "usd": 0.184215,
      "usd_client": 0.18421499999999996,
      "usd_recomputed": 0.184215,
      "usd_recomputed_low": 0.170634
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [44496,54055,18765], "cwx": [5079,2240,2767], "in": [4276,6960,611], "model": [0,0,0], "models": ["claude-haiku-4-5-20251001"], "out": [1571,4110,511]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.074711,
      "usd_client": 0.0512885,
      "usd_recomputed": 0.074711,
      "usd_recomputed_low": 0.067146,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 7,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [234727], "cwx": [17627], "in": [43946], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [null], "out": [5738], "what": ["per_model_total"]},
      "success": true,
      "turns": 10,
      "usd": 0.1313627,
      "usd_client": 0.1313627,
      "usd_recomputed": 0.131363,
      "usd_recomputed_low": 0.118142
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [27971,35133,18608], "cwx": [5079,2083,250], "in": [2165,4470,611], "model": [0,0,0], "models": ["claude-haiku-4-5-20251001"], "out": [924,888,378]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": false,
      "turns": 3,
      "usd": 0.041191,
      "usd_client": 0.0305111,
      "usd_recomputed": 0.041191,
      "usd_recomputed_low": 0.035632,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 8,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [215919], "cwx": [18020], "in": [43982], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [null], "out": [5987], "what": ["per_model_total"]},
      "success": true,
      "turns": 9,
      "usd": 0.1315489,
      "usd_client": 0.13154890000000002,
      "usd_recomputed": 0.131549,
      "usd_recomputed_low": 0.118034
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [27972,35256,18730], "cwx": [5080,2204,1830], "in": [2208,3632,611], "model": [0,0,0], "models": ["claude-haiku-4-5-20251001"], "out": [1023,2359,580]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.052685,
      "usd_client": 0.042339600000000005,
      "usd_recomputed": 0.052685,
      "usd_recomputed_low": 0.045849,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 9,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [290870], "cwx": [18142], "in": [87904], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [null], "out": [5632], "what": ["per_model_total"]},
      "success": true,
      "turns": 12,
      "usd": 0.181435,
      "usd_client": 0.18143499999999999,
      "usd_recomputed": 0.181435,
      "usd_recomputed_low": 0.167828
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [11446,16525,43699,19014,39528,19960,44703,41239,20831,24149], "cwx": [5079,2489,0,554,392,448,0,423,3318,2504], "in": [650,10,181,8,778,608,175,892,8,608], "model": [0,0,0,0,0,0,0,0,0,0], "models": ["claude-haiku-4-5-20251001"], "out": [303,241,297,222,850,220,187,1543,2481,547]},
      "calls_from": "ledger",
      "coverage": {"calls": 10, "of": 11, "sum_complete": true},
      "rest": {"cr": [43699], "in": [175], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [1], "out": [155], "what": ["minus_saved_rows"]},
      "success": true,
      "turns": 8,
      "usd": 0.102216,
      "usd_client": 0.0915027,
      "usd_recomputed": 0.102216,
      "usd_recomputed_low": 0.090811,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 10,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [239246], "cwx": [18586], "in": [87863], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [null], "out": [6279], "what": ["per_model_total"]},
      "success": true,
      "turns": 10,
      "usd": 0.1803546,
      "usd_client": 0.18035460000000003,
      "usd_recomputed": 0.180355,
      "usd_recomputed_low": 0.166415
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [44496,35308,18783], "cwx": [5079,2258,2732], "in": [4431,4547,608], "model": [0,0,0], "models": ["claude-haiku-4-5-20251001"], "out": [1693,3292,563]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.067323,
      "usd_client": 0.050705099999999996,
      "usd_recomputed": 0.067323,
      "usd_recomputed_low": 0.059771,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 11,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [263675], "cwx": [18502], "in": [43982], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [null], "out": [5944], "what": ["per_model_total"]},
      "success": true,
      "turns": 11,
      "usd": 0.1370735,
      "usd_client": 0.13707350000000001,
      "usd_recomputed": 0.137073,
      "usd_recomputed_low": 0.123197
     }
    },
    {
     "gateway": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"cr": [61024,35140,18614], "cwx": [5080,2088,1764], "in": [5597,5333,608], "model": [0,0,0], "models": ["claude-haiku-4-5-20251001"], "out": [1343,2431,520]},
      "calls_from": "ledger",
      "coverage": {"calls": 3, "of": 3, "sum_complete": true},
      "success": true,
      "turns": 3,
      "usd": 0.06235,
      "usd_client": 0.0434964,
      "usd_recomputed": 0.06235,
      "usd_recomputed_low": 0.055651,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 12,
     "plain": {
      "answer_checks": {"react_version_count_exact": true},
      "calls": {"model": [], "models": []},
      "calls_from": "client_totals",
      "coverage": {"calls": 0, "of": null, "sum_complete": true},
      "rest": {"cr": [206136], "cwx": [19488], "in": [87874], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [null], "out": [7412], "what": ["per_model_total"]},
      "success": true,
      "turns": 9,
      "usd": 0.1845236,
      "usd_client": 0.18452359999999998,
      "usd_recomputed": 0.184524,
      "usd_recomputed_low": 0.169908
     }
    }
   ],
   "pairs_planned": 12
  },
  {
   "analysis": {"gateway_usd_mean": 0.01998, "gateway_wins": 12, "n": 12, "pct": -19.2, "pct_ci95": [-22.5,-15.8], "plain_usd_mean": 0.02471, "success_gateway": "12/12", "success_plain": "12/12", "verdict": "less"},
   "id": "git",
   "invalid_attempts": 2,
   "label": "Repo history question (git log)",
   "pairs": [
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [56704], "cwx": [6834], "in": [296], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [226], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.020764,
      "usd_client": 0.0207644,
      "usd_recomputed": 0.020764,
      "usd_recomputed_low": 0.015639
     },
     "pair": 1,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0249187,
      "usd_client": 0.024918700000000002,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [56703], "cwx": [6892], "in": [295], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [203], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.020764,
      "usd_client": 0.0207643,
      "usd_recomputed": 0.020764,
      "usd_recomputed_low": 0.015595
     },
     "pair": 2,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.027604,
      "usd_client": 0.027604,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 2, "sum_complete": true},
      "rest": {"cr": [34973], "cwx": [6696], "in": [290], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [2], "out": [126], "what": ["session_total"]},
      "success": true,
      "turns": 2,
      "usd": 0.017809,
      "usd_client": 0.0178093,
      "usd_recomputed": 0.017809,
      "usd_recomputed_low": 0.012787
     },
     "pair": 3,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.0253449,
      "usd_client": 0.0253449,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [56655], "cwx": [6750], "in": [296], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [184], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.020382,
      "usd_client": 0.0203815,
      "usd_recomputed": 0.020382,
      "usd_recomputed_low": 0.015319
     },
     "pair": 4,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0245073,
      "usd_client": 0.0245073,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [56697], "cwx": [6793], "in": [353], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [196], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.020589,
      "usd_client": 0.0205887,
      "usd_recomputed": 0.020589,
      "usd_recomputed_low": 0.015494
     },
     "pair": 5,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0250857,
      "usd_client": 0.0250857,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 4, "sum_complete": true},
      "rest": {"cr": [75539], "cwx": [5446], "in": [359], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [4], "out": [266], "what": ["session_total"]},
      "success": true,
      "turns": 4,
      "usd": 0.020135,
      "usd_client": 0.0201349,
      "usd_recomputed": 0.020135,
      "usd_recomputed_low": 0.01605
     },
     "pair": 6,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0250902,
      "usd_client": 0.025090199999999997,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [54877], "cwx": [5046], "in": [354], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [191], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.016889,
      "usd_client": 0.0168887,
      "usd_recomputed": 0.016889,
      "usd_recomputed_low": 0.013104
     },
     "pair": 7,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 2,
      "usd": 0.0216917,
      "usd_client": 0.0216917,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [56656], "cwx": [6750], "in": [353], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [195], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.020494,
      "usd_client": 0.020493599999999997,
      "usd_recomputed": 0.020494,
      "usd_recomputed_low": 0.015431
     },
     "pair": 8,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0250531,
      "usd_client": 0.025053100000000002,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [56683], "cwx": [6780], "in": [353], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [169], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.020426,
      "usd_client": 0.0204263,
      "usd_recomputed": 0.020426,
      "usd_recomputed_low": 0.015341
     },
     "pair": 9,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0246814,
      "usd_client": 0.0246814,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [56650], "cwx": [6770], "in": [353], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [185], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.020483,
      "usd_client": 0.020483,
      "usd_recomputed": 0.020483,
      "usd_recomputed_low": 0.015406
     },
     "pair": 10,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0230388,
      "usd_client": 0.023038799999999998,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [56653], "cwx": [6749], "in": [353], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [183], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.020431,
      "usd_client": 0.0204313,
      "usd_recomputed": 0.020431,
      "usd_recomputed_low": 0.01537
     },
     "pair": 11,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0250247,
      "usd_client": 0.025024699999999997,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [56699], "cwx": [6797], "in": [353], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [188], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.020557,
      "usd_client": 0.0205569,
      "usd_recomputed": 0.020557,
      "usd_recomputed_low": 0.015459
     },
     "pair": 12,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0245285,
      "usd_client": 0.024528499999999998,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    }
   ],
   "pairs_planned": 12
  },
  {
   "analysis": {"gateway_usd_mean": 0.02232, "gateway_wins": 18, "n": 20, "pct": -28.9, "pct_ci95": [-53.8,-3.9], "plain_usd_mean": 0.03137, "success_gateway": "20/20", "success_plain": "20/20", "verdict": "less"},
   "id": "grep",
   "invalid_attempts": 1,
   "label": "Code search (grep)",
   "pairs": [
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [57247], "cwx": [7697], "in": [487], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [348], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.023346,
      "usd_client": 0.023345699999999997,
      "usd_recomputed": 0.023346,
      "usd_recomputed_low": 0.017573
     },
     "pair": 1,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.0803128,
      "usd_client": 0.0803128,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [57247], "cwx": [7572], "in": [487], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [224], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.022476,
      "usd_client": 0.0224757,
      "usd_recomputed": 0.022476,
      "usd_recomputed_low": 0.016797
     },
     "pair": 2,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0267334,
      "usd_client": 0.0267334,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [55355], "cwx": [5807], "in": [540], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [208], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.01873,
      "usd_client": 0.018729500000000003,
      "usd_recomputed": 0.018729,
      "usd_recomputed_low": 0.014374
     },
     "pair": 3,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0265968,
      "usd_client": 0.0265968,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [57256], "cwx": [7575], "in": [540], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [220], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.022516,
      "usd_client": 0.0225156,
      "usd_recomputed": 0.022516,
      "usd_recomputed_low": 0.016834
     },
     "pair": 4,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0274347,
      "usd_client": 0.0274347,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [57254], "cwx": [7572], "in": [540], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [218], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.022499,
      "usd_client": 0.0224994,
      "usd_recomputed": 0.022499,
      "usd_recomputed_low": 0.01682
     },
     "pair": 5,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0266039,
      "usd_client": 0.0266039,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [102105], "cwx": [7622], "in": [3343], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [397], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.030783,
      "usd_client": 0.0248019,
      "usd_recomputed": 0.030783,
      "usd_recomputed_low": 0.025066,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 6,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0277873,
      "usd_client": 0.027787299999999997,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [57250], "cwx": [7572], "in": [544], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [194], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.022383,
      "usd_client": 0.022382999999999997,
      "usd_recomputed": 0.022383,
      "usd_recomputed_low": 0.016704
     },
     "pair": 7,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0229915,
      "usd_client": 0.0229915,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [57253], "cwx": [7573], "in": [544], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [222], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.022525,
      "usd_client": 0.0225253,
      "usd_recomputed": 0.022525,
      "usd_recomputed_low": 0.016846
     },
     "pair": 8,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0266468,
      "usd_client": 0.026646799999999998,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [57249], "cwx": [7590], "in": [544], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [251], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.022704,
      "usd_client": 0.0227039,
      "usd_recomputed": 0.022704,
      "usd_recomputed_low": 0.017011
     },
     "pair": 9,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0267339,
      "usd_client": 0.0267339,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [55347], "cwx": [5799], "in": [544], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [190], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.018627,
      "usd_client": 0.0186267,
      "usd_recomputed": 0.018627,
      "usd_recomputed_low": 0.014277
     },
     "pair": 10,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0276364,
      "usd_client": 0.0276364,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [57249], "cwx": [7577], "in": [544], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [208], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.022463,
      "usd_client": 0.022462899999999997,
      "usd_recomputed": 0.022463,
      "usd_recomputed_low": 0.01678
     },
     "pair": 11,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.0790653,
      "usd_client": 0.0790653,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [57250], "cwx": [7570], "in": [536], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [216], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.022481,
      "usd_client": 0.022481,
      "usd_recomputed": 0.022481,
      "usd_recomputed_low": 0.016803
     },
     "pair": 12,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0265781,
      "usd_client": 0.0265781,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [57243], "cwx": [7574], "in": [542], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [218], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.022504,
      "usd_client": 0.022504299999999998,
      "usd_recomputed": 0.022504,
      "usd_recomputed_low": 0.016824
     },
     "pair": 13,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0273182,
      "usd_client": 0.027318199999999997,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 2, "sum_complete": true},
      "rest": {"cr": [34822], "cwx": [6560], "in": [537], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [2], "out": [146], "what": ["session_total"]},
      "success": true,
      "turns": 2,
      "usd": 0.017869,
      "usd_client": 0.0178692,
      "usd_recomputed": 0.017869,
      "usd_recomputed_low": 0.012949
     },
     "pair": 14,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0264945,
      "usd_client": 0.0264945,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 2, "sum_complete": true},
      "rest": {"cr": [34822], "cwx": [6557], "in": [535], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [2], "out": [141], "what": ["session_total"]},
      "success": true,
      "turns": 2,
      "usd": 0.017836,
      "usd_client": 0.0178362,
      "usd_recomputed": 0.017836,
      "usd_recomputed_low": 0.012918
     },
     "pair": 15,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0228906,
      "usd_client": 0.0228906,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [57252], "cwx": [7573], "in": [540], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [205], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.022436,
      "usd_client": 0.0224362,
      "usd_recomputed": 0.022436,
      "usd_recomputed_low": 0.016756
     },
     "pair": 16,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0266079,
      "usd_client": 0.0266079,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [55360], "cwx": [5812], "in": [540], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [223], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.018815,
      "usd_client": 0.018815000000000002,
      "usd_recomputed": 0.018815,
      "usd_recomputed_low": 0.014456
     },
     "pair": 17,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0227218,
      "usd_client": 0.0227218,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [57247], "cwx": [7566], "in": [540], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [216], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.022477,
      "usd_client": 0.022476700000000002,
      "usd_recomputed": 0.022477,
      "usd_recomputed_low": 0.016802
     },
     "pair": 18,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0264668,
      "usd_client": 0.026466800000000002,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 5, "sum_complete": true},
      "rest": {"cr": [102699], "cwx": [8136], "in": [552], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [5], "out": [643], "what": ["session_total"]},
      "success": true,
      "turns": 5,
      "usd": 0.030309,
      "usd_client": 0.030308900000000007,
      "usd_recomputed": 0.030309,
      "usd_recomputed_low": 0.024207
     },
     "pair": 19,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0229396,
      "usd_client": 0.022939600000000004,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [57253], "cwx": [7577], "in": [528], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [229], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.022552,
      "usd_client": 0.0225523,
      "usd_recomputed": 0.022552,
      "usd_recomputed_low": 0.01687
     },
     "pair": 20,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0267675,
      "usd_client": 0.0267675,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    }
   ],
   "pairs_planned": 20
  },
  {
   "analysis": {"gateway_usd_mean": 0.02658, "gateway_wins": 18, "n": 20, "pct": -16.1, "pct_ci95": [-22.8,-9.3], "plain_usd_mean": 0.03167, "success_gateway": "20/20", "success_plain": "20/20", "verdict": "less"},
   "id": "json",
   "invalid_attempts": 2,
   "label": "One large JSON file",
   "pairs": [
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 4, "sum_complete": true},
      "rest": {"cr": [80018], "cwx": [7850], "in": [400], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [4], "out": [278], "what": ["session_total"]},
      "success": true,
      "turns": 4,
      "usd": 0.025492,
      "usd_client": 0.0254918,
      "usd_recomputed": 0.025492,
      "usd_recomputed_low": 0.019604
     },
     "pair": 1,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.0299731,
      "usd_client": 0.029973099999999996,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [102201], "cwx": [7683], "in": [5897], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [508], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.034023,
      "usd_client": 0.0272949,
      "usd_recomputed": 0.034023,
      "usd_recomputed_low": 0.028261,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 2,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.0296565,
      "usd_client": 0.0296565,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [57282], "cwx": [7613], "in": [394], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [246], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.022578,
      "usd_client": 0.0225782,
      "usd_recomputed": 0.022578,
      "usd_recomputed_low": 0.016868
     },
     "pair": 3,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 5,
      "usd": 0.0335048,
      "usd_client": 0.0335048,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 4, "sum_complete": true},
      "rest": {"cr": [80113], "cwx": [8243], "in": [400], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [4], "out": [530], "what": ["session_total"]},
      "success": true,
      "turns": 4,
      "usd": 0.027547,
      "usd_client": 0.027547300000000004,
      "usd_recomputed": 0.027547,
      "usd_recomputed_low": 0.021365
     },
     "pair": 4,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 5,
      "usd": 0.0330622,
      "usd_client": 0.03306220000000001,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [79771], "cwx": [7705], "in": [671], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [560], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.026858,
      "usd_client": 0.0236685,
      "usd_recomputed": 0.026858,
      "usd_recomputed_low": 0.021079,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 5,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.030645,
      "usd_client": 0.030645000000000002,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 4, "sum_complete": true},
      "rest": {"cr": [80008], "cwx": [7848], "in": [400], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [4], "out": [283], "what": ["session_total"]},
      "success": true,
      "turns": 4,
      "usd": 0.025512,
      "usd_client": 0.0255118,
      "usd_recomputed": 0.025512,
      "usd_recomputed_low": 0.019626
     },
     "pair": 6,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.0299423,
      "usd_client": 0.029942299999999998,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [57282], "cwx": [7623], "in": [394], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [258], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.022658,
      "usd_client": 0.022658199999999996,
      "usd_recomputed": 0.022658,
      "usd_recomputed_low": 0.016941
     },
     "pair": 7,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 6,
      "usd": 0.0381162,
      "usd_client": 0.03811620000000001,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [102212], "cwx": [7681], "in": [5865], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [471], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.033803,
      "usd_client": 0.027243499999999997,
      "usd_recomputed": 0.033803,
      "usd_recomputed_low": 0.028042,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 8,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 5,
      "usd": 0.0337074,
      "usd_client": 0.033707400000000005,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 4, "sum_complete": true},
      "rest": {"cr": [80161], "cwx": [8283], "in": [400], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [4], "out": [537], "what": ["session_total"]},
      "success": true,
      "turns": 4,
      "usd": 0.027667,
      "usd_client": 0.027667099999999997,
      "usd_recomputed": 0.027667,
      "usd_recomputed_low": 0.021455
     },
     "pair": 9,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.0307484,
      "usd_client": 0.030748399999999995,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 5, "sum_complete": true},
      "rest": {"cr": [102905], "cwx": [8221], "in": [406], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [5], "out": [355], "what": ["session_total"]},
      "success": true,
      "turns": 5,
      "usd": 0.028914,
      "usd_client": 0.0289135,
      "usd_recomputed": 0.028914,
      "usd_recomputed_low": 0.022748
     },
     "pair": 10,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.0305286,
      "usd_client": 0.0305286,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 4, "sum_complete": true},
      "rest": {"cr": [80154], "cwx": [8116], "in": [400], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [4], "out": [544], "what": ["session_total"]},
      "success": true,
      "turns": 4,
      "usd": 0.027367,
      "usd_client": 0.0273674,
      "usd_recomputed": 0.027367,
      "usd_recomputed_low": 0.02128
     },
     "pair": 11,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.0302365,
      "usd_client": 0.0302365,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 4, "sum_complete": true},
      "rest": {"cr": [102477], "cwx": [8019], "in": [581], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [4], "out": [409], "what": ["session_total"]},
      "success": true,
      "turns": 4,
      "usd": 0.028912,
      "usd_client": 0.0261472,
      "usd_recomputed": 0.028912,
      "usd_recomputed_low": 0.022897,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 12,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.0296379,
      "usd_client": 0.029637900000000002,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 4, "sum_complete": true},
      "rest": {"cr": [80154], "cwx": [8283], "in": [400], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [4], "out": [542], "what": ["session_total"]},
      "success": true,
      "turns": 4,
      "usd": 0.027691,
      "usd_client": 0.027691399999999998,
      "usd_recomputed": 0.027691,
      "usd_recomputed_low": 0.021479
     },
     "pair": 13,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.0305262,
      "usd_client": 0.0305262,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 4, "sum_complete": true},
      "rest": {"cr": [76364], "cwx": [6056], "in": [400], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [4], "out": [272], "what": ["session_total"]},
      "success": true,
      "turns": 4,
      "usd": 0.021508,
      "usd_client": 0.021508399999999997,
      "usd_recomputed": 0.021508,
      "usd_recomputed_low": 0.016966
     },
     "pair": 14,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.0305195,
      "usd_client": 0.030519500000000005,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 2, "sum_complete": true},
      "rest": {"cr": [79744], "cwx": [7425], "in": [3071], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [2], "out": [301], "what": ["session_total"]},
      "success": true,
      "turns": 2,
      "usd": 0.0274,
      "usd_client": 0.0210191,
      "usd_recomputed": 0.0274,
      "usd_recomputed_low": 0.021832,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 15,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.030106,
      "usd_client": 0.030106,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 4, "sum_complete": true},
      "rest": {"cr": [80027], "cwx": [7854], "in": [400], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [4], "out": [280], "what": ["session_total"]},
      "success": true,
      "turns": 4,
      "usd": 0.025511,
      "usd_client": 0.0255107,
      "usd_recomputed": 0.025511,
      "usd_recomputed_low": 0.01962
     },
     "pair": 16,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.0317428,
      "usd_client": 0.0317428,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [57289], "cwx": [7553], "in": [394], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [196], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.022209,
      "usd_client": 0.0222089,
      "usd_recomputed": 0.022209,
      "usd_recomputed_low": 0.016544
     },
     "pair": 17,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.0298614,
      "usd_client": 0.0298614,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 4, "sum_complete": true},
      "rest": {"cr": [80028], "cwx": [8021], "in": [400], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [4], "out": [275], "what": ["session_total"]},
      "success": true,
      "turns": 4,
      "usd": 0.02582,
      "usd_client": 0.0258198,
      "usd_recomputed": 0.02582,
      "usd_recomputed_low": 0.019804
     },
     "pair": 18,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.0298924,
      "usd_client": 0.0298924,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 2, "sum_complete": true},
      "rest": {"cr": [57266], "cwx": [7407], "in": [635], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [2], "out": [159], "what": ["session_total"]},
      "success": true,
      "turns": 2,
      "usd": 0.021971,
      "usd_client": 0.0195713,
      "usd_recomputed": 0.021971,
      "usd_recomputed_low": 0.016415,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 19,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 4,
      "usd": 0.0302635,
      "usd_client": 0.030263500000000002,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 4, "sum_complete": true},
      "rest": {"cr": [102411], "cwx": [7788], "in": [557], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [4], "out": [372], "what": ["session_total"]},
      "success": true,
      "turns": 4,
      "usd": 0.028234,
      "usd_client": 0.0257046,
      "usd_recomputed": 0.028234,
      "usd_recomputed_low": 0.022393,
      "usd_vs_client": {"relation": "above", "why": "edge"}
     },
     "pair": 20,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 7,
      "usd": 0.0407088,
      "usd_client": 0.0407088,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    }
   ],
   "pairs_planned": 20
  },
  {
   "analysis": {"gateway_usd_mean": 0.02034, "gateway_wins": 8, "n": 8, "pct": -20, "pct_ci95": [-24.5,-15.4], "plain_usd_mean": 0.02542, "success_gateway": "8/8", "success_plain": "8/8", "verdict": "less"},
   "id": "edit",
   "invalid_attempts": 1,
   "label": "One-line edit",
   "pairs": [
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [54479], "cwx": [4926], "in": [504], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [218], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.016894,
      "usd_client": 0.0168939,
      "usd_recomputed": 0.016894,
      "usd_recomputed_low": 0.013199
     },
     "pair": 1,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0253199,
      "usd_client": 0.025319900000000003,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [56363], "cwx": [6682], "in": [561], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [217], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.020646,
      "usd_client": 0.0206463,
      "usd_recomputed": 0.020646,
      "usd_recomputed_low": 0.015635
     },
     "pair": 2,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0254445,
      "usd_client": 0.0254445,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [56376], "cwx": [6692], "in": [561], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [226], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.020713,
      "usd_client": 0.020712599999999998,
      "usd_recomputed": 0.020713,
      "usd_recomputed_low": 0.015694
     },
     "pair": 3,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0254352,
      "usd_client": 0.0254352,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [56377], "cwx": [6693], "in": [561], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [227], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.02072,
      "usd_client": 0.0207197,
      "usd_recomputed": 0.02072,
      "usd_recomputed_low": 0.0157
     },
     "pair": 4,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0253287,
      "usd_client": 0.025328700000000003,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [56379], "cwx": [6696], "in": [561], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [229], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.020736,
      "usd_client": 0.0207359,
      "usd_recomputed": 0.020736,
      "usd_recomputed_low": 0.015714
     },
     "pair": 5,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.02544,
      "usd_client": 0.025439999999999997,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [56379], "cwx": [6696], "in": [561], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [230], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.020741,
      "usd_client": 0.0207409,
      "usd_recomputed": 0.020741,
      "usd_recomputed_low": 0.015719
     },
     "pair": 6,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0254423,
      "usd_client": 0.0254423,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [56383], "cwx": [6699], "in": [559], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [234], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.020765,
      "usd_client": 0.0207653,
      "usd_recomputed": 0.020765,
      "usd_recomputed_low": 0.015741
     },
     "pair": 7,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0255085,
      "usd_client": 0.0255085,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    },
    {
     "gateway": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": 3, "sum_complete": true},
      "rest": {"cr": [56421], "cwx": [6849], "in": [569], "model": [0], "models": ["claude-haiku-4-5-20251001"], "n": [3], "out": [320], "what": ["session_total"]},
      "success": true,
      "turns": 3,
      "usd": 0.021509,
      "usd_client": 0.021509100000000003,
      "usd_recomputed": 0.021509,
      "usd_recomputed_low": 0.016372
     },
     "pair": 8,
     "plain": {
      "calls": {"model": [], "models": []},
      "calls_from": "none",
      "coverage": {"calls": 0, "of": null, "sum_complete": false},
      "success": true,
      "turns": 3,
      "usd": 0.0254231,
      "usd_client": 0.025423100000000004,
      "usd_recomputed": null,
      "usd_recomputed_reason": "no_tokens"
     }
    }
   ],
   "pairs_planned": 8
  }
 ]
}