{
 "note": "Receipts of the Haiku thinking study: Claude Haiku 4.5 through Claude Code with thinking off (MAX_THINKING_TOKENS=0) against the recorded thinking-on arms. Routing arms carry per-arm totals computed from each arm’s per-call log and per-case results; hard-task receipts are every attempt of the thinking-off run. Every attempt is kept, failures included. The thinking-on hard-task receipts live in the hard head-to-head extract.",
 "batches": [
  {
   "batch": "routing-1",
   "route": "claude-cli",
   "startedAt": "2026-10-06T21:29:57.270Z",
   "finishedAt": "2026-10-06T21:36:50.735Z",
   "stopReason": null,
   "trimmed": [],
   "calls": 82
  },
  {
   "batch": "hard-1",
   "route": "claude-cli",
   "startedAt": "2026-10-06T21:37:22.828Z",
   "finishedAt": "2026-10-06T21:39:31.930Z",
   "stopReason": null,
   "trimmed": [],
   "calls": 24
  }
 ],
 "caseSets": [
  {
   "purpose": "control.failure_class",
   "label": "Failure class",
   "version": "failure-class-2026-10-05.1",
   "askedCases": 18,
   "keys": 18
  },
  {
   "purpose": "frontdoor.intent",
   "label": "Message intent",
   "version": "frontdoor-intent-2026-10-04.2",
   "askedCases": 20,
   "keys": 20
  },
  {
   "purpose": "memory.is_rule",
   "label": "Is it a rule?",
   "version": "memory-is-rule-2026-10-04.2",
   "askedCases": 12,
   "keys": 12
  },
  {
   "purpose": "context.shape",
   "label": "Context shape",
   "version": "context-shape-2026-10-05.1",
   "askedCases": 32,
   "keys": 144
  }
 ],
 "routing": {
  "arms": [
   {
    "id": "haiku-thinking-off",
    "label": "Claude Haiku 4.5 (thinking off)",
    "thinking": "off",
    "thinkingSetting": "MAX_THINKING_TOKENS=0",
    "effort": null,
    "source": "this study",
    "calls": 82,
    "okCalls": 82,
    "errorCalls": 0,
    "models": {
     "claude-haiku-4-5": 82
    },
    "latencyMs": {
     "wallP50": 4662,
     "wallP95": 8184,
     "wallMin": 2198,
     "wallMax": 11573,
     "apiP50": 3786,
     "apiP95": 7431,
     "apiMin": 1371,
     "apiMax": 10829
    },
    "tokensPerDecision": {
     "input": 1534.3,
     "cacheRead": 0,
     "cacheWrite": 0,
     "output": 365.9,
     "thinking": 0,
     "visible": 365.9
    },
    "thinkingTokens": {
     "min": 0,
     "max": 0,
     "median": 0,
     "callsWithThinking": 0
    },
    "listCostPer1000Usd": 3.364,
    "cliReportedCostPer1000Usd": 3.364,
    "stopped": null,
    "pooled": {
     "cases": 82,
     "exact": 71,
     "exactRate": 0.8659,
     "exactCi": [
      0.7755,
      0.9234
     ],
     "keys": 194,
     "keysCorrect": 177,
     "keyAccuracy": 0.9124,
     "keyCi": [
      0.8642,
      0.9446
     ]
    },
    "purposes": {
     "control.failure_class": {
      "label": "Failure class",
      "version": "failure-class-2026-10-05.1",
      "calls": 18,
      "errors": 0,
      "cases": 18,
      "exact": 15,
      "exactCi": [
       0.6078,
       0.9416
      ],
      "keys": 18,
      "keysCorrect": 15
     },
     "frontdoor.intent": {
      "label": "Message intent",
      "version": "frontdoor-intent-2026-10-04.2",
      "calls": 20,
      "errors": 0,
      "cases": 20,
      "exact": 20,
      "exactCi": [
       0.8389,
       1
      ],
      "keys": 20,
      "keysCorrect": 20
     },
     "memory.is_rule": {
      "label": "Is it a rule?",
      "version": "memory-is-rule-2026-10-04.2",
      "calls": 12,
      "errors": 0,
      "cases": 12,
      "exact": 12,
      "exactCi": [
       0.7575,
       1
      ],
      "keys": 12,
      "keysCorrect": 12
     },
     "context.shape": {
      "label": "Context shape",
      "version": "context-shape-2026-10-05.1",
      "calls": 32,
      "errors": 0,
      "cases": 32,
      "exact": 24,
      "exactCi": [
       0.5789,
       0.8675
      ],
      "keys": 144,
      "keysCorrect": 130
     }
    },
    "ranOn": "2026-10-06"
   },
   {
    "id": "haiku-thinking-on",
    "label": "Claude Haiku 4.5 (thinking on)",
    "thinking": "on",
    "thinkingSetting": "CLI default",
    "effort": null,
    "source": "reference: recorded routing run",
    "calls": 82,
    "okCalls": 82,
    "errorCalls": 0,
    "models": {
     "claude-haiku-4-5": 82
    },
    "latencyMs": {
     "wallP50": 12543,
     "wallP95": 34481,
     "wallMin": 5857,
     "wallMax": 51278,
     "apiP50": 10508,
     "apiP95": 32132,
     "apiMin": 4328,
     "apiMax": 49487
    },
    "tokensPerDecision": {
     "input": 1831.4,
     "cacheRead": 0,
     "cacheWrite": 0,
     "output": 1418.6,
     "thinking": 1100.8,
     "visible": 317.8
    },
    "thinkingTokens": {
     "min": 285,
     "max": 4443,
     "median": 799.5,
     "callsWithThinking": 82
    },
    "listCostPer1000Usd": 8.924,
    "cliReportedCostPer1000Usd": 8.924,
    "stopped": null,
    "pooled": {
     "cases": 82,
     "exact": 73,
     "exactRate": 0.8902,
     "exactCi": [
      0.8044,
      0.9412
     ],
     "keys": 194,
     "keysCorrect": 183,
     "keyAccuracy": 0.9433,
     "keyCi": [
      0.9013,
      0.968
     ]
    },
    "purposes": {
     "control.failure_class": {
      "label": "Failure class",
      "version": "failure-class-2026-10-05.1",
      "calls": 18,
      "errors": 0,
      "cases": 18,
      "exact": 17,
      "exactCi": [
       0.7424,
       0.9901
      ],
      "keys": 18,
      "keysCorrect": 17
     },
     "frontdoor.intent": {
      "label": "Message intent",
      "version": "frontdoor-intent-2026-10-04.2",
      "calls": 20,
      "errors": 0,
      "cases": 20,
      "exact": 20,
      "exactCi": [
       0.8389,
       1
      ],
      "keys": 20,
      "keysCorrect": 20
     },
     "memory.is_rule": {
      "label": "Is it a rule?",
      "version": "memory-is-rule-2026-10-04.2",
      "calls": 12,
      "errors": 0,
      "cases": 12,
      "exact": 12,
      "exactCi": [
       0.7575,
       1
      ],
      "keys": 12,
      "keysCorrect": 12
     },
     "context.shape": {
      "label": "Context shape",
      "version": "context-shape-2026-10-05.1",
      "calls": 32,
      "errors": 0,
      "cases": 32,
      "exact": 24,
      "exactCi": [
       0.5789,
       0.8675
      ],
      "keys": 144,
      "keysCorrect": 134
     }
    },
    "ranOn": "2026-10-05"
   },
   {
    "id": "sonnet-low",
    "label": "Claude Sonnet 5.5 (low)",
    "thinking": "effort low",
    "thinkingSetting": "--effort low",
    "effort": "low",
    "source": "reference: recorded routing run",
    "calls": 82,
    "okCalls": 82,
    "errorCalls": 0,
    "models": {
     "claude-sonnet-5-5": 82
    },
    "latencyMs": {
     "wallP50": 2597,
     "wallP95": 4298,
     "wallMin": 1993,
     "wallMax": 5583,
     "apiP50": 1596,
     "apiP95": 2583,
     "apiMin": 1060,
     "apiMax": 4757
    },
    "tokensPerDecision": {
     "input": 2,
     "cacheRead": 230.8,
     "cacheWrite": 1551.9,
     "output": 106.6,
     "thinking": 1.5,
     "visible": 105
    },
    "thinkingTokens": {
     "min": 0,
     "max": 63,
     "median": 0,
     "callsWithThinking": 2
    },
    "listCostPer1000Usd": 7.324,
    "cliReportedCostPer1000Usd": 7.324,
    "stopped": null,
    "pooled": {
     "cases": 82,
     "exact": 77,
     "exactRate": 0.939,
     "exactCi": [
      0.8651,
      0.9737
     ],
     "keys": 194,
     "keysCorrect": 189,
     "keyAccuracy": 0.9742,
     "keyCi": [
      0.9411,
      0.9889
     ]
    },
    "purposes": {
     "control.failure_class": {
      "label": "Failure class",
      "version": "failure-class-2026-10-05.1",
      "calls": 18,
      "errors": 0,
      "cases": 18,
      "exact": 18,
      "exactCi": [
       0.8241,
       1
      ],
      "keys": 18,
      "keysCorrect": 18
     },
     "frontdoor.intent": {
      "label": "Message intent",
      "version": "frontdoor-intent-2026-10-04.2",
      "calls": 20,
      "errors": 0,
      "cases": 20,
      "exact": 20,
      "exactCi": [
       0.8389,
       1
      ],
      "keys": 20,
      "keysCorrect": 20
     },
     "memory.is_rule": {
      "label": "Is it a rule?",
      "version": "memory-is-rule-2026-10-04.2",
      "calls": 12,
      "errors": 0,
      "cases": 12,
      "exact": 12,
      "exactCi": [
       0.7575,
       1
      ],
      "keys": 12,
      "keysCorrect": 12
     },
     "context.shape": {
      "label": "Context shape",
      "version": "context-shape-2026-10-05.1",
      "calls": 32,
      "errors": 0,
      "cases": 32,
      "exact": 27,
      "exactCi": [
       0.6825,
       0.9314
      ],
      "keys": 144,
      "keysCorrect": 139
     }
    },
    "ranOn": "2026-10-05",
    "costBasis": "Calculation uses recorded one-hour cache writes at $4 per million tokens; the earlier five-minute basis was $4.996 per 1,000 decisions."
   }
  ],
  "paired": {
   "offVsOn": {
    "bothRight": 67,
    "onlyA": 4,
    "onlyB": 6,
    "bothWrong": 5,
    "discordant": [
     {
      "purpose": "control.failure_class",
      "caseId": "rate-limiter-test",
      "off": false,
      "on": true
     },
     {
      "purpose": "control.failure_class",
      "caseId": "empty-output",
      "off": false,
      "on": true
     },
     {
      "purpose": "context.shape",
      "caseId": "new-design-decision",
      "off": false,
      "on": true
     },
     {
      "purpose": "context.shape",
      "caseId": "ml-replan-oversized",
      "off": true,
      "on": false
     },
     {
      "purpose": "context.shape",
      "caseId": "ml-cont-failing-suite",
      "off": false,
      "on": true
     },
     {
      "purpose": "context.shape",
      "caseId": "ml-new-changelog-list",
      "off": true,
      "on": false
     },
     {
      "purpose": "context.shape",
      "caseId": "ml-new-backfill",
      "off": false,
      "on": true
     },
     {
      "purpose": "context.shape",
      "caseId": "asked-followup-csv",
      "off": true,
      "on": false
     },
     {
      "purpose": "context.shape",
      "caseId": "asked-clarify-limit",
      "off": false,
      "on": true
     },
     {
      "purpose": "context.shape",
      "caseId": "asked-clarify-branch",
      "off": true,
      "on": false
     }
    ],
    "cases": 82,
    "mcnemarExactP": 0.7539
   },
   "offVsSonnet": {
    "bothRight": 70,
    "onlyA": 1,
    "onlyB": 7,
    "bothWrong": 4,
    "discordant": [
     {
      "purpose": "control.failure_class",
      "caseId": "rate-limiter-test",
      "off": false,
      "sonnet": true
     },
     {
      "purpose": "control.failure_class",
      "caseId": "own-server-refused",
      "off": false,
      "sonnet": true
     },
     {
      "purpose": "control.failure_class",
      "caseId": "empty-output",
      "off": false,
      "sonnet": true
     },
     {
      "purpose": "context.shape",
      "caseId": "ml-cont-failing-suite",
      "off": false,
      "sonnet": true
     },
     {
      "purpose": "context.shape",
      "caseId": "ml-new-changelog-list",
      "off": true,
      "sonnet": false
     },
     {
      "purpose": "context.shape",
      "caseId": "ml-new-backfill",
      "off": false,
      "sonnet": true
     },
     {
      "purpose": "context.shape",
      "caseId": "asked-clarify-limit",
      "off": false,
      "sonnet": true
     },
     {
      "purpose": "context.shape",
      "caseId": "asked-followup-notes",
      "off": false,
      "sonnet": true
     }
    ],
    "cases": 82,
    "mcnemarExactP": 0.0703
   },
   "onVsSonnet": {
    "bothRight": 72,
    "onlyA": 1,
    "onlyB": 5,
    "bothWrong": 4,
    "discordant": [
     {
      "purpose": "control.failure_class",
      "caseId": "own-server-refused",
      "on": false,
      "sonnet": true
     },
     {
      "purpose": "context.shape",
      "caseId": "new-design-decision",
      "on": true,
      "sonnet": false
     },
     {
      "purpose": "context.shape",
      "caseId": "ml-replan-oversized",
      "on": false,
      "sonnet": true
     },
     {
      "purpose": "context.shape",
      "caseId": "asked-followup-csv",
      "on": false,
      "sonnet": true
     },
     {
      "purpose": "context.shape",
      "caseId": "asked-followup-notes",
      "on": false,
      "sonnet": true
     },
     {
      "purpose": "context.shape",
      "caseId": "asked-clarify-branch",
      "on": false,
      "sonnet": true
     }
    ],
    "cases": 82,
    "mcnemarExactP": 0.2188
   }
  },
  "cases": [
   {
    "purpose": "control.failure_class",
    "caseId": "missing-linker",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "control.failure_class",
    "caseId": "old-jdk",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "control.failure_class",
    "caseId": "chromedriver-version",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "control.failure_class",
    "caseId": "oom-killed",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "control.failure_class",
    "caseId": "test-timeout",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "control.failure_class",
    "caseId": "rate-limiter-test",
    "off": {
     "exact": false,
     "keys": 1,
     "keysCorrect": 0,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "control.failure_class",
    "caseId": "own-server-refused",
    "off": {
     "exact": false,
     "keys": 1,
     "keysCorrect": 0,
     "error": false
    },
    "on": {
     "exact": false,
     "keys": 1,
     "keysCorrect": 0,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "control.failure_class",
    "caseId": "error-message-test",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "control.failure_class",
    "caseId": "jest-assertion",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "control.failure_class",
    "caseId": "typescript-error",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "control.failure_class",
    "caseId": "eslint-errors",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "control.failure_class",
    "caseId": "python-assertion",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "control.failure_class",
    "caseId": "go-panic",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "control.failure_class",
    "caseId": "snapshot-mismatch",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "control.failure_class",
    "caseId": "missing-module",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "control.failure_class",
    "caseId": "eresolve",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "control.failure_class",
    "caseId": "missing-fixture",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "control.failure_class",
    "caseId": "empty-output",
    "off": {
     "exact": false,
     "keys": 1,
     "keysCorrect": 0,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "frontdoor.intent",
    "caseId": "new-dashboard",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "frontdoor.intent",
    "caseId": "whats-open-this-week",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "frontdoor.intent",
    "caseId": "fix-login",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "frontdoor.intent",
    "caseId": "migration-progress",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "frontdoor.intent",
    "caseId": "make-it-weekly",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "frontdoor.intent",
    "caseId": "approved-ship-it",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "frontdoor.intent",
    "caseId": "hold-everything",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "frontdoor.intent",
    "caseId": "cancel-redesign",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "frontdoor.intent",
    "caseId": "thanks",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "frontdoor.intent",
    "caseId": "churn-question",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "frontdoor.intent",
    "caseId": "also-canada",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "frontdoor.intent",
    "caseId": "option-b",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "frontdoor.intent",
    "caseId": "did-deploy-go-out",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "frontdoor.intent",
    "caseId": "detailed-reporting",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "frontdoor.intent",
    "caseId": "blog-post",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "frontdoor.intent",
    "caseId": "anything-blocked",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "frontdoor.intent",
    "caseId": "hello",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "frontdoor.intent",
    "caseId": "resume-please",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "frontdoor.intent",
    "caseId": "stop-using-tabs",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "frontdoor.intent",
    "caseId": "set-up-ci",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "memory.is_rule",
    "caseId": "watch-tip",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "memory.is_rule",
    "caseId": "make-targets",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "memory.is_rule",
    "caseId": "keep-in-mind",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "memory.is_rule",
    "caseId": "build-badge",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "memory.is_rule",
    "caseId": "branch-name",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "memory.is_rule",
    "caseId": "docstrings-norm",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "memory.is_rule",
    "caseId": "conventional-commits",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "memory.is_rule",
    "caseId": "pnpm-not-npm",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "memory.is_rule",
    "caseId": "example-line",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "memory.is_rule",
    "caseId": "ci-runs",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "memory.is_rule",
    "caseId": "secrets-in-vault",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "memory.is_rule",
    "caseId": "prs-need-tests",
    "off": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 1,
     "keysCorrect": 1,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "new-pagination",
    "off": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "new-migration-large",
    "off": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "new-oversized",
    "off": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "new-design-decision",
    "off": {
     "exact": false,
     "keys": 4,
     "keysCorrect": 1,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "sonnet": {
     "exact": false,
     "keys": 4,
     "keysCorrect": 3,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "new-onboarding-doc",
    "off": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "cont-mid-implementation",
    "off": {
     "exact": true,
     "keys": 3,
     "keysCorrect": 3,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 3,
     "keysCorrect": 3,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 3,
     "keysCorrect": 3,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "cont-near-done",
    "off": {
     "exact": false,
     "keys": 4,
     "keysCorrect": 3,
     "error": false
    },
    "on": {
     "exact": false,
     "keys": 4,
     "keysCorrect": 3,
     "error": false
    },
    "sonnet": {
     "exact": false,
     "keys": 4,
     "keysCorrect": 3,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "cont-oversized",
    "off": {
     "exact": true,
     "keys": 3,
     "keysCorrect": 3,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 3,
     "keysCorrect": 3,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 3,
     "keysCorrect": 3,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "followup-add-param",
    "off": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "clarify-open-questions",
    "off": {
     "exact": false,
     "keys": 4,
     "keysCorrect": 2,
     "error": false
    },
    "on": {
     "exact": false,
     "keys": 4,
     "keysCorrect": 2,
     "error": false
    },
    "sonnet": {
     "exact": false,
     "keys": 4,
     "keysCorrect": 3,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "ml-new-bulk-export",
    "off": {
     "exact": true,
     "keys": 6,
     "keysCorrect": 6,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 6,
     "keysCorrect": 6,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 6,
     "keysCorrect": 6,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "ml-new-runbook",
    "off": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "ml-new-repeat-incident",
    "off": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "ml-new-style-guide",
    "off": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "ml-cont-export-branch",
    "off": {
     "exact": true,
     "keys": 6,
     "keysCorrect": 6,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 6,
     "keysCorrect": 6,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 6,
     "keysCorrect": 6,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "ml-cont-no-summary",
    "off": {
     "exact": false,
     "keys": 4,
     "keysCorrect": 3,
     "error": false
    },
    "on": {
     "exact": false,
     "keys": 4,
     "keysCorrect": 3,
     "error": false
    },
    "sonnet": {
     "exact": false,
     "keys": 4,
     "keysCorrect": 3,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "ml-cont-sso-docs",
    "off": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "ml-cont-open-question",
    "off": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "ml-new-split-service",
    "off": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "ml-replan-oversized",
    "off": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "on": {
     "exact": false,
     "keys": 4,
     "keysCorrect": 3,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "ml-cont-failing-suite",
    "off": {
     "exact": false,
     "keys": 5,
     "keysCorrect": 3,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "ml-new-retention-policy",
    "off": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "ml-new-empty-state",
    "off": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "ml-new-renewal-email",
    "off": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "ml-new-changelog-list",
    "off": {
     "exact": true,
     "keys": 6,
     "keysCorrect": 6,
     "error": false
    },
    "on": {
     "exact": false,
     "keys": 6,
     "keysCorrect": 5,
     "error": false
    },
    "sonnet": {
     "exact": false,
     "keys": 6,
     "keysCorrect": 5,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "ml-cont-latency",
    "off": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "ml-replan-rollback",
    "off": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "ml-new-backfill",
    "off": {
     "exact": false,
     "keys": 6,
     "keysCorrect": 4,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 6,
     "keysCorrect": 6,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 6,
     "keysCorrect": 6,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "asked-followup-csv",
    "off": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "on": {
     "exact": false,
     "keys": 4,
     "keysCorrect": 2,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "asked-clarify-limit",
    "off": {
     "exact": false,
     "keys": 5,
     "keysCorrect": 4,
     "error": false
    },
    "on": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "asked-followup-notes",
    "off": {
     "exact": false,
     "keys": 5,
     "keysCorrect": 3,
     "error": false
    },
    "on": {
     "exact": false,
     "keys": 5,
     "keysCorrect": 4,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 5,
     "keysCorrect": 5,
     "error": false
    }
   },
   {
    "purpose": "context.shape",
    "caseId": "asked-clarify-branch",
    "off": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    },
    "on": {
     "exact": false,
     "keys": 4,
     "keysCorrect": 3,
     "error": false
    },
    "sonnet": {
     "exact": true,
     "keys": 4,
     "keysCorrect": 4,
     "error": false
    }
   }
  ]
 },
 "probes": [
  {
   "route": "routing",
   "at": "2026-10-06T21:29:13.221Z",
   "wallMs": 4508,
   "outputTokens": 321,
   "usageThinkingTokens": 0,
   "modelUsageThinkingTokens": [
    0
   ]
  },
  {
   "route": "hard",
   "at": "2026-10-06T21:29:20.001Z",
   "totalMs": 2762,
   "outputTokens": 199,
   "reasoningTokens": 0,
   "passed": true
  }
 ],
 "hard": {
  "cases": [
   {
    "id": "merge-ranges",
    "title": "Fix an interval-merge function (off-by-one and edge cases)",
    "kind": "code-fix",
    "difficulty": "hard",
    "format": "js",
    "validator": "sandbox"
   },
   {
    "id": "day-hours",
    "title": "Fix a time-zone day-length function (DST)",
    "kind": "code-fix",
    "difficulty": "hard",
    "format": "js",
    "validator": "sandbox"
   },
   {
    "id": "csv-parse",
    "title": "Write a CSV parser (quoted newlines, strict errors)",
    "kind": "code-write",
    "difficulty": "hard",
    "format": "js",
    "validator": "sandbox"
   },
   {
    "id": "event-loop-order",
    "title": "Predict JavaScript event-loop output order",
    "kind": "reasoning",
    "difficulty": "hard",
    "format": "line",
    "validator": "exact",
    "expected": "A1,S,M1,A2,Q1,TH,M3,M2,Q2,B1,M4,R1,T1,T2,T2M,A3"
   },
   {
    "id": "talk-schedule",
    "title": "Solve a multi-constraint room schedule",
    "kind": "constraint",
    "difficulty": "hard",
    "format": "json",
    "validator": "sandbox"
   },
   {
    "id": "semver-regex",
    "title": "Write a strict SemVer 2.0.0 regex",
    "kind": "regex",
    "difficulty": "hard",
    "format": "regex",
    "validator": "sandbox"
   },
   {
    "id": "money-refactor",
    "title": "Refactor to remove duplication, keep 20 tests green",
    "kind": "refactor",
    "difficulty": "hard",
    "format": "js",
    "validator": "sandbox"
   },
   {
    "id": "q1-sql",
    "title": "Write a SQLite reporting query (fan-out, ties, boundaries)",
    "kind": "sql",
    "difficulty": "hard",
    "format": "sql",
    "validator": "sandbox"
   }
  ],
  "controls": {
   "ranAt": "2026-10-06T21:27:56.463Z",
   "allOk": true,
   "cases": [
    {
     "caseId": "merge-ranges",
     "referencePassed": true,
     "referenceChecks": 18,
     "wrongAnswers": 5,
     "wrongAnswersFailed": 5,
     "wrappedReferenceIsFormatMiss": true
    },
    {
     "caseId": "day-hours",
     "referencePassed": true,
     "referenceChecks": 36,
     "wrongAnswers": 3,
     "wrongAnswersFailed": 3,
     "wrappedReferenceIsFormatMiss": true
    },
    {
     "caseId": "csv-parse",
     "referencePassed": true,
     "referenceChecks": 22,
     "wrongAnswers": 3,
     "wrongAnswersFailed": 3,
     "wrappedReferenceIsFormatMiss": true
    },
    {
     "caseId": "event-loop-order",
     "referencePassed": true,
     "referenceChecks": 1,
     "wrongAnswers": 3,
     "wrongAnswersFailed": 3,
     "wrappedReferenceIsFormatMiss": true
    },
    {
     "caseId": "talk-schedule",
     "referencePassed": true,
     "referenceChecks": 14,
     "wrongAnswers": 3,
     "wrongAnswersFailed": 3,
     "wrappedReferenceIsFormatMiss": true
    },
    {
     "caseId": "semver-regex",
     "referencePassed": true,
     "referenceChecks": 30,
     "wrongAnswers": 3,
     "wrongAnswersFailed": 3,
     "wrappedReferenceIsFormatMiss": true
    },
    {
     "caseId": "money-refactor",
     "referencePassed": true,
     "referenceChecks": 25,
     "wrongAnswers": 2,
     "wrongAnswersFailed": 2,
     "wrappedReferenceIsFormatMiss": true
    },
    {
     "caseId": "q1-sql",
     "referencePassed": true,
     "referenceChecks": 8,
     "wrongAnswers": 4,
     "wrongAnswersFailed": 4,
     "wrappedReferenceIsFormatMiss": true
    }
   ]
  },
  "receipts": [
   {
    "batch": "hard-1",
    "caseId": "merge-ranges",
    "caseTitle": "Fix an interval-merge function (off-by-one and edge cases)",
    "caseKind": "code-fix",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 1,
    "startedAt": "2026-10-06T21:37:22.828Z",
    "finishedAt": "2026-10-06T21:37:26.018Z",
    "status": "completed",
    "firstUsefulMs": 841,
    "totalMs": 2693,
    "modelTurnMs": 1881,
    "inputTokens": 3876,
    "outputTokens": 197,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": true,
    "formatMiss": false,
    "answerCorrect": true,
    "checks": 18,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 457,
    "error": null,
    "failureCount": 0,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "day-hours",
    "caseTitle": "Fix a time-zone day-length function (DST)",
    "caseKind": "code-fix",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 1,
    "startedAt": "2026-10-06T21:37:26.019Z",
    "finishedAt": "2026-10-06T21:37:31.367Z",
    "status": "completed",
    "firstUsefulMs": 822,
    "totalMs": 4878,
    "modelTurnMs": 4027,
    "inputTokens": 3903,
    "outputTokens": 481,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": false,
    "formatMiss": false,
    "answerCorrect": false,
    "checks": 36,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 1305,
    "error": null,
    "failureCount": 30,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "csv-parse",
    "caseTitle": "Write a CSV parser (quoted newlines, strict errors)",
    "caseKind": "code-write",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 1,
    "startedAt": "2026-10-06T21:37:31.368Z",
    "finishedAt": "2026-10-06T21:37:37.017Z",
    "status": "completed",
    "firstUsefulMs": 893,
    "totalMs": 5165,
    "modelTurnMs": 4255,
    "inputTokens": 3899,
    "outputTokens": 512,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": false,
    "formatMiss": false,
    "answerCorrect": false,
    "checks": 22,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 1592,
    "error": null,
    "failureCount": 6,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "event-loop-order",
    "caseTitle": "Predict JavaScript event-loop output order",
    "caseKind": "reasoning",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 1,
    "startedAt": "2026-10-06T21:37:37.018Z",
    "finishedAt": "2026-10-06T21:37:38.946Z",
    "status": "completed",
    "firstUsefulMs": 914,
    "totalMs": 1745,
    "modelTurnMs": 843,
    "inputTokens": 3922,
    "outputTokens": 50,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": false,
    "formatMiss": false,
    "answerCorrect": false,
    "checks": 1,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 47,
    "error": null,
    "failureCount": 1,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "talk-schedule",
    "caseTitle": "Solve a multi-constraint room schedule",
    "caseKind": "constraint",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 1,
    "startedAt": "2026-10-06T21:37:38.946Z",
    "finishedAt": "2026-10-06T21:37:42.020Z",
    "status": "completed",
    "firstUsefulMs": 844,
    "totalMs": 1958,
    "modelTurnMs": 1144,
    "inputTokens": 4003,
    "outputTokens": 112,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": false,
    "formatMiss": false,
    "answerCorrect": false,
    "checks": 14,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 229,
    "error": null,
    "failureCount": 1,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "semver-regex",
    "caseTitle": "Write a strict SemVer 2.0.0 regex",
    "caseKind": "regex",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 1,
    "startedAt": "2026-10-06T21:37:42.021Z",
    "finishedAt": "2026-10-06T21:37:44.676Z",
    "status": "completed",
    "firstUsefulMs": 899,
    "totalMs": 2187,
    "modelTurnMs": 1345,
    "inputTokens": 3850,
    "outputTokens": 123,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": false,
    "formatMiss": false,
    "answerCorrect": false,
    "checks": 30,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 155,
    "error": null,
    "failureCount": 1,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "money-refactor",
    "caseTitle": "Refactor to remove duplication, keep 20 tests green",
    "caseKind": "refactor",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 1,
    "startedAt": "2026-10-06T21:37:44.676Z",
    "finishedAt": "2026-10-06T21:37:49.787Z",
    "status": "completed",
    "firstUsefulMs": 884,
    "totalMs": 4658,
    "modelTurnMs": 3738,
    "inputTokens": 4190,
    "outputTokens": 438,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 4187,
    "reasoningTokens": 0,
    "passed": true,
    "formatMiss": false,
    "answerCorrect": true,
    "checks": 25,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 1259,
    "error": null,
    "failureCount": 0,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "q1-sql",
    "caseTitle": "Write a SQLite reporting query (fan-out, ties, boundaries)",
    "caseKind": "sql",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 1,
    "startedAt": "2026-10-06T21:37:49.788Z",
    "finishedAt": "2026-10-06T21:37:54.302Z",
    "status": "completed",
    "firstUsefulMs": 843,
    "totalMs": 4043,
    "modelTurnMs": 3252,
    "inputTokens": 3994,
    "outputTokens": 387,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": false,
    "formatMiss": false,
    "answerCorrect": false,
    "checks": 8,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 896,
    "error": null,
    "failureCount": 2,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "merge-ranges",
    "caseTitle": "Fix an interval-merge function (off-by-one and edge cases)",
    "caseKind": "code-fix",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 2,
    "startedAt": "2026-10-06T21:37:54.303Z",
    "finishedAt": "2026-10-06T21:37:57.432Z",
    "status": "completed",
    "firstUsefulMs": 804,
    "totalMs": 2661,
    "modelTurnMs": 1906,
    "inputTokens": 3876,
    "outputTokens": 199,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": true,
    "formatMiss": false,
    "answerCorrect": true,
    "checks": 18,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 460,
    "error": null,
    "failureCount": 0,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "day-hours",
    "caseTitle": "Fix a time-zone day-length function (DST)",
    "caseKind": "code-fix",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 2,
    "startedAt": "2026-10-06T21:37:57.432Z",
    "finishedAt": "2026-10-06T21:38:02.258Z",
    "status": "completed",
    "firstUsefulMs": 843,
    "totalMs": 4366,
    "modelTurnMs": 3561,
    "inputTokens": 3903,
    "outputTokens": 450,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": false,
    "formatMiss": false,
    "answerCorrect": false,
    "checks": 36,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 1205,
    "error": null,
    "failureCount": 30,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "csv-parse",
    "caseTitle": "Write a CSV parser (quoted newlines, strict errors)",
    "caseKind": "code-write",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 2,
    "startedAt": "2026-10-06T21:38:02.258Z",
    "finishedAt": "2026-10-06T21:38:07.442Z",
    "status": "completed",
    "firstUsefulMs": 830,
    "totalMs": 4729,
    "modelTurnMs": 3929,
    "inputTokens": 3900,
    "outputTokens": 479,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": false,
    "formatMiss": false,
    "answerCorrect": false,
    "checks": 22,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 1444,
    "error": null,
    "failureCount": 1,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "event-loop-order",
    "caseTitle": "Predict JavaScript event-loop output order",
    "caseKind": "reasoning",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 2,
    "startedAt": "2026-10-06T21:38:07.442Z",
    "finishedAt": "2026-10-06T21:38:09.338Z",
    "status": "completed",
    "firstUsefulMs": 843,
    "totalMs": 1717,
    "modelTurnMs": 858,
    "inputTokens": 3922,
    "outputTokens": 50,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": false,
    "formatMiss": false,
    "answerCorrect": false,
    "checks": 1,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 47,
    "error": null,
    "failureCount": 1,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "talk-schedule",
    "caseTitle": "Solve a multi-constraint room schedule",
    "caseKind": "constraint",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 2,
    "startedAt": "2026-10-06T21:38:09.339Z",
    "finishedAt": "2026-10-06T21:38:12.382Z",
    "status": "completed",
    "firstUsefulMs": 853,
    "totalMs": 1976,
    "modelTurnMs": 1135,
    "inputTokens": 4003,
    "outputTokens": 112,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": false,
    "formatMiss": false,
    "answerCorrect": false,
    "checks": 14,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 229,
    "error": null,
    "failureCount": 1,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "semver-regex",
    "caseTitle": "Write a strict SemVer 2.0.0 regex",
    "caseKind": "regex",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 2,
    "startedAt": "2026-10-06T21:38:12.382Z",
    "finishedAt": "2026-10-06T21:38:14.995Z",
    "status": "completed",
    "firstUsefulMs": 869,
    "totalMs": 2122,
    "modelTurnMs": 1306,
    "inputTokens": 3850,
    "outputTokens": 124,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": false,
    "formatMiss": false,
    "answerCorrect": false,
    "checks": 30,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 157,
    "error": null,
    "failureCount": 1,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "money-refactor",
    "caseTitle": "Refactor to remove duplication, keep 20 tests green",
    "caseKind": "refactor",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 2,
    "startedAt": "2026-10-06T21:38:14.995Z",
    "finishedAt": "2026-10-06T21:38:18.657Z",
    "status": "completed",
    "firstUsefulMs": 984,
    "totalMs": 3203,
    "modelTurnMs": 2437,
    "inputTokens": 4193,
    "outputTokens": 383,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 4190,
    "reasoningTokens": 0,
    "passed": false,
    "formatMiss": false,
    "answerCorrect": false,
    "checks": 25,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 1097,
    "error": null,
    "failureCount": 1,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "q1-sql",
    "caseTitle": "Write a SQLite reporting query (fan-out, ties, boundaries)",
    "caseKind": "sql",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 2,
    "startedAt": "2026-10-06T21:38:18.658Z",
    "finishedAt": "2026-10-06T21:38:23.693Z",
    "status": "completed",
    "firstUsefulMs": 924,
    "totalMs": 4043,
    "modelTurnMs": 3233,
    "inputTokens": 3995,
    "outputTokens": 387,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": false,
    "formatMiss": false,
    "answerCorrect": false,
    "checks": 1,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 898,
    "error": null,
    "failureCount": 1,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "merge-ranges",
    "caseTitle": "Fix an interval-merge function (off-by-one and edge cases)",
    "caseKind": "code-fix",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 3,
    "startedAt": "2026-10-06T21:38:23.694Z",
    "finishedAt": "2026-10-06T21:38:26.835Z",
    "status": "completed",
    "firstUsefulMs": 886,
    "totalMs": 2533,
    "modelTurnMs": 1607,
    "inputTokens": 3878,
    "outputTokens": 167,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": true,
    "formatMiss": false,
    "answerCorrect": true,
    "checks": 18,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 381,
    "error": null,
    "failureCount": 0,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "day-hours",
    "caseTitle": "Fix a time-zone day-length function (DST)",
    "caseKind": "code-fix",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 3,
    "startedAt": "2026-10-06T21:38:26.835Z",
    "finishedAt": "2026-10-06T21:39:00.906Z",
    "status": "completed",
    "firstUsefulMs": 1070,
    "totalMs": 3629,
    "modelTurnMs": 2820,
    "inputTokens": 3902,
    "outputTokens": 382,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": false,
    "formatMiss": false,
    "answerCorrect": false,
    "checks": null,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 999,
    "error": null,
    "failureCount": 1,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "csv-parse",
    "caseTitle": "Write a CSV parser (quoted newlines, strict errors)",
    "caseKind": "code-write",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 3,
    "startedAt": "2026-10-06T21:39:00.906Z",
    "finishedAt": "2026-10-06T21:39:14.390Z",
    "status": "completed",
    "firstUsefulMs": 904,
    "totalMs": 13006,
    "modelTurnMs": 12232,
    "inputTokens": 3899,
    "outputTokens": 1628,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": false,
    "formatMiss": false,
    "answerCorrect": false,
    "checks": 0,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 5145,
    "error": null,
    "failureCount": 1,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "event-loop-order",
    "caseTitle": "Predict JavaScript event-loop output order",
    "caseKind": "reasoning",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 3,
    "startedAt": "2026-10-06T21:39:14.390Z",
    "finishedAt": "2026-10-06T21:39:16.262Z",
    "status": "completed",
    "firstUsefulMs": 828,
    "totalMs": 1697,
    "modelTurnMs": 881,
    "inputTokens": 3922,
    "outputTokens": 50,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": false,
    "formatMiss": false,
    "answerCorrect": false,
    "checks": 1,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 47,
    "error": null,
    "failureCount": 1,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "talk-schedule",
    "caseTitle": "Solve a multi-constraint room schedule",
    "caseKind": "constraint",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 3,
    "startedAt": "2026-10-06T21:39:16.263Z",
    "finishedAt": "2026-10-06T21:39:19.879Z",
    "status": "completed",
    "firstUsefulMs": 1628,
    "totalMs": 2598,
    "modelTurnMs": 1737,
    "inputTokens": 4003,
    "outputTokens": 112,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": false,
    "formatMiss": false,
    "answerCorrect": false,
    "checks": 14,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 229,
    "error": null,
    "failureCount": 1,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "semver-regex",
    "caseTitle": "Write a strict SemVer 2.0.0 regex",
    "caseKind": "regex",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 3,
    "startedAt": "2026-10-06T21:39:19.880Z",
    "finishedAt": "2026-10-06T21:39:22.500Z",
    "status": "completed",
    "firstUsefulMs": 856,
    "totalMs": 2139,
    "modelTurnMs": 1289,
    "inputTokens": 3848,
    "outputTokens": 124,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": false,
    "formatMiss": false,
    "answerCorrect": false,
    "checks": 30,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 157,
    "error": null,
    "failureCount": 1,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "money-refactor",
    "caseTitle": "Refactor to remove duplication, keep 20 tests green",
    "caseKind": "refactor",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 3,
    "startedAt": "2026-10-06T21:39:22.500Z",
    "finishedAt": "2026-10-06T21:39:26.832Z",
    "status": "completed",
    "firstUsefulMs": 834,
    "totalMs": 3870,
    "modelTurnMs": 3051,
    "inputTokens": 4190,
    "outputTokens": 395,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 4187,
    "reasoningTokens": 0,
    "passed": false,
    "formatMiss": false,
    "answerCorrect": false,
    "checks": 25,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 1138,
    "error": null,
    "failureCount": 1,
    "lenientExtractor": null
   },
   {
    "batch": "hard-1",
    "caseId": "q1-sql",
    "caseTitle": "Write a SQLite reporting query (fan-out, ties, boundaries)",
    "caseKind": "sql",
    "caseDifficulty": "hard",
    "route": "claude-cli",
    "modelRequested": "haiku",
    "modelReported": "claude-haiku-4-5-20251001",
    "effort": "default",
    "thinking": "off",
    "rep": 3,
    "startedAt": "2026-10-06T21:39:26.833Z",
    "finishedAt": "2026-10-06T21:39:31.929Z",
    "status": "completed",
    "firstUsefulMs": 929,
    "totalMs": 4064,
    "modelTurnMs": 3213,
    "inputTokens": 3995,
    "outputTokens": 395,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "reasoningTokens": 0,
    "passed": false,
    "formatMiss": false,
    "answerCorrect": false,
    "checks": 1,
    "cliVersion": "2.1.286 (Claude Code)",
    "speed": "standard",
    "outputChars": 923,
    "error": null,
    "failureCount": 1,
    "lenientExtractor": null
   }
  ]
 },
 "provenance": {
  "protocolCreatedAt": "2026-10-06T21:29:01.607477Z",
  "firstNewCountedCallAt": "2026-10-06T21:29:57.270Z",
  "protocolAmendedAfterRun": true,
  "frozenInitialProtocolAvailable": false,
  "preBatchUsageReadingsAvailable": false,
  "countedCallCap": 106,
  "totalCallCapIncludingProbes": 112,
  "hardTimeoutSeconds": 300,
  "hardOutputTokenCap": 16000,
  "note": "Controls and reused reference calls predate protocol creation. The protocol file existed before the new probes and counted calls. Later amendments changed it; no frozen initial copy verifies the original wording. Saved pre-batch usage readings are absent. Later corroboration does not prove the pre-batch gate. Timings came from a shared Mac with no recorded host-load control.",
  "hostLoadControlRecorded": false,
  "controlsAt": "2026-10-06T21:27:56.463Z"
 }
}