{
 "schema": "agent-public-bench/routing-holdout@1",
 "route": "routing-holdout",
 "startedAt": "2026-10-06T21:37:55.514Z",
 "finishedAt": "2026-10-06T21:54:45.129Z",
 "question": "Do Jev 1.13 and the Claude routers keep their accuracy on routing decisions that nobody tuned against any router answer?",
 "suite": {
  "version": "routing-holdout-2026-10-06.1",
  "frozenAt": "2026-10-06T21:37:02.582Z",
  "cases": 56,
  "counts": {
   "control.failure_class": 14,
   "frontdoor.intent": 14,
   "memory.is_rule": 14,
   "context.shape": 14
  },
  "labelledKeys": 125,
  "distribution": {
   "control.failure_class": {
    "failure": {
     "transient": 3,
     "impossible-here": 4,
     "flaky": 3,
     "real": 4
    }
   },
   "frontdoor.intent": {
    "intent": {
     "request": 3,
     "followup": 3,
     "status": 3,
     "decision": 2,
     "control": 3
    }
   },
   "memory.is_rule": {
    "kind": {
     "obligation": 4,
     "prohibition": 3,
     "preference": 3,
     "not-a-rule": 4
    }
   },
   "context.shape": {
    "artifacts": {
     "no": 9
    },
    "knowledge": {
     "search": 8,
     "none": 1,
     "facts": 5
    },
    "memories": {
     "yes": 9,
     "no": 1
    },
    "examples": {
     "yes": 4,
     "no": 9
    },
    "scope": {
     "small": 8,
     "medium": 5,
     "large": 1
    },
    "complexity": {
     "simple": 3,
     "moderate": 9,
     "complex": 2
    },
    "turn": {
     "follow-up": 2,
     "clarification": 1,
     "new-task": 1
    },
    "transcript": {
     "recent": 4,
     "summary": 1
    }
   }
  },
  "author": "one author (a Claude model), blind to every router answer until the labels were frozen",
  "paraphraseCheck": "nearest existing case by combined token and word-bigram Jaccard; 17 close cases rewritten before the freeze, plus one case settled by a rule; highest recorded token similarity after rewriting 0.25"
 },
 "labelAgreement": {
  "secondLabeller": "GPT-6.1 Sol · Codex CLI (medium), 4 calls, blind to the author labels",
  "totals": {
   "keys": 125,
   "agree": 115,
   "agreeRate": 0.92
  },
  "perKey": [
   {
    "purpose": "control.failure_class",
    "key": "failure",
    "n": 14,
    "agree": 14,
    "agreeRate": 1,
    "kappa": 1,
    "kappaStrict": 1
   },
   {
    "purpose": "frontdoor.intent",
    "key": "intent",
    "n": 14,
    "agree": 14,
    "agreeRate": 1,
    "kappa": 1,
    "kappaStrict": 1
   },
   {
    "purpose": "memory.is_rule",
    "key": "kind",
    "n": 14,
    "agree": 14,
    "agreeRate": 1,
    "kappa": 1,
    "kappaStrict": 1
   },
   {
    "purpose": "context.shape",
    "key": "artifacts",
    "n": 9,
    "agree": 1,
    "agreeRate": 0.111,
    "kappa": 0,
    "kappaStrict": 0
   },
   {
    "purpose": "context.shape",
    "key": "knowledge",
    "n": 14,
    "agree": 14,
    "agreeRate": 1,
    "kappa": 1,
    "kappaStrict": 0.567
   },
   {
    "purpose": "context.shape",
    "key": "memories",
    "n": 10,
    "agree": 10,
    "agreeRate": 1,
    "kappa": 1,
    "kappaStrict": 1
   },
   {
    "purpose": "context.shape",
    "key": "examples",
    "n": 13,
    "agree": 11,
    "agreeRate": 0.846,
    "kappa": 0.683,
    "kappaStrict": 0.683
   },
   {
    "purpose": "context.shape",
    "key": "scope",
    "n": 14,
    "agree": 14,
    "agreeRate": 1,
    "kappa": 1,
    "kappaStrict": 0.533
   },
   {
    "purpose": "context.shape",
    "key": "complexity",
    "n": 14,
    "agree": 14,
    "agreeRate": 1,
    "kappa": 1,
    "kappaStrict": 0.462
   },
   {
    "purpose": "context.shape",
    "key": "turn",
    "n": 4,
    "agree": 4,
    "agreeRate": 1,
    "kappa": 1,
    "kappaStrict": 0.556
   },
   {
    "purpose": "context.shape",
    "key": "transcript",
    "n": 5,
    "agree": 5,
    "agreeRate": 1,
    "kappa": 1,
    "kappaStrict": 0.25
   }
  ],
  "kappaNote": "Cohen's kappa per key (calculation). kappa: the author label is the second label when the author accepts it, else the author's first label. kappaStrict: the author's first label only."
 },
 "calls": {
  "jev": {
   "counted": 168,
   "probe": 1
  },
  "claude": {
   "claude-haiku": {
    "counted": 56,
    "probe": 1
   },
   "claude-sonnet": {
    "counted": 56,
    "probe": 1
   }
  },
  "codexLabeller": {
   "counted": 4,
   "probe": 1
  }
 },
 "routers": [
  {
   "id": "jev",
   "label": "Jev 1.13 (TypeSafe)",
   "route": "direct HTTPS to the vendor API with the production request body",
   "calls": 168,
   "okCalls": 168,
   "errorCalls": 0,
   "probeCalls": 1,
   "startedAt": "2026-10-06T21:37:55.514Z",
   "endedAt": "2026-10-06T21:38:19.944Z",
   "stopped": null,
   "trimmedCases": 0,
   "pooled": {
    "cases": 56,
    "exact": 46,
    "exactRate": 0.8214,
    "exactCi": [
     0.7016,
     0.9
    ],
    "keys": 125,
    "keysCorrect": 113,
    "keyAccuracy": 0.904,
    "keyCi": [
     0.8397,
     0.9442
    ]
   },
   "secondary": {
    "cases": 56,
    "exact": 46,
    "exactRate": 0.8214,
    "exactCi": [
     0.7016,
     0.9
    ],
    "keys": 115,
    "keysCorrect": 103,
    "keyAccuracy": 0.8957,
    "keyCi": [
     0.8264,
     0.9393
    ]
   },
   "purposes": {
    "control.failure_class": {
     "cases": 14,
     "exact": 13,
     "exactCi": [
      0.6853,
      0.9873
     ],
     "keys": 14,
     "keysCorrect": 13,
     "keyCi": [
      0.6853,
      0.9873
     ],
     "errors": 0,
     "questions": [
      {
       "key": "failure",
       "scored": 14,
       "correct": 13
      }
     ],
     "confusion": {
      "failure": {
       "transient": {
        "transient": 3
       },
       "impossible-here": {
        "impossible-here": 4
       },
       "flaky": {
        "flaky": 2,
        "real": 1
       },
       "real": {
        "real": 4
       }
      }
     }
    },
    "frontdoor.intent": {
     "cases": 14,
     "exact": 12,
     "exactCi": [
      0.6006,
      0.9599
     ],
     "keys": 14,
     "keysCorrect": 12,
     "keyCi": [
      0.6006,
      0.9599
     ],
     "errors": 0,
     "questions": [
      {
       "key": "intent",
       "scored": 14,
       "correct": 12
      }
     ],
     "confusion": {
      "intent": {
       "request": {
        "request": 3
       },
       "followup": {
        "followup": 3
       },
       "status": {
        "status": 3
       },
       "decision": {
        "decision": 1,
        "none-of-these": 1
       },
       "control": {
        "control": 2,
        "request": 1
       }
      }
     }
    },
    "memory.is_rule": {
     "cases": 14,
     "exact": 13,
     "exactCi": [
      0.6853,
      0.9873
     ],
     "keys": 14,
     "keysCorrect": 13,
     "keyCi": [
      0.6853,
      0.9873
     ],
     "errors": 0,
     "questions": [
      {
       "key": "kind",
       "scored": 14,
       "correct": 13
      }
     ],
     "confusion": {
      "kind": {
       "obligation": {
        "obligation": 3,
        "not-a-rule": 1
       },
       "prohibition": {
        "prohibition": 3
       },
       "preference": {
        "preference": 3
       },
       "not-a-rule": {
        "not-a-rule": 4
       }
      }
     }
    },
    "context.shape": {
     "cases": 14,
     "exact": 8,
     "exactCi": [
      0.3259,
      0.7862
     ],
     "keys": 83,
     "keysCorrect": 75,
     "keyCi": [
      0.8212,
      0.9503
     ],
     "errors": 0,
     "questions": [
      {
       "key": "artifacts",
       "scored": 9,
       "correct": 9
      },
      {
       "key": "knowledge",
       "scored": 14,
       "correct": 12
      },
      {
       "key": "memories",
       "scored": 10,
       "correct": 10
      },
      {
       "key": "examples",
       "scored": 13,
       "correct": 13
      },
      {
       "key": "scope",
       "scored": 14,
       "correct": 11
      },
      {
       "key": "complexity",
       "scored": 14,
       "correct": 12
      },
      {
       "key": "turn",
       "scored": 4,
       "correct": 4
      },
      {
       "key": "transcript",
       "scored": 5,
       "correct": 4
      }
     ],
     "confusion": {
      "artifacts": {
       "no": {
        "no": 9
       }
      },
      "knowledge": {
       "search": {
        "search": 7,
        "facts": 1
       },
       "none": {
        "none": 1
       },
       "facts": {
        "none": 1,
        "facts": 3,
        "search": 1
       }
      },
      "memories": {
       "yes": {
        "yes": 9
       },
       "no": {
        "no": 1
       }
      },
      "examples": {
       "yes": {
        "yes": 4
       },
       "no": {
        "no": 9
       }
      },
      "scope": {
       "small": {
        "medium": 8
       },
       "medium": {
        "medium": 4,
        "large": 1
       },
       "large": {
        "large": 1
       }
      },
      "complexity": {
       "simple": {
        "moderate": 2,
        "simple": 1
       },
       "moderate": {
        "complex": 1,
        "moderate": 6,
        "simple": 2
       },
       "complex": {
        "complex": 2
       }
      },
      "turn": {
       "follow-up": {
        "follow-up": 2
       },
       "clarification": {
        "clarification": 1
       },
       "new-task": {
        "follow-up": 1
       }
      },
      "transcript": {
       "recent": {
        "full": 1,
        "recent": 2,
        "summary": 1
       },
       "summary": {
        "summary": 1
       }
      }
     }
    }
   },
   "reps": {
    "1": {
     "cases": 56,
     "exact": 46,
     "exactRate": 0.8214,
     "exactCi": [
      0.7016,
      0.9
     ],
     "keys": 125,
     "keysCorrect": 113,
     "keyAccuracy": 0.904,
     "keyCi": [
      0.8397,
      0.9442
     ]
    },
    "2": {
     "cases": 56,
     "exact": 46,
     "exactRate": 0.8214,
     "exactCi": [
      0.7016,
      0.9
     ],
     "keys": 125,
     "keysCorrect": 112,
     "keyAccuracy": 0.896,
     "keyCi": [
      0.8302,
      0.9382
     ]
    },
    "3": {
     "cases": 56,
     "exact": 47,
     "exactRate": 0.8393,
     "exactCi": [
      0.7219,
      0.9131
     ],
     "keys": 125,
     "keysCorrect": 113,
     "keyAccuracy": 0.904,
     "keyCi": [
      0.8397,
      0.9442
     ]
    }
   },
   "primaryRep": 1,
   "stability": {
    "cases": 56,
    "stableCases": 53,
    "stableRate": 0.9464,
    "stableCi": [
     0.8539,
     0.9816
    ],
    "keys": 130,
    "stableKeys": 127,
    "perPurpose": {
     "control.failure_class": {
      "stable": 13,
      "total": 14
     },
     "frontdoor.intent": {
      "stable": 14,
      "total": 14
     },
     "memory.is_rule": {
      "stable": 14,
      "total": 14
     },
     "context.shape": {
      "stable": 12,
      "total": 14
     }
    }
   },
   "latencyMs": {
    "wallP50": 139,
    "wallP95": 192,
    "ttfbP50": 139,
    "min": 101,
    "max": 290,
    "n": 168,
    "note": "all counted calls, 3 reps"
   },
   "tokensPerDecision": {
    "input": 729.8,
    "output": 119.3
   },
   "listCostPer1000Usd": 0.03065,
   "costNote": "calculation: reported input tokens x $0.042 per million (output free), per 1,000 decisions"
  },
  {
   "id": "claude-haiku",
   "label": "Claude Haiku 4.5 · Claude Code",
   "route": "Claude Code CLI, one call per decision, CLI default effort (extended thinking)",
   "effort": "CLI default (extended thinking)",
   "calls": 56,
   "okCalls": 56,
   "errorCalls": 0,
   "probeCalls": 1,
   "startedAt": "2026-10-06T21:40:42.593Z",
   "endedAt": "2026-10-06T21:52:12.790Z",
   "stopped": null,
   "trimmedCases": 0,
   "pooled": {
    "cases": 56,
    "exact": 44,
    "exactRate": 0.7857,
    "exactCi": [
     0.6618,
     0.8729
    ],
    "keys": 125,
    "keysCorrect": 102,
    "keyAccuracy": 0.816,
    "keyCi": [
     0.739,
     0.8741
    ]
   },
   "secondary": {
    "cases": 56,
    "exact": 46,
    "exactRate": 0.8214,
    "exactCi": [
     0.7016,
     0.9
    ],
    "keys": 115,
    "keysCorrect": 98,
    "keyAccuracy": 0.8522,
    "keyCi": [
     0.776,
     0.9056
    ]
   },
   "purposes": {
    "control.failure_class": {
     "cases": 14,
     "exact": 13,
     "exactCi": [
      0.6853,
      0.9873
     ],
     "keys": 14,
     "keysCorrect": 13,
     "keyCi": [
      0.6853,
      0.9873
     ],
     "errors": 0,
     "questions": [
      {
       "key": "failure",
       "scored": 14,
       "correct": 13
      }
     ],
     "confusion": {
      "failure": {
       "transient": {
        "transient": 3
       },
       "impossible-here": {
        "impossible-here": 4
       },
       "flaky": {
        "flaky": 2,
        "real": 1
       },
       "real": {
        "real": 4
       }
      }
     }
    },
    "frontdoor.intent": {
     "cases": 14,
     "exact": 13,
     "exactCi": [
      0.6853,
      0.9873
     ],
     "keys": 14,
     "keysCorrect": 13,
     "keyCi": [
      0.6853,
      0.9873
     ],
     "errors": 0,
     "questions": [
      {
       "key": "intent",
       "scored": 14,
       "correct": 13
      }
     ],
     "confusion": {
      "intent": {
       "request": {
        "request": 3
       },
       "followup": {
        "followup": 3
       },
       "status": {
        "status": 3
       },
       "decision": {
        "decision": 2
       },
       "control": {
        "control": 2,
        "request": 1
       }
      }
     }
    },
    "memory.is_rule": {
     "cases": 14,
     "exact": 13,
     "exactCi": [
      0.6853,
      0.9873
     ],
     "keys": 14,
     "keysCorrect": 13,
     "keyCi": [
      0.6853,
      0.9873
     ],
     "errors": 0,
     "questions": [
      {
       "key": "kind",
       "scored": 14,
       "correct": 13
      }
     ],
     "confusion": {
      "kind": {
       "obligation": {
        "obligation": 3,
        "not-a-rule": 1
       },
       "prohibition": {
        "prohibition": 3
       },
       "preference": {
        "preference": 3
       },
       "not-a-rule": {
        "not-a-rule": 4
       }
      }
     }
    },
    "context.shape": {
     "cases": 14,
     "exact": 5,
     "exactCi": [
      0.1634,
      0.6124
     ],
     "keys": 83,
     "keysCorrect": 63,
     "keyCi": [
      0.6569,
      0.8383
     ],
     "errors": 0,
     "questions": [
      {
       "key": "artifacts",
       "scored": 9,
       "correct": 4
      },
      {
       "key": "knowledge",
       "scored": 14,
       "correct": 12
      },
      {
       "key": "memories",
       "scored": 10,
       "correct": 9
      },
      {
       "key": "examples",
       "scored": 13,
       "correct": 9
      },
      {
       "key": "scope",
       "scored": 14,
       "correct": 10
      },
      {
       "key": "complexity",
       "scored": 14,
       "correct": 11
      },
      {
       "key": "turn",
       "scored": 4,
       "correct": 4
      },
      {
       "key": "transcript",
       "scored": 5,
       "correct": 4
      }
     ],
     "confusion": {
      "artifacts": {
       "no": {
        "yes": 5,
        "no": 4
       }
      },
      "knowledge": {
       "search": {
        "search": 8
       },
       "none": {
        "none": 1
       },
       "facts": {
        "facts": 1,
        "none": 1,
        "search": 3
       }
      },
      "memories": {
       "yes": {
        "yes": 8,
        "no": 1
       },
       "no": {
        "no": 1
       }
      },
      "examples": {
       "yes": {
        "yes": 4
       },
       "no": {
        "yes": 4,
        "no": 5
       }
      },
      "scope": {
       "small": {
        "medium": 8
       },
       "medium": {
        "medium": 3,
        "large": 1,
        "small": 1
       },
       "large": {
        "large": 1
       }
      },
      "complexity": {
       "simple": {
        "moderate": 3
       },
       "moderate": {
        "complex": 1,
        "moderate": 5,
        "simple": 3
       },
       "complex": {
        "complex": 2
       }
      },
      "turn": {
       "follow-up": {
        "follow-up": 2
       },
       "clarification": {
        "clarification": 1
       },
       "new-task": {
        "follow-up": 1
       }
      },
      "transcript": {
       "recent": {
        "recent": 3,
        "summary": 1
       },
       "summary": {
        "summary": 1
       }
      }
     }
    }
   },
   "latencyMs": {
    "wallP50": 9444,
    "wallP95": 25413,
    "apiP50": 7522,
    "apiP95": 23913,
    "cliOverheadP50": 1718,
    "n": 56
   },
   "tokensPerDecision": {
    "input": 1774.9,
    "cacheRead": 0,
    "cacheWrite": 0,
    "cacheWrite1h": 0,
    "output": 1070.7
   },
   "listCostPer1000Usd": 7.129,
   "cliReportedCostPer1000Usd": 7.129,
   "listCostPer1000UsdTunedConvention": 7.129,
   "costNote": "calculation: CLI-reported tokens x list price, cache writes priced by their reported duration (1 hour at 2x input, 5 minutes at 1.25x), per 1,000 decisions"
  },
  {
   "id": "claude-sonnet",
   "label": "Claude Sonnet 5.5 (low) · Claude Code",
   "route": "Claude Code CLI, one call per decision, effort low",
   "effort": "low",
   "calls": 56,
   "okCalls": 56,
   "errorCalls": 0,
   "probeCalls": 1,
   "startedAt": "2026-10-06T21:52:23.355Z",
   "endedAt": "2026-10-06T21:54:45.129Z",
   "stopped": null,
   "trimmedCases": 0,
   "pooled": {
    "cases": 56,
    "exact": 49,
    "exactRate": 0.875,
    "exactCi": [
     0.7637,
     0.9381
    ],
    "keys": 125,
    "keysCorrect": 115,
    "keyAccuracy": 0.92,
    "keyCi": [
     0.859,
     0.956
    ]
   },
   "secondary": {
    "cases": 56,
    "exact": 52,
    "exactRate": 0.9286,
    "exactCi": [
     0.8302,
     0.9719
    ],
    "keys": 115,
    "keysCorrect": 109,
    "keyAccuracy": 0.9478,
    "keyCi": [
     0.8908,
     0.9759
    ]
   },
   "purposes": {
    "control.failure_class": {
     "cases": 14,
     "exact": 14,
     "exactCi": [
      0.7847,
      1
     ],
     "keys": 14,
     "keysCorrect": 14,
     "keyCi": [
      0.7847,
      1
     ],
     "errors": 0,
     "questions": [
      {
       "key": "failure",
       "scored": 14,
       "correct": 14
      }
     ],
     "confusion": {
      "failure": {
       "transient": {
        "transient": 3
       },
       "impossible-here": {
        "impossible-here": 4
       },
       "flaky": {
        "flaky": 3
       },
       "real": {
        "real": 4
       }
      }
     }
    },
    "frontdoor.intent": {
     "cases": 14,
     "exact": 14,
     "exactCi": [
      0.7847,
      1
     ],
     "keys": 14,
     "keysCorrect": 14,
     "keyCi": [
      0.7847,
      1
     ],
     "errors": 0,
     "questions": [
      {
       "key": "intent",
       "scored": 14,
       "correct": 14
      }
     ],
     "confusion": {
      "intent": {
       "request": {
        "request": 3
       },
       "followup": {
        "followup": 3
       },
       "status": {
        "status": 3
       },
       "decision": {
        "decision": 2
       },
       "control": {
        "control": 3
       }
      }
     }
    },
    "memory.is_rule": {
     "cases": 14,
     "exact": 13,
     "exactCi": [
      0.6853,
      0.9873
     ],
     "keys": 14,
     "keysCorrect": 13,
     "keyCi": [
      0.6853,
      0.9873
     ],
     "errors": 0,
     "questions": [
      {
       "key": "kind",
       "scored": 14,
       "correct": 13
      }
     ],
     "confusion": {
      "kind": {
       "obligation": {
        "obligation": 3,
        "not-a-rule": 1
       },
       "prohibition": {
        "prohibition": 3
       },
       "preference": {
        "preference": 3
       },
       "not-a-rule": {
        "not-a-rule": 4
       }
      }
     }
    },
    "context.shape": {
     "cases": 14,
     "exact": 8,
     "exactCi": [
      0.3259,
      0.7862
     ],
     "keys": 83,
     "keysCorrect": 74,
     "keyCi": [
      0.8066,
      0.9419
     ],
     "errors": 0,
     "questions": [
      {
       "key": "artifacts",
       "scored": 9,
       "correct": 7
      },
      {
       "key": "knowledge",
       "scored": 14,
       "correct": 13
      },
      {
       "key": "memories",
       "scored": 10,
       "correct": 10
      },
      {
       "key": "examples",
       "scored": 13,
       "correct": 10
      },
      {
       "key": "scope",
       "scored": 14,
       "correct": 12
      },
      {
       "key": "complexity",
       "scored": 14,
       "correct": 13
      },
      {
       "key": "turn",
       "scored": 4,
       "correct": 4
      },
      {
       "key": "transcript",
       "scored": 5,
       "correct": 5
      }
     ],
     "confusion": {
      "artifacts": {
       "no": {
        "yes": 2,
        "no": 7
       }
      },
      "knowledge": {
       "search": {
        "search": 8
       },
       "none": {
        "none": 1
       },
       "facts": {
        "search": 4,
        "facts": 1
       }
      },
      "memories": {
       "yes": {
        "yes": 9
       },
       "no": {
        "no": 1
       }
      },
      "examples": {
       "yes": {
        "yes": 4
       },
       "no": {
        "no": 6,
        "yes": 3
       }
      },
      "scope": {
       "small": {
        "medium": 7,
        "small": 1
       },
       "medium": {
        "medium": 4,
        "small": 1
       },
       "large": {
        "large": 1
       }
      },
      "complexity": {
       "simple": {
        "moderate": 2,
        "simple": 1
       },
       "moderate": {
        "complex": 1,
        "moderate": 7,
        "simple": 1
       },
       "complex": {
        "complex": 2
       }
      },
      "turn": {
       "follow-up": {
        "follow-up": 2
       },
       "clarification": {
        "clarification": 1
       },
       "new-task": {
        "follow-up": 1
       }
      },
      "transcript": {
       "recent": {
        "recent": 4
       },
       "summary": {
        "summary": 1
       }
      }
     }
    }
   },
   "latencyMs": {
    "wallP50": 2359,
    "wallP95": 3657,
    "apiP50": 1485,
    "apiP95": 2377,
    "cliOverheadP50": 861,
    "n": 56
   },
   "tokensPerDecision": {
    "input": 2,
    "cacheRead": 125.3,
    "cacheWrite": 1577.4,
    "cacheWrite1h": 1577.4,
    "output": 90.5
   },
   "listCostPer1000Usd": 7.244,
   "cliReportedCostPer1000Usd": 7.244,
   "listCostPer1000UsdTunedConvention": 4.877,
   "costNote": "calculation: CLI-reported tokens x list price, cache writes priced by their reported duration (1 hour at 2x input, 5 minutes at 1.25x), per 1,000 decisions"
  }
 ],
 "pairs": [
  {
   "a": "jev",
   "b": "claude-haiku",
   "cases": 56,
   "bothRight": 42,
   "onlyA": 4,
   "onlyB": 2,
   "bothWrong": 8,
   "mcnemarExactP": 0.6875,
   "discordant": [
    "holdout-25",
    "holdout-43",
    "holdout-44",
    "holdout-49",
    "holdout-53",
    "holdout-55"
   ]
  },
  {
   "a": "jev",
   "b": "claude-sonnet",
   "cases": 56,
   "bothRight": 42,
   "onlyA": 4,
   "onlyB": 7,
   "bothWrong": 3,
   "mcnemarExactP": 0.5488,
   "discordant": [
    "holdout-09",
    "holdout-25",
    "holdout-28",
    "holdout-43",
    "holdout-46",
    "holdout-47",
    "holdout-48",
    "holdout-49",
    "holdout-50",
    "holdout-51",
    "holdout-55"
   ]
  },
  {
   "a": "claude-haiku",
   "b": "claude-sonnet",
   "cases": 56,
   "bothRight": 42,
   "onlyA": 2,
   "onlyB": 7,
   "bothWrong": 5,
   "mcnemarExactP": 0.1797,
   "discordant": [
    "holdout-09",
    "holdout-28",
    "holdout-44",
    "holdout-46",
    "holdout-47",
    "holdout-48",
    "holdout-50",
    "holdout-51",
    "holdout-53"
   ]
  }
 ],
 "cases": [
  {
   "id": "holdout-01",
   "purpose": "control.failure_class",
   "tags": [
    "transient",
    "git"
   ],
   "labels": {
    "failure": [
     "transient"
    ]
   },
   "secondLabel": {
    "failure": "transient"
   },
   "agreedKeys": [
    "failure"
   ],
   "nearestExisting": {
    "id": "git-timeout",
    "tokenJaccard": 0.094
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "failure": true
    },
    "claude-haiku": {
     "failure": true
    },
    "claude-sonnet": {
     "failure": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-02",
   "purpose": "control.failure_class",
   "tags": [
    "transient",
    "install"
   ],
   "labels": {
    "failure": [
     "transient"
    ]
   },
   "secondLabel": {
    "failure": "transient"
   },
   "agreedKeys": [
    "failure"
   ],
   "nearestExisting": {
    "id": "npm-429",
    "tokenJaccard": 0.091
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "failure": true
    },
    "claude-haiku": {
     "failure": true
    },
    "claude-sonnet": {
     "failure": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-03",
   "purpose": "control.failure_class",
   "tags": [
    "transient",
    "install",
    "5xx"
   ],
   "labels": {
    "failure": [
     "transient"
    ]
   },
   "secondLabel": {
    "failure": "transient"
   },
   "agreedKeys": [
    "failure"
   ],
   "nearestExisting": {
    "id": "npm-429",
    "tokenJaccard": 0.243
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "failure": true
    },
    "claude-haiku": {
     "failure": true
    },
    "claude-sonnet": {
     "failure": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-04",
   "purpose": "control.failure_class",
   "tags": [
    "impossible-here",
    "system-library"
   ],
   "labels": {
    "failure": [
     "impossible-here"
    ]
   },
   "secondLabel": {
    "failure": "impossible-here"
   },
   "agreedKeys": [
    "failure"
   ],
   "nearestExisting": {
    "id": "missing-linker",
    "tokenJaccard": 0.176
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "failure": true
    },
    "claude-haiku": {
     "failure": true
    },
    "claude-sonnet": {
     "failure": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-05",
   "purpose": "control.failure_class",
   "tags": [
    "impossible-here",
    "runtime-version"
   ],
   "labels": {
    "failure": [
     "impossible-here"
    ]
   },
   "secondLabel": {
    "failure": "impossible-here"
   },
   "agreedKeys": [
    "failure"
   ],
   "nearestExisting": {
    "id": "postgres-refused",
    "tokenJaccard": 0.085
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "failure": true
    },
    "claude-haiku": {
     "failure": true
    },
    "claude-sonnet": {
     "failure": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-06",
   "purpose": "control.failure_class",
   "tags": [
    "impossible-here",
    "hardware"
   ],
   "labels": {
    "failure": [
     "impossible-here"
    ]
   },
   "secondLabel": {
    "failure": "impossible-here"
   },
   "agreedKeys": [
    "failure"
   ],
   "nearestExisting": {
    "id": "python-assertion",
    "tokenJaccard": 0.085
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "failure": true
    },
    "claude-haiku": {
     "failure": true
    },
    "claude-sonnet": {
     "failure": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-07",
   "purpose": "control.failure_class",
   "tags": [
    "impossible-here",
    "toolchain"
   ],
   "labels": {
    "failure": [
     "impossible-here"
    ]
   },
   "secondLabel": {
    "failure": "impossible-here"
   },
   "agreedKeys": [
    "failure"
   ],
   "nearestExisting": {
    "id": "test-timeout",
    "tokenJaccard": 0.065
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "failure": true
    },
    "claude-haiku": {
     "failure": true
    },
    "claude-sonnet": {
     "failure": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-08",
   "purpose": "control.failure_class",
   "tags": [
    "flaky",
    "clock"
   ],
   "labels": {
    "failure": [
     "flaky"
    ]
   },
   "secondLabel": {
    "failure": "flaky"
   },
   "agreedKeys": [
    "failure"
   ],
   "nearestExisting": {
    "id": "python-assertion",
    "tokenJaccard": 0.171
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "failure": true
    },
    "claude-haiku": {
     "failure": true
    },
    "claude-sonnet": {
     "failure": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-09",
   "purpose": "control.failure_class",
   "tags": [
    "flaky",
    "order"
   ],
   "labels": {
    "failure": [
     "flaky"
    ]
   },
   "secondLabel": {
    "failure": "flaky"
   },
   "agreedKeys": [
    "failure"
   ],
   "nearestExisting": {
    "id": "python-assertion",
    "tokenJaccard": 0.175
   },
   "exact": {
    "jev": false,
    "claude-haiku": false,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "failure": false
    },
    "claude-haiku": {
     "failure": false
    },
    "claude-sonnet": {
     "failure": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": false,
    "2": false,
    "3": true
   }
  },
  {
   "id": "holdout-10",
   "purpose": "control.failure_class",
   "tags": [
    "flaky",
    "timing"
   ],
   "labels": {
    "failure": [
     "flaky"
    ]
   },
   "secondLabel": {
    "failure": "flaky"
   },
   "agreedKeys": [
    "failure"
   ],
   "nearestExisting": {
    "id": "git-timeout",
    "tokenJaccard": 0.088
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "failure": true
    },
    "claude-haiku": {
     "failure": true
    },
    "claude-sonnet": {
     "failure": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-11",
   "purpose": "control.failure_class",
   "tags": [
    "real",
    "compile"
   ],
   "labels": {
    "failure": [
     "real"
    ]
   },
   "secondLabel": {
    "failure": "real"
   },
   "agreedKeys": [
    "failure"
   ],
   "nearestExisting": {
    "id": "git-timeout",
    "tokenJaccard": 0.118
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "failure": true
    },
    "claude-haiku": {
     "failure": true
    },
    "claude-sonnet": {
     "failure": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-12",
   "purpose": "control.failure_class",
   "tags": [
    "real",
    "missing-import"
   ],
   "labels": {
    "failure": [
     "real"
    ]
   },
   "secondLabel": {
    "failure": "real"
   },
   "agreedKeys": [
    "failure"
   ],
   "nearestExisting": {
    "id": "python-assertion",
    "tokenJaccard": 0.097
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "failure": true
    },
    "claude-haiku": {
     "failure": true
    },
    "claude-sonnet": {
     "failure": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-13",
   "purpose": "control.failure_class",
   "tags": [
    "real",
    "dependency-conflict"
   ],
   "labels": {
    "failure": [
     "real"
    ]
   },
   "secondLabel": {
    "failure": "real"
   },
   "agreedKeys": [
    "failure"
   ],
   "nearestExisting": {
    "id": "pip-503",
    "tokenJaccard": 0.092
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "failure": true
    },
    "claude-haiku": {
     "failure": true
    },
    "claude-sonnet": {
     "failure": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-14",
   "purpose": "control.failure_class",
   "tags": [
    "real",
    "assertion"
   ],
   "labels": {
    "failure": [
     "real"
    ]
   },
   "secondLabel": {
    "failure": "real"
   },
   "agreedKeys": [
    "failure"
   ],
   "nearestExisting": {
    "id": "postgres-refused",
    "tokenJaccard": 0.096
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "failure": true
    },
    "claude-haiku": {
     "failure": true
    },
    "claude-sonnet": {
     "failure": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-15",
   "purpose": "frontdoor.intent",
   "tags": [
    "request"
   ],
   "labels": {
    "intent": [
     "request"
    ]
   },
   "secondLabel": {
    "intent": "request"
   },
   "agreedKeys": [
    "intent"
   ],
   "nearestExisting": {
    "id": "new-dashboard",
    "tokenJaccard": 0.15
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "intent": true
    },
    "claude-haiku": {
     "intent": true
    },
    "claude-sonnet": {
     "intent": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-16",
   "purpose": "frontdoor.intent",
   "tags": [
    "request"
   ],
   "labels": {
    "intent": [
     "request"
    ]
   },
   "secondLabel": {
    "intent": "request"
   },
   "agreedKeys": [
    "intent"
   ],
   "nearestExisting": {
    "id": "reply-status-question",
    "tokenJaccard": 0.095
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "intent": true
    },
    "claude-haiku": {
     "intent": true
    },
    "claude-sonnet": {
     "intent": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-17",
   "purpose": "frontdoor.intent",
   "tags": [
    "request",
    "standing-instruction"
   ],
   "labels": {
    "intent": [
     "request"
    ]
   },
   "secondLabel": {
    "intent": "request"
   },
   "agreedKeys": [
    "intent"
   ],
   "nearestExisting": {
    "id": "key-status-question",
    "tokenJaccard": 0.133
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "intent": true
    },
    "claude-haiku": {
     "intent": true
    },
    "claude-sonnet": {
     "intent": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-18",
   "purpose": "frontdoor.intent",
   "tags": [
    "followup"
   ],
   "labels": {
    "intent": [
     "followup"
    ]
   },
   "secondLabel": {
    "intent": "followup"
   },
   "agreedKeys": [
    "intent"
   ],
   "nearestExisting": {
    "id": "make-it-weekly",
    "tokenJaccard": 0.15
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "intent": true
    },
    "claude-haiku": {
     "intent": true
    },
    "claude-sonnet": {
     "intent": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-19",
   "purpose": "frontdoor.intent",
   "tags": [
    "followup"
   ],
   "labels": {
    "intent": [
     "followup"
    ]
   },
   "secondLabel": {
    "intent": "followup"
   },
   "agreedKeys": [
    "intent"
   ],
   "nearestExisting": {
    "id": "set-up-ci",
    "tokenJaccard": 0.105
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "intent": true
    },
    "claude-haiku": {
     "intent": true
    },
    "claude-sonnet": {
     "intent": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-20",
   "purpose": "frontdoor.intent",
   "tags": [
    "followup"
   ],
   "labels": {
    "intent": [
     "followup"
    ]
   },
   "secondLabel": {
    "intent": "followup"
   },
   "agreedKeys": [
    "intent"
   ],
   "nearestExisting": {
    "id": "migration-progress",
    "tokenJaccard": 0.1
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "intent": true
    },
    "claude-haiku": {
     "intent": true
    },
    "claude-sonnet": {
     "intent": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-21",
   "purpose": "frontdoor.intent",
   "tags": [
    "status"
   ],
   "labels": {
    "intent": [
     "status"
    ]
   },
   "secondLabel": {
    "intent": "status"
   },
   "agreedKeys": [
    "intent"
   ],
   "nearestExisting": {
    "id": "fix-login",
    "tokenJaccard": 0.154
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "intent": true
    },
    "claude-haiku": {
     "intent": true
    },
    "claude-sonnet": {
     "intent": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-22",
   "purpose": "frontdoor.intent",
   "tags": [
    "status"
   ],
   "labels": {
    "intent": [
     "status"
    ]
   },
   "secondLabel": {
    "intent": "status"
   },
   "agreedKeys": [
    "intent"
   ],
   "nearestExisting": {
    "id": "migration-progress",
    "tokenJaccard": 0.154
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "intent": true
    },
    "claude-haiku": {
     "intent": true
    },
    "claude-sonnet": {
     "intent": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-23",
   "purpose": "frontdoor.intent",
   "tags": [
    "status"
   ],
   "labels": {
    "intent": [
     "status"
    ]
   },
   "secondLabel": {
    "intent": "status"
   },
   "agreedKeys": [
    "intent"
   ],
   "nearestExisting": {
    "id": "cancel-redesign",
    "tokenJaccard": 0.2
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "intent": true
    },
    "claude-haiku": {
     "intent": true
    },
    "claude-sonnet": {
     "intent": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-24",
   "purpose": "frontdoor.intent",
   "tags": [
    "decision",
    "approval"
   ],
   "labels": {
    "intent": [
     "decision"
    ]
   },
   "secondLabel": {
    "intent": "decision"
   },
   "agreedKeys": [
    "intent"
   ],
   "nearestExisting": {
    "id": "reply-to-question",
    "tokenJaccard": 0.25
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "intent": true
    },
    "claude-haiku": {
     "intent": true
    },
    "claude-sonnet": {
     "intent": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-25",
   "purpose": "frontdoor.intent",
   "tags": [
    "decision",
    "denial"
   ],
   "labels": {
    "intent": [
     "decision"
    ]
   },
   "secondLabel": {
    "intent": "decision"
   },
   "agreedKeys": [
    "intent"
   ],
   "nearestExisting": {
    "id": "where-are-we",
    "tokenJaccard": 0.154
   },
   "exact": {
    "jev": false,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "intent": false
    },
    "claude-haiku": {
     "intent": true
    },
    "claude-sonnet": {
     "intent": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": false,
    "2": false,
    "3": false
   }
  },
  {
   "id": "holdout-26",
   "purpose": "frontdoor.intent",
   "tags": [
    "control",
    "pause"
   ],
   "labels": {
    "intent": [
     "control"
    ]
   },
   "secondLabel": {
    "intent": "control"
   },
   "agreedKeys": [
    "intent"
   ],
   "nearestExisting": {
    "id": "resume-please",
    "tokenJaccard": 0.071
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "intent": true
    },
    "claude-haiku": {
     "intent": true
    },
    "claude-sonnet": {
     "intent": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-27",
   "purpose": "frontdoor.intent",
   "tags": [
    "control",
    "stop"
   ],
   "labels": {
    "intent": [
     "control"
    ]
   },
   "secondLabel": {
    "intent": "control"
   },
   "agreedKeys": [
    "intent"
   ],
   "nearestExisting": {
    "id": "approved-ship-it",
    "tokenJaccard": 0.1
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "intent": true
    },
    "claude-haiku": {
     "intent": true
    },
    "claude-sonnet": {
     "intent": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-28",
   "purpose": "frontdoor.intent",
   "tags": [
    "control",
    "setting"
   ],
   "labels": {
    "intent": [
     "control"
    ]
   },
   "secondLabel": {
    "intent": "control"
   },
   "agreedKeys": [
    "intent"
   ],
   "nearestExisting": {
    "id": "cancel-redesign",
    "tokenJaccard": 0.059
   },
   "exact": {
    "jev": false,
    "claude-haiku": false,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "intent": false
    },
    "claude-haiku": {
     "intent": false
    },
    "claude-sonnet": {
     "intent": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": false,
    "2": false,
    "3": false
   }
  },
  {
   "id": "holdout-29",
   "purpose": "memory.is_rule",
   "tags": [
    "obligation"
   ],
   "labels": {
    "kind": [
     "obligation"
    ]
   },
   "secondLabel": {
    "kind": "obligation"
   },
   "agreedKeys": [
    "kind"
   ],
   "nearestExisting": {
    "id": "lint-before-pr",
    "tokenJaccard": 0.056
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "kind": true
    },
    "claude-haiku": {
     "kind": true
    },
    "claude-sonnet": {
     "kind": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-30",
   "purpose": "memory.is_rule",
   "tags": [
    "obligation"
   ],
   "labels": {
    "kind": [
     "obligation"
    ]
   },
   "secondLabel": {
    "kind": "obligation"
   },
   "agreedKeys": [
    "kind"
   ],
   "nearestExisting": {
    "id": "make-targets",
    "tokenJaccard": 0.125
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "kind": true
    },
    "claude-haiku": {
     "kind": true
    },
    "claude-sonnet": {
     "kind": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-31",
   "purpose": "memory.is_rule",
   "tags": [
    "obligation",
    "list-item"
   ],
   "labels": {
    "kind": [
     "obligation"
    ]
   },
   "secondLabel": {
    "kind": "obligation"
   },
   "agreedKeys": [
    "kind"
   ],
   "nearestExisting": {
    "id": "make-targets",
    "tokenJaccard": 0.118
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "kind": true
    },
    "claude-haiku": {
     "kind": true
    },
    "claude-sonnet": {
     "kind": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-32",
   "purpose": "memory.is_rule",
   "tags": [
    "obligation"
   ],
   "labels": {
    "kind": [
     "obligation"
    ]
   },
   "secondLabel": {
    "kind": "obligation"
   },
   "agreedKeys": [
    "kind"
   ],
   "nearestExisting": {
    "id": "feel-free",
    "tokenJaccard": 0.053
   },
   "exact": {
    "jev": false,
    "claude-haiku": false,
    "claude-sonnet": false
   },
   "keyCorrect": {
    "jev": {
     "kind": false
    },
    "claude-haiku": {
     "kind": false
    },
    "claude-sonnet": {
     "kind": false
    }
   },
   "errors": {},
   "jevReps": {
    "1": false,
    "2": false,
    "3": false
   }
  },
  {
   "id": "holdout-33",
   "purpose": "memory.is_rule",
   "tags": [
    "prohibition"
   ],
   "labels": {
    "kind": [
     "prohibition"
    ]
   },
   "secondLabel": {
    "kind": "prohibition"
   },
   "agreedKeys": [
    "kind"
   ],
   "nearestExisting": {
    "id": "ideally-small-prs",
    "tokenJaccard": 0.091
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "kind": true
    },
    "claude-haiku": {
     "kind": true
    },
    "claude-sonnet": {
     "kind": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-34",
   "purpose": "memory.is_rule",
   "tags": [
    "prohibition"
   ],
   "labels": {
    "kind": [
     "prohibition",
     "obligation"
    ]
   },
   "secondLabel": {
    "kind": "prohibition"
   },
   "agreedKeys": [
    "kind"
   ],
   "nearestExisting": {
    "id": "reviews-required",
    "tokenJaccard": 0.105
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "kind": true
    },
    "claude-haiku": {
     "kind": true
    },
    "claude-sonnet": {
     "kind": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-35",
   "purpose": "memory.is_rule",
   "tags": [
    "prohibition"
   ],
   "labels": {
    "kind": [
     "prohibition"
    ]
   },
   "secondLabel": {
    "kind": "prohibition"
   },
   "agreedKeys": [
    "kind"
   ],
   "nearestExisting": {
    "id": "always-test",
    "tokenJaccard": 0.067
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "kind": true
    },
    "claude-haiku": {
     "kind": true
    },
    "claude-sonnet": {
     "kind": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-36",
   "purpose": "memory.is_rule",
   "tags": [
    "preference"
   ],
   "labels": {
    "kind": [
     "preference"
    ]
   },
   "secondLabel": {
    "kind": "preference"
   },
   "agreedKeys": [
    "kind"
   ],
   "nearestExisting": {
    "id": "describes-package",
    "tokenJaccard": 0.133
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "kind": true
    },
    "claude-haiku": {
     "kind": true
    },
    "claude-sonnet": {
     "kind": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-37",
   "purpose": "memory.is_rule",
   "tags": [
    "preference"
   ],
   "labels": {
    "kind": [
     "preference"
    ]
   },
   "secondLabel": {
    "kind": "preference"
   },
   "agreedKeys": [
    "kind"
   ],
   "nearestExisting": {
    "id": "question",
    "tokenJaccard": 0.056
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "kind": true
    },
    "claude-haiku": {
     "kind": true
    },
    "claude-sonnet": {
     "kind": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-38",
   "purpose": "memory.is_rule",
   "tags": [
    "preference"
   ],
   "labels": {
    "kind": [
     "preference"
    ]
   },
   "secondLabel": {
    "kind": "preference"
   },
   "agreedKeys": [
    "kind"
   ],
   "nearestExisting": {
    "id": "make-targets",
    "tokenJaccard": 0.095
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "kind": true
    },
    "claude-haiku": {
     "kind": true
    },
    "claude-sonnet": {
     "kind": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-39",
   "purpose": "memory.is_rule",
   "tags": [
    "not-a-rule",
    "description"
   ],
   "labels": {
    "kind": [
     "not-a-rule"
    ]
   },
   "secondLabel": {
    "kind": "not-a-rule"
   },
   "agreedKeys": [
    "kind"
   ],
   "nearestExisting": {
    "id": "watch-tip",
    "tokenJaccard": 0.095
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "kind": true
    },
    "claude-haiku": {
     "kind": true
    },
    "claude-sonnet": {
     "kind": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-40",
   "purpose": "memory.is_rule",
   "tags": [
    "not-a-rule",
    "description"
   ],
   "labels": {
    "kind": [
     "not-a-rule"
    ]
   },
   "secondLabel": {
    "kind": "not-a-rule"
   },
   "agreedKeys": [
    "kind"
   ],
   "nearestExisting": {
    "id": "keep-in-mind",
    "tokenJaccard": 0.08
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "kind": true
    },
    "claude-haiku": {
     "kind": true
    },
    "claude-sonnet": {
     "kind": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-41",
   "purpose": "memory.is_rule",
   "tags": [
    "not-a-rule",
    "history"
   ],
   "labels": {
    "kind": [
     "not-a-rule"
    ]
   },
   "secondLabel": {
    "kind": "not-a-rule"
   },
   "agreedKeys": [
    "kind"
   ],
   "nearestExisting": {
    "id": "feel-free",
    "tokenJaccard": 0.056
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "kind": true
    },
    "claude-haiku": {
     "kind": true
    },
    "claude-sonnet": {
     "kind": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-42",
   "purpose": "memory.is_rule",
   "tags": [
    "not-a-rule",
    "tip",
    "list-item"
   ],
   "labels": {
    "kind": [
     "not-a-rule"
    ]
   },
   "secondLabel": {
    "kind": "not-a-rule"
   },
   "agreedKeys": [
    "kind"
   ],
   "nearestExisting": {
    "id": "ci-runs",
    "tokenJaccard": 0.214
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "kind": true
    },
    "claude-haiku": {
     "kind": true
    },
    "claude-sonnet": {
     "kind": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-43",
   "purpose": "context.shape",
   "tags": [
    "new-task",
    "writing",
    "examples"
   ],
   "labels": {
    "artifacts": [
     "no"
    ],
    "knowledge": [
     "search"
    ],
    "memories": [
     "yes"
    ],
    "examples": [
     "yes"
    ],
    "scope": [
     "small",
     "medium"
    ],
    "complexity": [
     "simple",
     "moderate"
    ]
   },
   "secondLabel": {
    "artifacts": "yes",
    "knowledge": "search",
    "memories": "yes",
    "examples": "yes",
    "scope": "small",
    "complexity": "moderate"
   },
   "agreedKeys": [
    "knowledge",
    "memories",
    "examples",
    "scope",
    "complexity"
   ],
   "nearestExisting": {
    "id": "asked-followup-notes",
    "tokenJaccard": 0.125
   },
   "exact": {
    "jev": true,
    "claude-haiku": false,
    "claude-sonnet": false
   },
   "keyCorrect": {
    "jev": {
     "artifacts": true,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": true,
     "complexity": true
    },
    "claude-haiku": {
     "artifacts": false,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": true,
     "complexity": true
    },
    "claude-sonnet": {
     "artifacts": false,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": true,
     "complexity": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-44",
   "purpose": "context.shape",
   "tags": [
    "new-task",
    "plan",
    "migration",
    "open-questions"
   ],
   "labels": {
    "artifacts": [
     "no"
    ],
    "knowledge": [
     "search"
    ],
    "memories": [
     "yes"
    ],
    "examples": [
     "no"
    ],
    "scope": [
     "medium"
    ],
    "complexity": [
     "moderate",
     "complex"
    ]
   },
   "secondLabel": {
    "artifacts": "yes",
    "knowledge": "search",
    "memories": "yes",
    "examples": "no",
    "scope": "medium",
    "complexity": "complex"
   },
   "agreedKeys": [
    "knowledge",
    "memories",
    "examples",
    "scope",
    "complexity"
   ],
   "nearestExisting": {
    "id": "ml-new-split-service",
    "tokenJaccard": 0.14
   },
   "exact": {
    "jev": true,
    "claude-haiku": false,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "artifacts": true,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": true,
     "complexity": true
    },
    "claude-haiku": {
     "artifacts": false,
     "knowledge": true,
     "memories": true,
     "examples": false,
     "scope": true,
     "complexity": true
    },
    "claude-sonnet": {
     "artifacts": true,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": true,
     "complexity": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-45",
   "purpose": "context.shape",
   "tags": [
    "follow-up",
    "feature"
   ],
   "labels": {
    "turn": [
     "follow-up"
    ],
    "transcript": [
     "recent",
     "full"
    ],
    "knowledge": [
     "search"
    ],
    "scope": [
     "small",
     "medium"
    ],
    "complexity": [
     "moderate"
    ]
   },
   "secondLabel": {
    "turn": "follow-up",
    "transcript": "full",
    "knowledge": "search",
    "memories": "yes",
    "examples": "yes",
    "scope": "medium",
    "complexity": "moderate"
   },
   "agreedKeys": [
    "turn",
    "transcript",
    "knowledge",
    "scope",
    "complexity"
   ],
   "nearestExisting": {
    "id": "asked-followup-csv",
    "tokenJaccard": 0.143
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "turn": true,
     "transcript": true,
     "knowledge": true,
     "scope": true,
     "complexity": true
    },
    "claude-haiku": {
     "turn": true,
     "transcript": true,
     "knowledge": true,
     "scope": true,
     "complexity": true
    },
    "claude-sonnet": {
     "turn": true,
     "transcript": true,
     "knowledge": true,
     "scope": true,
     "complexity": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-46",
   "purpose": "context.shape",
   "tags": [
    "clarification",
    "feature"
   ],
   "labels": {
    "turn": [
     "clarification"
    ],
    "transcript": [
     "recent",
     "full"
    ],
    "knowledge": [
     "search"
    ],
    "examples": [
     "no"
    ],
    "scope": [
     "small",
     "medium"
    ],
    "complexity": [
     "moderate"
    ]
   },
   "secondLabel": {
    "turn": "clarification",
    "transcript": "full",
    "knowledge": "search",
    "memories": "yes",
    "examples": "no",
    "scope": "small",
    "complexity": "moderate"
   },
   "agreedKeys": [
    "turn",
    "transcript",
    "knowledge",
    "examples",
    "scope",
    "complexity"
   ],
   "nearestExisting": {
    "id": "asked-clarify-limit",
    "tokenJaccard": 0.102
   },
   "exact": {
    "jev": false,
    "claude-haiku": false,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "turn": true,
     "transcript": true,
     "knowledge": false,
     "examples": true,
     "scope": true,
     "complexity": true
    },
    "claude-haiku": {
     "turn": true,
     "transcript": true,
     "knowledge": true,
     "examples": true,
     "scope": true,
     "complexity": false
    },
    "claude-sonnet": {
     "turn": true,
     "transcript": true,
     "knowledge": true,
     "examples": true,
     "scope": true,
     "complexity": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": false,
    "2": false,
    "3": false
   }
  },
  {
   "id": "holdout-47",
   "purpose": "context.shape",
   "tags": [
    "continuation",
    "replan",
    "summary"
   ],
   "labels": {
    "transcript": [
     "summary",
     "recent"
    ],
    "knowledge": [
     "search"
    ],
    "memories": [
     "yes"
    ],
    "examples": [
     "no"
    ],
    "scope": [
     "medium",
     "large"
    ],
    "complexity": [
     "moderate",
     "complex"
    ]
   },
   "secondLabel": {
    "transcript": "summary",
    "knowledge": "search",
    "memories": "yes",
    "examples": "yes",
    "scope": "large",
    "complexity": "complex"
   },
   "agreedKeys": [
    "transcript",
    "knowledge",
    "memories",
    "scope",
    "complexity"
   ],
   "nearestExisting": {
    "id": "ml-cont-latency",
    "tokenJaccard": 0.161
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": false
   },
   "keyCorrect": {
    "jev": {
     "transcript": true,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": true,
     "complexity": true
    },
    "claude-haiku": {
     "transcript": true,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": true,
     "complexity": true
    },
    "claude-sonnet": {
     "transcript": true,
     "knowledge": true,
     "memories": true,
     "examples": false,
     "scope": true,
     "complexity": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-48",
   "purpose": "context.shape",
   "tags": [
    "new-task",
    "mechanical"
   ],
   "labels": {
    "artifacts": [
     "no"
    ],
    "knowledge": [
     "none"
    ],
    "memories": [
     "no"
    ],
    "examples": [
     "no"
    ],
    "scope": [
     "small"
    ],
    "complexity": [
     "simple",
     "moderate"
    ]
   },
   "secondLabel": {
    "artifacts": "yes",
    "knowledge": "none",
    "memories": "no",
    "examples": "no",
    "scope": "small",
    "complexity": "simple"
   },
   "agreedKeys": [
    "knowledge",
    "memories",
    "examples",
    "scope",
    "complexity"
   ],
   "nearestExisting": {
    "id": "ml-cont-latency",
    "tokenJaccard": 0.119
   },
   "exact": {
    "jev": false,
    "claude-haiku": false,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "artifacts": true,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": false,
     "complexity": true
    },
    "claude-haiku": {
     "artifacts": false,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": false,
     "complexity": true
    },
    "claude-sonnet": {
     "artifacts": true,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": true,
     "complexity": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": false,
    "2": false,
    "3": false
   }
  },
  {
   "id": "holdout-49",
   "purpose": "context.shape",
   "tags": [
    "new-task",
    "docs",
    "examples"
   ],
   "labels": {
    "artifacts": [
     "no"
    ],
    "knowledge": [
     "search"
    ],
    "memories": [
     "yes"
    ],
    "examples": [
     "yes"
    ],
    "scope": [
     "small",
     "medium"
    ],
    "complexity": [
     "moderate"
    ]
   },
   "secondLabel": {
    "artifacts": "yes",
    "knowledge": "search",
    "memories": "yes",
    "examples": "yes",
    "scope": "small",
    "complexity": "moderate"
   },
   "agreedKeys": [
    "knowledge",
    "memories",
    "examples",
    "scope",
    "complexity"
   ],
   "nearestExisting": {
    "id": "ml-new-empty-state",
    "tokenJaccard": 0.156
   },
   "exact": {
    "jev": false,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "artifacts": true,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": true,
     "complexity": false
    },
    "claude-haiku": {
     "artifacts": true,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": true,
     "complexity": true
    },
    "claude-sonnet": {
     "artifacts": true,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": true,
     "complexity": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": false,
    "2": false,
    "3": false
   }
  },
  {
   "id": "holdout-50",
   "purpose": "context.shape",
   "tags": [
    "new-task",
    "plan",
    "design",
    "open-questions"
   ],
   "labels": {
    "artifacts": [
     "no"
    ],
    "knowledge": [
     "search"
    ],
    "memories": [
     "yes"
    ],
    "examples": [
     "no"
    ],
    "scope": [
     "medium"
    ],
    "complexity": [
     "complex"
    ]
   },
   "secondLabel": {
    "artifacts": "yes",
    "knowledge": "search",
    "memories": "yes",
    "examples": "no",
    "scope": "medium",
    "complexity": "complex"
   },
   "agreedKeys": [
    "knowledge",
    "memories",
    "examples",
    "scope",
    "complexity"
   ],
   "nearestExisting": {
    "id": "asked-followup-csv",
    "tokenJaccard": 0.125
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": false
   },
   "keyCorrect": {
    "jev": {
     "artifacts": true,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": true,
     "complexity": true
    },
    "claude-haiku": {
     "artifacts": true,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": true,
     "complexity": true
    },
    "claude-sonnet": {
     "artifacts": true,
     "knowledge": true,
     "memories": true,
     "examples": false,
     "scope": true,
     "complexity": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-51",
   "purpose": "context.shape",
   "tags": [
    "continuation",
    "refactor",
    "long-running"
   ],
   "labels": {
    "transcript": [
     "recent",
     "full"
    ],
    "knowledge": [
     "facts",
     "search"
    ],
    "memories": [
     "yes"
    ],
    "examples": [
     "no"
    ],
    "scope": [
     "medium"
    ],
    "complexity": [
     "moderate"
    ]
   },
   "secondLabel": {
    "transcript": "recent",
    "knowledge": "search",
    "memories": "yes",
    "examples": "no",
    "scope": "medium",
    "complexity": "moderate"
   },
   "agreedKeys": [
    "transcript",
    "knowledge",
    "memories",
    "examples",
    "scope",
    "complexity"
   ],
   "nearestExisting": {
    "id": "ml-cont-sso-docs",
    "tokenJaccard": 0.083
   },
   "exact": {
    "jev": false,
    "claude-haiku": false,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "transcript": false,
     "knowledge": false,
     "memories": true,
     "examples": true,
     "scope": false,
     "complexity": true
    },
    "claude-haiku": {
     "transcript": false,
     "knowledge": true,
     "memories": true,
     "examples": false,
     "scope": false,
     "complexity": false
    },
    "claude-sonnet": {
     "transcript": true,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": true,
     "complexity": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": false,
    "2": false,
    "3": false
   }
  },
  {
   "id": "holdout-52",
   "purpose": "context.shape",
   "tags": [
    "new-task",
    "writing",
    "incident",
    "long-request"
   ],
   "labels": {
    "artifacts": [
     "no"
    ],
    "knowledge": [
     "facts",
     "search"
    ],
    "memories": [
     "yes"
    ],
    "examples": [
     "yes"
    ],
    "scope": [
     "medium",
     "large"
    ],
    "complexity": [
     "moderate"
    ]
   },
   "secondLabel": {
    "artifacts": "no",
    "knowledge": "facts",
    "memories": "yes",
    "examples": "yes",
    "scope": "large",
    "complexity": "moderate"
   },
   "agreedKeys": [
    "artifacts",
    "knowledge",
    "memories",
    "examples",
    "scope",
    "complexity"
   ],
   "nearestExisting": {
    "id": "followup-new-requirement",
    "tokenJaccard": 0.083
   },
   "exact": {
    "jev": false,
    "claude-haiku": false,
    "claude-sonnet": false
   },
   "keyCorrect": {
    "jev": {
     "artifacts": true,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": true,
     "complexity": false
    },
    "claude-haiku": {
     "artifacts": true,
     "knowledge": false,
     "memories": false,
     "examples": true,
     "scope": false,
     "complexity": false
    },
    "claude-sonnet": {
     "artifacts": true,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": false,
     "complexity": false
    }
   },
   "errors": {},
   "jevReps": {
    "1": false,
    "2": false,
    "3": false
   }
  },
  {
   "id": "holdout-53",
   "purpose": "context.shape",
   "tags": [
    "new-task",
    "investigation",
    "inbound"
   ],
   "labels": {
    "turn": [
     "new-task",
     "follow-up"
    ],
    "artifacts": [
     "no"
    ],
    "knowledge": [
     "facts",
     "search"
    ],
    "memories": [
     "yes"
    ],
    "examples": [
     "no"
    ],
    "scope": [
     "small",
     "medium"
    ],
    "complexity": [
     "moderate"
    ]
   },
   "secondLabel": {
    "turn": "follow-up",
    "artifacts": "yes",
    "knowledge": "search",
    "memories": "yes",
    "examples": "no",
    "scope": "medium",
    "complexity": "moderate"
   },
   "agreedKeys": [
    "turn",
    "knowledge",
    "memories",
    "examples",
    "scope",
    "complexity"
   ],
   "nearestExisting": {
    "id": "followup-add-param",
    "tokenJaccard": 0.107
   },
   "exact": {
    "jev": true,
    "claude-haiku": false,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "turn": true,
     "artifacts": true,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": true,
     "complexity": true
    },
    "claude-haiku": {
     "turn": true,
     "artifacts": false,
     "knowledge": true,
     "memories": true,
     "examples": false,
     "scope": true,
     "complexity": true
    },
    "claude-sonnet": {
     "turn": true,
     "artifacts": true,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": true,
     "complexity": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-54",
   "purpose": "context.shape",
   "tags": [
    "follow-up",
    "ui",
    "examples"
   ],
   "labels": {
    "turn": [
     "follow-up"
    ],
    "transcript": [
     "recent",
     "full"
    ],
    "knowledge": [
     "facts",
     "search"
    ],
    "examples": [
     "yes"
    ],
    "scope": [
     "small",
     "medium"
    ],
    "complexity": [
     "moderate"
    ]
   },
   "secondLabel": {
    "turn": "follow-up",
    "transcript": "full",
    "knowledge": "search",
    "memories": "yes",
    "examples": "yes",
    "scope": "small",
    "complexity": "moderate"
   },
   "agreedKeys": [
    "turn",
    "transcript",
    "knowledge",
    "examples",
    "scope",
    "complexity"
   ],
   "nearestExisting": {
    "id": "clarify-status-question",
    "tokenJaccard": 0.105
   },
   "exact": {
    "jev": true,
    "claude-haiku": true,
    "claude-sonnet": true
   },
   "keyCorrect": {
    "jev": {
     "turn": true,
     "transcript": true,
     "knowledge": true,
     "examples": true,
     "scope": true,
     "complexity": true
    },
    "claude-haiku": {
     "turn": true,
     "transcript": true,
     "knowledge": true,
     "examples": true,
     "scope": true,
     "complexity": true
    },
    "claude-sonnet": {
     "turn": true,
     "transcript": true,
     "knowledge": true,
     "examples": true,
     "scope": true,
     "complexity": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-55",
   "purpose": "context.shape",
   "tags": [
    "new-task",
    "plan",
    "large"
   ],
   "labels": {
    "artifacts": [
     "no"
    ],
    "knowledge": [
     "search"
    ],
    "memories": [
     "yes"
    ],
    "examples": [
     "no"
    ],
    "scope": [
     "large"
    ],
    "complexity": [
     "complex"
    ]
   },
   "secondLabel": {
    "artifacts": "yes",
    "knowledge": "search",
    "memories": "yes",
    "examples": "yes",
    "scope": "large",
    "complexity": "complex"
   },
   "agreedKeys": [
    "knowledge",
    "memories",
    "scope",
    "complexity"
   ],
   "nearestExisting": {
    "id": "new-oversized",
    "tokenJaccard": 0.096
   },
   "exact": {
    "jev": true,
    "claude-haiku": false,
    "claude-sonnet": false
   },
   "keyCorrect": {
    "jev": {
     "artifacts": true,
     "knowledge": true,
     "memories": true,
     "examples": true,
     "scope": true,
     "complexity": true
    },
    "claude-haiku": {
     "artifacts": true,
     "knowledge": true,
     "memories": true,
     "examples": false,
     "scope": true,
     "complexity": true
    },
    "claude-sonnet": {
     "artifacts": true,
     "knowledge": true,
     "memories": true,
     "examples": false,
     "scope": true,
     "complexity": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": true,
    "2": true,
    "3": true
   }
  },
  {
   "id": "holdout-56",
   "purpose": "context.shape",
   "tags": [
    "new-task",
    "ops",
    "facts"
   ],
   "labels": {
    "artifacts": [
     "no"
    ],
    "knowledge": [
     "facts"
    ],
    "examples": [
     "no"
    ],
    "scope": [
     "small"
    ],
    "complexity": [
     "simple",
     "moderate"
    ]
   },
   "secondLabel": {
    "artifacts": "yes",
    "knowledge": "facts",
    "memories": "yes",
    "examples": "no",
    "scope": "small",
    "complexity": "moderate"
   },
   "agreedKeys": [
    "knowledge",
    "examples",
    "scope",
    "complexity"
   ],
   "nearestExisting": {
    "id": "clarify-answer",
    "tokenJaccard": 0.158
   },
   "exact": {
    "jev": false,
    "claude-haiku": false,
    "claude-sonnet": false
   },
   "keyCorrect": {
    "jev": {
     "artifacts": true,
     "knowledge": true,
     "examples": true,
     "scope": false,
     "complexity": true
    },
    "claude-haiku": {
     "artifacts": false,
     "knowledge": false,
     "examples": true,
     "scope": false,
     "complexity": true
    },
    "claude-sonnet": {
     "artifacts": false,
     "knowledge": false,
     "examples": true,
     "scope": false,
     "complexity": true
    }
   },
   "errors": {},
   "jevReps": {
    "1": false,
    "2": false,
    "3": false
   }
  }
 ],
 "caveats": [
  "The case author and two of the three routers are Claude models, so a same-family label bias is possible. A second labeller (GPT-6.1 Sol) labelled every case blind; the secondary scoring keeps only the keys where its label is inside our acceptable set.",
  "56 cases (14 per decision type) from one author: per-type intervals are very wide.",
  "The routes differ: Jev ran over direct HTTPS, the Claude routers through the Claude Code CLI, which adds start-up time and tokens.",
  "Jev figures are its first repetition; repetitions 2 and 3 and the stability across all three are reported beside it.",
  "The artifacts label violates the protocol rule to leave judgement calls unscored. The author accepted only no on all 9 labelled cases. The second labeller chose no on 1 of 9. Frozen labels stay unchanged; sensitivity calculations are not new runs.",
  "The protocol file was created at 21:35:45.935 UTC on 2026-10-06. It followed the case drafts and an uncounted labeller probe. It preceded the first counted labelling call at 21:35:49.3 UTC, the label freeze at 21:37:02.582 UTC and every router call. It was not written before every call.",
  "The shared Mac also ran local arena model servers from 15:18 local time (20:18 UTC), before all holdout calls. Host contention may affect wall times, especially CLI times. We did not rerun the timing.",
  "The three one-question case sets are near a ceiling: every router scored 12 to 14 of 14 on each. The 95% Wilson interval for 14 of 14 is 78.5% to 100%. These sets provide little room to separate routers.",
  "Per-question Wilson intervals treat questions as independent. Several questions share each context-shape case, so those intervals can understate uncertainty. Label-agreement intervals have the same limit.",
  "Set-aware kappa uses the second label as the author label when the acceptable set contains it. This adaptive calculation can inflate agreement. Only kappaStrict compares two fixed labels.",
  "The tuned set was revised against Jev answers. Its size and decision mix differ from the holdout. Overlapping intervals neither show nor rule out a home advantage.",
  "The three paired McNemar tests have no multiple-test correction. Their p values do not prove equal accuracy."
 ],
 "costRoundingNote": "Calculation: Haiku cost is $7.1285 per 1,000 decisions before rounding, or $7.129 to three decimals. The original extract rounded the list-price figure to $7.128 because of floating-point error."
}