{"$k":["id","title","columns","rows"],"$r":[["routing-holdout-pairs","Same unseen cases, two routers: exact McNemar test",{"$k":["key","label","unit"],"$r":[["pair","Pair","text"],["cases","Cases","count"],["bothRight","Both right","count"],["onlyA","Only first right","count"],["onlyB","Only second right","count"],["bothWrong","Both wrong","count"],["p","Exact McNemar p","\u0001"]]},{"$k":["pair","cases","bothRight","onlyA","onlyB","bothWrong","p"],"$r":[["Jev 1.13 (TypeSafe) vs Claude Haiku 4.5 · Claude Code",56,42,4,2,8,0.688],["Jev 1.13 (TypeSafe) vs Claude Sonnet 5.5 (low) · Claude Code",56,42,4,7,3,0.549],["Claude Haiku 4.5 · Claude Code vs Claude Sonnet 5.5 (low) · Claude Code",56,42,2,7,5,0.18]]}],["routing-holdout-label-agreement","Label agreement: the author vs a second, blind labeller (GPT-6.1 Sol)",{"$k":["key","label","unit"],"$r":[["decision","Decision","text"],["question","Question","text"],["n","Labelled","count"],["agree","Second label inside our acceptable set (95% Wilson interval)","text"],["first","Second label equals our first label (95% Wilson interval)","text"],["kappa","Set-aware kappa (calculation; adaptive author label)","score"],["kappaStrict","Cohen’s kappa, fixed first label (calculation)","score"]]},{"$k":["decision","question","n","agree","first","kappa","kappaStrict"],"$r":[["Failure class","failure",14,"14/14 (78.5% to 100.0%)","14/14 (78.5% to 100.0%)",1,1],["Message intent","intent",14,"14/14 (78.5% to 100.0%)","14/14 (78.5% to 100.0%)",1,1],["Is it a rule?","kind",14,"14/14 (78.5% to 100.0%)","14/14 (78.5% to 100.0%)",1,1],["Context shape","artifacts",9,"1/9 (2.0% to 43.5%)","1/9 (2.0% to 43.5%)",0,0],["Context shape","knowledge",14,"14/14 (78.5% to 100.0%)","11/14 (52.4% to 92.4%)",1,0.567],["Context shape","memories",10,"10/10 (72.2% to 100.0%)","10/10 (72.2% to 100.0%)",1,1],["Context shape","examples",13,"11/13 (57.8% to 95.7%)","11/13 (57.8% to 95.7%)",0.683,0.683],["Context shape","scope",14,"14/14 (78.5% to 100.0%)","10/14 (45.4% to 88.3%)",1,0.533],["Context shape","complexity",14,"14/14 (78.5% to 100.0%)","10/14 (45.4% to 88.3%)",1,0.462],["Context shape","turn",4,"4/4 (51.0% to 100.0%)","3/4 (30.1% to 95.4%)",1,0.556],["Context shape","transcript",5,"5/5 (56.6% to 100.0%)","2/5 (11.8% to 76.9%)",1,0.25]]}],["routing-holdout-tuned-by-type","Tuned set vs unseen holdout, by decision group: exact rate with 95% intervals",{"$k":["key","label","unit"],"$r":[["router","Router","text"],["group","Decision group","text"],["tuned","Tuned set","text"],["holdout","Unseen holdout","text"],["change","Change (points, calculation)","text"],["intervals","Intervals","text"]]},{"$k":["router","group","tuned","holdout","change","intervals"],"$r":[["Jev 1.13 (TypeSafe)","The 3 one-question types (failure class, message intent and is it a rule?)","50/50, 100.0% (92.9% to 100.0%)","38/42, 90.5% (77.9% to 96.2%)","−9.5","overlap"],["Jev 1.13 (TypeSafe)","Context shape (several questions per case)","24/32, 75.0% (57.9% to 86.7%)","8/14, 57.1% (32.6% to 78.6%)","−17.9","overlap"],["Claude Haiku 4.5 · Claude Code","The 3 one-question types (failure class, message intent and is it a rule?)","49/50, 98.0% (89.5% to 99.6%)","39/42, 92.9% (81.0% to 97.5%)","−5.1","overlap"],["Claude Haiku 4.5 · Claude Code","Context shape (several questions per case)","24/32, 75.0% (57.9% to 86.7%)","5/14, 35.7% (16.3% to 61.2%)","−39.3","overlap"],["Claude Sonnet 5.5 (low) · Claude Code","The 3 one-question types (failure class, message intent and is it a rule?)","50/50, 100.0% (92.9% to 100.0%)","41/42, 97.6% (87.7% to 99.6%)","−2.4","overlap"],["Claude Sonnet 5.5 (low) · Claude Code","Context shape (several questions per case)","27/32, 84.4% (68.2% to 93.1%)","8/14, 57.1% (32.6% to 78.6%)","−27.2","overlap"]]}],["routing-holdout-cases","Every holdout case: labels and who answered it exactly",{"$k":["key","label","unit"],"$r":[["case","Case","text"],["decision","Decision","text"],["labels","Acceptable answers","text"],["agreed","Keys both labellers accept","text"],["jev","Jev 1.13 (TypeSafe)","text"],["claude-haiku","Claude Haiku 4.5 · Claude Code","text"],["claude-sonnet","Claude Sonnet 5.5 (low) · Claude Code","text"],["jevReps","Jev exact in 3 repetitions","text"]]},{"$k":["case","decision","labels","agreed","jev","claude-haiku","claude-sonnet","jevReps"],"$r":[["holdout-01","Failure class","failure: transient","1/1","exact","exact","exact","3/3"],["holdout-02","Failure class","failure: transient","1/1","exact","exact","exact","3/3"],["holdout-03","Failure class","failure: transient","1/1","exact","exact","exact","3/3"],["holdout-04","Failure class","failure: impossible-here","1/1","exact","exact","exact","3/3"],["holdout-05","Failure class","failure: impossible-here","1/1","exact","exact","exact","3/3"],["holdout-06","Failure class","failure: impossible-here","1/1","exact","exact","exact","3/3"],["holdout-07","Failure class","failure: impossible-here","1/1","exact","exact","exact","3/3"],["holdout-08","Failure class","failure: flaky","1/1","exact","exact","exact","3/3"],["holdout-09","Failure class","failure: flaky","1/1","0/1 keys","0/1 keys","exact","1/3"],["holdout-10","Failure class","failure: flaky","1/1","exact","exact","exact","3/3"],["holdout-11","Failure class","failure: real","1/1","exact","exact","exact","3/3"],["holdout-12","Failure class","failure: real","1/1","exact","exact","exact","3/3"],["holdout-13","Failure class","failure: real","1/1","exact","exact","exact","3/3"],["holdout-14","Failure class","failure: real","1/1","exact","exact","exact","3/3"],["holdout-15","Message intent","intent: request","1/1","exact","exact","exact","3/3"],["holdout-16","Message intent","intent: request","1/1","exact","exact","exact","3/3"],["holdout-17","Message intent","intent: request","1/1","exact","exact","exact","3/3"],["holdout-18","Message intent","intent: followup","1/1","exact","exact","exact","3/3"],["holdout-19","Message intent","intent: followup","1/1","exact","exact","exact","3/3"],["holdout-20","Message intent","intent: followup","1/1","exact","exact","exact","3/3"],["holdout-21","Message intent","intent: status","1/1","exact","exact","exact","3/3"],["holdout-22","Message intent","intent: status","1/1","exact","exact","exact","3/3"],["holdout-23","Message intent","intent: status","1/1","exact","exact","exact","3/3"],["holdout-24","Message intent","intent: decision","1/1","exact","exact","exact","3/3"],["holdout-25","Message intent","intent: decision","1/1","0/1 keys","exact","exact","0/3"],["holdout-26","Message intent","intent: control","1/1","exact","exact","exact","3/3"],["holdout-27","Message intent","intent: control","1/1","exact","exact","exact","3/3"],["holdout-28","Message intent","intent: control","1/1","0/1 keys","0/1 keys","exact","0/3"],["holdout-29","Is it a rule?","kind: obligation","1/1","exact","exact","exact","3/3"],["holdout-30","Is it a rule?","kind: obligation","1/1","exact","exact","exact","3/3"],["holdout-31","Is it a rule?","kind: obligation","1/1","exact","exact","exact","3/3"],["holdout-32","Is it a rule?","kind: obligation","1/1","0/1 keys","0/1 keys","0/1 keys","0/3"],["holdout-33","Is it a rule?","kind: prohibition","1/1","exact","exact","exact","3/3"],["holdout-34","Is it a rule?","kind: prohibition or obligation","1/1","exact","exact","exact","3/3"],["holdout-35","Is it a rule?","kind: prohibition","1/1","exact","exact","exact","3/3"],["holdout-36","Is it a rule?","kind: preference","1/1","exact","exact","exact","3/3"],["holdout-37","Is it a rule?","kind: preference","1/1","exact","exact","exact","3/3"],["holdout-38","Is it a rule?","kind: preference","1/1","exact","exact","exact","3/3"],["holdout-39","Is it a rule?","kind: not-a-rule","1/1","exact","exact","exact","3/3"],["holdout-40","Is it a rule?","kind: not-a-rule","1/1","exact","exact","exact","3/3"],["holdout-41","Is it a rule?","kind: not-a-rule","1/1","exact","exact","exact","3/3"],["holdout-42","Is it a rule?","kind: not-a-rule","1/1","exact","exact","exact","3/3"],["holdout-43","Context shape","artifacts: no; knowledge: search; memories: yes; examples: yes; scope: small or medium; complexity: simple or moderate","5/6","exact","5/6 keys","5/6 keys","3/3"],["holdout-44","Context shape","artifacts: no; knowledge: search; memories: yes; examples: no; scope: medium; complexity: moderate or complex","5/6","exact","4/6 keys","exact","3/3"],["holdout-45","Context shape","turn: follow-up; transcript: recent or full; knowledge: search; scope: small or medium; complexity: moderate","5/5","exact","exact","exact","3/3"],["holdout-46","Context shape","turn: clarification; transcript: recent or full; knowledge: search; examples: no; scope: small or medium; complexity: moderate","6/6","5/6 keys","5/6 keys","exact","0/3"],["holdout-47","Context shape","transcript: summary or recent; knowledge: search; memories: yes; examples: no; scope: medium or large; complexity: moderate or complex","5/6","exact","exact","5/6 keys","3/3"],["holdout-48","Context shape","artifacts: no; knowledge: none; memories: no; examples: no; scope: small; complexity: simple or moderate","5/6","5/6 keys","4/6 keys","exact","0/3"],["holdout-49","Context shape","artifacts: no; knowledge: search; memories: yes; examples: yes; scope: small or medium; complexity: moderate","5/6","5/6 keys","exact","exact","0/3"],["holdout-50","Context shape","artifacts: no; knowledge: search; memories: yes; examples: no; scope: medium; complexity: complex","5/6","exact","exact","5/6 keys","3/3"],["holdout-51","Context shape","transcript: recent or full; knowledge: facts or search; memories: yes; examples: no; scope: medium; complexity: moderate","6/6","3/6 keys","2/6 keys","exact","0/3"],["holdout-52","Context shape","artifacts: no; knowledge: facts or search; memories: yes; examples: yes; scope: medium or large; complexity: moderate","6/6","5/6 keys","2/6 keys","4/6 keys","0/3"],["holdout-53","Context shape","turn: new-task or follow-up; artifacts: no; knowledge: facts or search; memories: yes; examples: no; scope: small or medium; complexity: moderate","6/7","exact","5/7 keys","exact","3/3"],["holdout-54","Context shape","turn: follow-up; transcript: recent or full; knowledge: facts or search; examples: yes; scope: small or medium; complexity: moderate","6/6","exact","exact","exact","3/3"],["holdout-55","Context shape","artifacts: no; knowledge: search; memories: yes; examples: no; scope: large; complexity: complex","4/6","exact","5/6 keys","5/6 keys","3/3"],["holdout-56","Context shape","artifacts: no; knowledge: facts; examples: no; scope: small; complexity: simple or moderate","4/5","4/5 keys","2/5 keys","2/5 keys","0/3"]]}],["routing-holdout-routers","Every router on the holdout",{"$k":["key","label","unit"],"$r":[["router","Router","text"],["route","Route","text"],["calls","Counted calls","count"],["exact","Exact","text"],["keys","Key accuracy","text"],["agreed","Exact, agreed keys only","text"],["tokens","Mean tokens per decision (calculation; input incl. cache / output)","text"],["cost","USD per 1,000 (calculation)","usd"]]},{"$k":["router","route","calls","exact","keys","agreed","tokens","cost"],"$r":[["Jev 1.13 (TypeSafe)","direct HTTPS to the vendor API with the production request body",168,"46 of 56 (82%, 95% interval 70% to 90%)","113 of 125 (90%, 95% interval 84% to 94%)","46 of 56 (82%, 95% interval 70% to 90%)","730 / 119.3",0.03065],["Claude Haiku 4.5 · Claude Code","Claude Code CLI, one call per decision, CLI default effort (extended thinking)",56,"44 of 56 (79%, 95% interval 66% to 87%)","102 of 125 (82%, 95% interval 74% to 87%)","46 of 56 (82%, 95% interval 70% to 90%)","1,775 / 1,071",7.129],["Claude Sonnet 5.5 (low) · Claude Code","Claude Code CLI, one call per decision, effort low",56,"49 of 56 (88%, 95% interval 76% to 94%)","115 of 125 (92%, 95% interval 86% to 96%)","52 of 56 (93%, 95% interval 83% to 97%)","1,705 / 90.5",7.244]]}]]}