{"$k":["slug","chart"],"$r":[["swe-bench-verified",{"id":"swebench-same-instance-leaderboard","title":"Resolved rate on the same 33 SWE-bench Verified instances","subtitle":"Agent vs 11 public mini-SWE-agent v2 runs, one attempt each","kind":"dot-range","unit":"rate","yLabel":"Resolved","series":[{"name":"Resolved rate","points":{"$k":["label","value","lo","hi","n","highlight"],"$r":[["GPT 5.2 (high)",0.8485,0.6908,0.9335,33,"\u0001"],["Gemini 3 Flash (high)",0.8182,0.6561,0.9139,33,"\u0001"],["GLM 5 (high)",0.7879,0.6225,0.8932,33,"\u0001"],["Agent (Sonnet 5.5, full pipeline)",0.7576,0.5898,0.8717,33,true],["Claude 4.5 Sonnet (high)",0.7576,0.5898,0.8717,33,"\u0001"],["Claude 4.5 Haiku (high)",0.7576,0.5898,0.8717,33,"\u0001"],["Claude 4.5 Opus (high)",0.7273,0.5578,0.8493,33,"\u0001"],["DeepSeek V3.2 (high)",0.7273,0.5578,0.8493,33,"\u0001"],["MiniMax M2.5 (high)",0.697,0.5266,0.8262,33,"\u0001"],["Claude 4.6 Opus",0.697,0.5266,0.8262,33,"\u0001"],["Kimi K2.5 (high)",0.697,0.5266,0.8262,33,"\u0001"],["GPT 5 mini",0.6364,0.4662,0.7781,33,"\u0001"]]}}],"note":"Dots show the rate; whiskers show the 95% Wilson interval for n = 33. The panel compares models under one harness; Agent is a full system on one model.","sourceIds":["agent-swebench-c1","agent-swebench-c2","swebench-leaderboard"]}],["swe-bench-verified",{"id":"swebench-model-calls","title":"Model calls per instance","subtitle":"Mean over the same 33 instances","kind":"bar","unit":"calls","yLabel":"Calls per instance","series":[{"name":"Mean calls","points":{"$k":["label","value","n","highlight"],"$r":[["DeepSeek V3.2 (high)",88.2,33,"\u0001"],["GLM 5 (high)",77.5,33,"\u0001"],["Claude 4.5 Haiku (high)",68.5,33,"\u0001"],["MiniMax M2.5 (high)",58.4,33,"\u0001"],["Kimi K2.5 (high)",56.7,33,"\u0001"],["Gemini 3 Flash (high)",54.2,33,"\u0001"],["Claude 4.5 Sonnet (high)",51,33,"\u0001"],["Agent",49.5,33,true],["Claude 4.5 Opus (high)",35.9,33,"\u0001"],["GPT 5.2 (high)",35.6,33,"\u0001"],["Claude 4.6 Opus",28.9,33,"\u0001"],["GPT 5 mini",20.8,33,"\u0001"]]}}],"note":"A panel call is one bash-agent step. An Agent call is one model request of any stage (research, plan, act, verify, review).","sourceIds":["agent-swebench-c1","agent-swebench-c2","swebench-leaderboard"]}],["cost-thought-experiments",{"id":"cost-per-resolved-agent-vs-panel","title":"Recorded cost per resolved instance: Agent vs the public panel","subtitle":"Same 33 SWE-bench Verified instances; all attempts in the numerator","kind":"bar","unit":"usd","yLabel":"USD per resolved instance","series":[{"name":"Cost per resolved instance","points":{"$k":["label","value","n","highlight"],"$r":[["Agent (notional)",3.706,25,true],["Claude 4.5 Opus (high)",1.184,24,"\u0001"],["Claude 4.5 Sonnet (high)",0.913,25,"\u0001"],["Claude 4.6 Opus",0.875,23,"\u0001"],["GLM 5 (high)",0.667,26,"\u0001"],["DeepSeek V3.2 (high)",0.637,24,"\u0001"],["GPT 5.2 (high)",0.628,28,"\u0001"],["Claude 4.5 Haiku (high)",0.479,25,"\u0001"],["Gemini 3 Flash (high)",0.436,27,"\u0001"],["Kimi K2.5 (high)",0.256,23,"\u0001"],["MiniMax M2.5 (high)",0.107,23,"\u0001"],["GPT 5 mini",0.08,21,"\u0001"]]}}],"note":"Recorded figures, not repricing. Panel costs are published API costs for a bash-only agent. Agent's figure is a list-price estimate of subscription calls and includes onboarding, planning, verification and review.","sourceIds":["agent-swebench-c1","agent-swebench-c2","swebench-leaderboard"]}]]}