{"$k":["slug","chart"],"$r":[["swe-bench-verified",{"id":"swebench-same-instance-leaderboard","title":"Resolved rate on the same 33 SWE-bench Verified instances","subtitle":"Agent vs 11 public mini-SWE-agent v2 runs, one attempt each","kind":"dot-range","unit":"rate","yLabel":"Resolved","series":[{"name":"Resolved rate","points":{"$k":["label","value","lo","hi","n","highlight"],"$r":[["GPT 5.2 (high)",0.8485,0.6908,0.9335,33,"\u0001"],["Gemini 3 Flash (high)",0.8182,0.6561,0.9139,33,"\u0001"],["GLM 5 (high)",0.7879,0.6225,0.8932,33,"\u0001"],["Agent (Sonnet 5.5, full pipeline)",0.7576,0.5898,0.8717,33,true],["Claude 4.5 Sonnet (high)",0.7576,0.5898,0.8717,33,"\u0001"],["Claude 4.5 Haiku (high)",0.7576,0.5898,0.8717,33,"\u0001"],["Claude 4.5 Opus (high)",0.7273,0.5578,0.8493,33,"\u0001"],["DeepSeek V3.2 (high)",0.7273,0.5578,0.8493,33,"\u0001"],["MiniMax M2.5 (high)",0.697,0.5266,0.8262,33,"\u0001"],["Claude 4.6 Opus",0.697,0.5266,0.8262,33,"\u0001"],["Kimi K2.5 (high)",0.697,0.5266,0.8262,33,"\u0001"],["GPT 5 mini",0.6364,0.4662,0.7781,33,"\u0001"]]}}],"note":"Dots show the rate; whiskers show the 95% Wilson interval for n = 33. The panel compares models under one harness; Agent is a full system on one model.","sourceIds":["agent-swebench-c1","agent-swebench-c2","swebench-leaderboard"]}],["swe-bench-verified",{"id":"swebench-by-difficulty-band","title":"Resolved rate by difficulty band","subtitle":"Band = how many of the 11 public panel models solved the instance","kind":"grouped-bar","unit":"rate","xLabel":"Difficulty band","yLabel":"Resolved","series":[{"name":"Agent","points":{"$k":["label","value","lo","hi","n","highlight"],"$r":[["No panel model solved it",0.25,0.0456,0.6994,4,true],["Under half solved it",0.75,0.3006,0.9544,4,true],["Half or more solved it",0.8182,0.523,0.9486,11,true],["Every panel model solved it",0.8571,0.6006,0.9599,14,true]]}},{"name":"Public panel mean","points":{"$k":["label","value","n"],"$r":[["No panel model solved it",0,4],["Under half solved it",0.2727,4],["Half or more solved it",0.8512,11],["Every panel model solved it",1,14]]}}],"note":"Agent whiskers are 95% Wilson intervals. The \"no panel model solved it\" band has 4 instances; Agent resolved 1 (matplotlib__matplotlib-21568).","sourceIds":["agent-swebench-c1","agent-swebench-c2","swebench-leaderboard","swebench-protocol"]}],["swe-bench-verified",{"id":"swebench-model-calls","title":"Model calls per instance","subtitle":"Mean over the same 33 instances","kind":"bar","unit":"calls","yLabel":"Calls per instance","series":[{"name":"Mean calls","points":{"$k":["label","value","n","highlight"],"$r":[["DeepSeek V3.2 (high)",88.2,33,"\u0001"],["GLM 5 (high)",77.5,33,"\u0001"],["Claude 4.5 Haiku (high)",68.5,33,"\u0001"],["MiniMax M2.5 (high)",58.4,33,"\u0001"],["Kimi K2.5 (high)",56.7,33,"\u0001"],["Gemini 3 Flash (high)",54.2,33,"\u0001"],["Claude 4.5 Sonnet (high)",51,33,"\u0001"],["Agent",49.5,33,true],["Claude 4.5 Opus (high)",35.9,33,"\u0001"],["GPT 5.2 (high)",35.6,33,"\u0001"],["Claude 4.6 Opus",28.9,33,"\u0001"],["GPT 5 mini",20.8,33,"\u0001"]]}}],"note":"A panel call is one bash-agent step. An Agent call is one model request of any stage (research, plan, act, verify, review).","sourceIds":["agent-swebench-c1","agent-swebench-c2","swebench-leaderboard"]}],["swe-bench-verified",{"id":"swebench-views","title":"Every way to slice the run, with intervals","subtitle":"Agent resolved rate and 95% Wilson interval per declared view","kind":"dot-range","unit":"rate","yLabel":"Resolved","series":[{"name":"Agent","points":{"$k":["label","value","lo","hi","n","highlight"],"$r":[["Campaign 1: 25-instance sample",0.72,0.5242,0.8572,25,"\u0001"],["Campaign 2: 8 compiled-extension instances",0.875,0.5291,0.9776,8,"\u0001"],["Original seed draw of 25",0.76,0.5657,0.885,25,"\u0001"],["All 33 attempted",0.7576,0.5898,0.8717,33,true]]}},{"name":"Public panel mean, same instances","points":{"$k":["label","value","n"],"$r":[["Campaign 1: 25-instance sample",0.7091,25],["Campaign 2: 8 compiled-extension instances",0.8409,8],["Original seed draw of 25",0.7091,25],["All 33 attempted",0.741,33]]}}],"note":"The original draw and \"all 33\" mix two platform builds.","sourceIds":["agent-swebench-c1","agent-swebench-c2","swebench-leaderboard","swebench-protocol"]}],["cost-thought-experiments",{"id":"cost-per-resolved-agent-vs-panel","title":"Recorded cost per resolved instance: Agent vs the public panel","subtitle":"Same 33 SWE-bench Verified instances; all attempts in the numerator","kind":"bar","unit":"usd","yLabel":"USD per resolved instance","series":[{"name":"Cost per resolved instance","points":{"$k":["label","value","n","highlight"],"$r":[["Agent (notional)",3.706,25,true],["Claude 4.5 Opus (high)",1.184,24,"\u0001"],["Claude 4.5 Sonnet (high)",0.913,25,"\u0001"],["Claude 4.6 Opus",0.875,23,"\u0001"],["GLM 5 (high)",0.667,26,"\u0001"],["DeepSeek V3.2 (high)",0.637,24,"\u0001"],["GPT 5.2 (high)",0.628,28,"\u0001"],["Claude 4.5 Haiku (high)",0.479,25,"\u0001"],["Gemini 3 Flash (high)",0.436,27,"\u0001"],["Kimi K2.5 (high)",0.256,23,"\u0001"],["MiniMax M2.5 (high)",0.107,23,"\u0001"],["GPT 5 mini",0.08,21,"\u0001"]]}}],"note":"Recorded figures, not repricing. Panel costs are published API costs for a bash-only agent. Agent's figure is a list-price estimate of subscription calls and includes onboarding, planning, verification and review.","sourceIds":["agent-swebench-c1","agent-swebench-c2","swebench-leaderboard"]}]]}