{
 "kind": "swebench-opus-vs-sonnet",
 "status": "interim",
 "declaredAt": "2026-10-06T12:21:45-05:00",
 "recordedAt": "2026-10-06",
 "note": "The Opus arm ran on the current platform build; the Sonnet arm is the earlier campaign attempt on the same instance (the same record as raw/swebench). Notional cost is the platform price table at each run commit on subscription runs: a list-price estimate, not an invoice.",
 "grader": "Official SWE-bench harness 5.0.2 with the official instance images, amd64 emulation",
 "opusPlatform": "4f6f4027",
 "halt": {
  "step": "usage gate",
  "before": "django__django-11885",
  "weeklyLeftPercent": 2,
  "fiveHourLeftPercent": 32,
  "weeklyReset": "2026-10-09"
 },
 "reported": {
  "opusResolved": 2,
  "sonnetResolved": 1,
  "mcnemarExactP": 1,
  "costRatioOpusToSonnet": 2.63
 },
 "instances": [
  {
   "order": 1,
   "instanceId": "django__django-10554",
   "band": "0-none",
   "panelSolvedOf11": 0,
   "status": "graded",
   "goldResolvedOnHost": true,
   "opus": {
    "model": "claude-opus-5-5",
    "resolved": true,
    "outcome": "resolved",
    "stopReason": "in_review",
    "costUsd": 9.9895,
    "costEstimated": true,
    "calls": 72,
    "workerMinutes": 25.29,
    "inputTokens": 7671512,
    "cachedTokens": 7028625,
    "outputTokens": 142402,
    "patchFiles": [
     "django/db/models/sql/compiler.py",
     "tests/queries/test_qs_combinators.py"
    ],
    "platform": "4f6f4027",
    "pipelineStages": {
     "research": {
      "calls": 14,
      "costUsd": 2.3369,
      "activeMin": 3.81
     },
     "plan": {
      "calls": 1,
      "costUsd": 0.2399,
      "activeMin": 1.26
     },
     "implement": {
      "calls": 23,
      "costUsd": 2.4494,
      "activeMin": 4.69
     },
     "validate": {
      "calls": 7,
      "costUsd": 1.352,
      "activeMin": 3.16
     },
     "review": {
      "calls": 3,
      "costUsd": 0.8626,
      "activeMin": 2.59
     },
     "deliver": {
      "calls": 16,
      "costUsd": 2.3531,
      "activeMin": 9.53
     }
    },
    "onboarding": {
     "calls": 1,
     "costUsd": 0.1147,
     "minutes": 0.51
    },
    "modelCallsOk": 72,
    "refusal": false,
    "watchdogStop": false
   },
   "sonnet": {
    "model": "claude-sonnet-5-5",
    "resolved": false,
    "outcome": "unresolved",
    "stopReason": "escalation",
    "costUsd": 1.5186,
    "costEstimated": true,
    "calls": 34,
    "workerMinutes": 4.74,
    "inputTokens": 3216448,
    "cachedTokens": 3084636,
    "outputTokens": 28058,
    "patchFiles": [
     "tests/queries/test_qs_combinators.py"
    ],
    "platform": "f0ac3a8a",
    "pipelineStages": {
     "research": {
      "calls": 12,
      "costUsd": 0.616,
      "activeMin": 1.2
     },
     "plan": {
      "calls": 1,
      "costUsd": 0.0799,
      "activeMin": 0.52
     },
     "implement": {
      "calls": 9,
      "costUsd": 0.4154,
      "activeMin": 1.27
     },
     "validate": {
      "calls": 1,
      "costUsd": 0.071,
      "activeMin": 0.23
     },
     "review": {
      "calls": 1,
      "costUsd": 0.0585,
      "activeMin": 0.6
     },
     "deliver": {
      "calls": 4,
      "costUsd": 0.1723,
      "activeMin": 0.55
     }
    },
    "onboarding": {
     "calls": 1,
     "costUsd": 0.0694,
     "minutes": 0.49
    },
    "campaign": "c1-f0ac3a8a"
   }
  },
  {
   "order": 2,
   "instanceId": "matplotlib__matplotlib-21568",
   "band": "0-none",
   "panelSolvedOf11": 0,
   "status": "graded",
   "goldResolvedOnHost": true,
   "opus": {
    "model": "claude-opus-5-5",
    "resolved": true,
    "outcome": "resolved",
    "stopReason": "escalation",
    "costUsd": 4.902,
    "costEstimated": true,
    "calls": 42,
    "workerMinutes": 9.76,
    "inputTokens": 3829566,
    "cachedTokens": 3457289,
    "outputTokens": 56050,
    "patchFiles": [
     "lib/matplotlib/dates.py",
     "lib/matplotlib/tests/test_dates.py"
    ],
    "platform": "4f6f4027",
    "pipelineStages": {
     "research": {
      "calls": 9,
      "costUsd": 1.5504,
      "activeMin": 1.25
     },
     "plan": {
      "calls": 1,
      "costUsd": 0.1739,
      "activeMin": 0.89
     },
     "implement": {
      "calls": 5,
      "costUsd": 0.4098,
      "activeMin": 0.54
     },
     "validate": {
      "calls": 10,
      "costUsd": 1.3507,
      "activeMin": 4.19
     },
     "review": {
      "calls": 3,
      "costUsd": 0.484,
      "activeMin": 1.22
     },
     "deliver": {
      "calls": 7,
      "costUsd": 0.5657,
      "activeMin": 1.44
     }
    },
    "onboarding": {
     "calls": 1,
     "costUsd": 0.0923,
     "minutes": 0.46
    },
    "modelCallsOk": 42,
    "refusal": false,
    "watchdogStop": false
   },
   "sonnet": {
    "model": "claude-sonnet-5-5",
    "resolved": true,
    "outcome": "resolved",
    "stopReason": "in_review",
    "costUsd": 3.0234,
    "costEstimated": true,
    "calls": 53,
    "workerMinutes": 9.37,
    "inputTokens": 5528274,
    "cachedTokens": 5180677,
    "outputTokens": 52118,
    "patchFiles": [
     "lib/matplotlib/dates.py",
     "lib/matplotlib/tests/test_dates.py"
    ],
    "platform": "236c0d3f",
    "pipelineStages": {
     "research": {
      "calls": 6,
      "costUsd": 0.661,
      "activeMin": 0.76
     },
     "plan": {
      "calls": 2,
      "costUsd": 0.1278,
      "activeMin": 0.51
     },
     "implement": {
      "calls": 9,
      "costUsd": 0.3983,
      "activeMin": 0.89
     },
     "validate": {
      "calls": 12,
      "costUsd": 0.7742,
      "activeMin": 3.26
     },
     "review": {
      "calls": 3,
      "costUsd": 0.2003,
      "activeMin": 1.06
     },
     "deliver": {
      "calls": 15,
      "costUsd": 0.7564,
      "activeMin": 2.44
     }
    },
    "onboarding": {
     "calls": 1,
     "costUsd": 0.0736,
     "minutes": 0.57
    },
    "campaign": "c2-compiled-236c0d3f"
   }
  },
  {
   "order": 3,
   "instanceId": "django__django-15022",
   "band": "1-under-half",
   "panelSolvedOf11": 4,
   "status": "graded",
   "goldResolvedOnHost": true,
   "opus": {
    "model": "claude-opus-5-5",
    "resolved": false,
    "outcome": "unresolved",
    "stopReason": "escalation",
    "costUsd": 7.8724,
    "costEstimated": true,
    "calls": 71,
    "workerMinutes": 20.26,
    "inputTokens": 7527852,
    "cachedTokens": 7076968,
    "outputTokens": 103734,
    "patchFiles": [
     "django/contrib/admin/options.py",
     "tests/admin_changelist/tests.py",
     "tests/admin_views/test_autocomplete_view.py"
    ],
    "platform": "4f6f4027",
    "pipelineStages": {
     "research": {
      "calls": 21,
      "costUsd": 2.3879,
      "activeMin": 3.32
     },
     "plan": {
      "calls": 2,
      "costUsd": 0.7004,
      "activeMin": 3.54
     },
     "implement": {
      "calls": 23,
      "costUsd": 2.0779,
      "activeMin": 5.94
     },
     "validate": {
      "calls": 3,
      "costUsd": 0.269,
      "activeMin": 0.59
     },
     "review": {
      "calls": 4,
      "costUsd": 0.6682,
      "activeMin": 3.21
     },
     "deliver": {
      "calls": 11,
      "costUsd": 1.4174,
      "activeMin": 3.42
     }
    },
    "onboarding": {
     "calls": 1,
     "costUsd": 0.1058,
     "minutes": 0.5
    },
    "modelCallsOk": 71,
    "refusal": false,
    "watchdogStop": false
   },
   "sonnet": {
    "model": "claude-sonnet-5-5",
    "resolved": false,
    "outcome": "unresolved",
    "stopReason": "in_review",
    "costUsd": 4.0978,
    "costEstimated": true,
    "calls": 62,
    "workerMinutes": 15,
    "inputTokens": 6399305,
    "cachedTokens": 5997010,
    "outputTokens": 94182,
    "patchFiles": [
     "django/contrib/admin/options.py",
     "tests/admin_changelist/tests.py"
    ],
    "platform": "f0ac3a8a",
    "pipelineStages": {
     "research": {
      "calls": 12,
      "costUsd": 1.0622,
      "activeMin": 2.16
     },
     "plan": {
      "calls": 8,
      "costUsd": 0.9531,
      "activeMin": 4.92
     },
     "implement": {
      "calls": 11,
      "costUsd": 0.5103,
      "activeMin": 1.48
     },
     "validate": {
      "calls": 8,
      "costUsd": 0.5766,
      "activeMin": 2.71
     },
     "review": {
      "calls": 4,
      "costUsd": 0.2467,
      "activeMin": 1.93
     },
     "deliver": {
      "calls": 13,
      "costUsd": 0.6284,
      "activeMin": 1.52
     }
    },
    "onboarding": {
     "calls": 1,
     "costUsd": 0.0765,
     "minutes": 0.52
    },
    "campaign": "c1-f0ac3a8a"
   }
  },
  {
   "order": 4,
   "instanceId": "django__django-11885",
   "band": "1-under-half",
   "panelSolvedOf11": 3,
   "status": "not-started",
   "goldResolvedOnHost": null,
   "opus": null,
   "sonnet": {
    "model": "claude-sonnet-5-5",
    "resolved": true,
    "outcome": "resolved",
    "stopReason": "in_review",
    "costUsd": 2.2653,
    "costEstimated": true,
    "calls": 38,
    "workerMinutes": 7.29,
    "inputTokens": 3925238,
    "cachedTokens": 3713701,
    "outputTokens": 47485,
    "patchFiles": [
     "django/db/models/deletion.py",
     "tests/delete/tests.py"
    ],
    "platform": "f0ac3a8a",
    "pipelineStages": {
     "research": {
      "calls": 10,
      "costUsd": 0.5827,
      "activeMin": 1.18
     },
     "plan": {
      "calls": 2,
      "costUsd": 0.2213,
      "activeMin": 1.2
     },
     "implement": {
      "calls": 9,
      "costUsd": 0.5531,
      "activeMin": 1.7
     },
     "validate": {
      "calls": 4,
      "costUsd": 0.2697,
      "activeMin": 0.74
     },
     "review": {
      "calls": 1,
      "costUsd": 0.1103,
      "activeMin": 1.22
     },
     "deliver": {
      "calls": 7,
      "costUsd": 0.4387,
      "activeMin": 1.01
     }
    },
    "onboarding": {
     "calls": 1,
     "costUsd": 0.0768,
     "minutes": 0.47
    },
    "campaign": "c1-f0ac3a8a"
   }
  },
  {
   "order": 5,
   "instanceId": "django__django-14631",
   "band": "2-half-plus",
   "panelSolvedOf11": 9,
   "status": "not-started",
   "goldResolvedOnHost": null,
   "opus": null,
   "sonnet": {
    "model": "claude-sonnet-5-5",
    "resolved": false,
    "outcome": "empty_patch",
    "stopReason": "escalation",
    "costUsd": 2.7991,
    "costEstimated": true,
    "calls": 47,
    "workerMinutes": 38.17,
    "inputTokens": 4895047,
    "cachedTokens": 4620790,
    "outputTokens": 60006,
    "patchFiles": [],
    "platform": "f0ac3a8a",
    "pipelineStages": {
     "research": {
      "calls": 13,
      "costUsd": 0.7584,
      "activeMin": 1.37
     },
     "plan": {
      "calls": 7,
      "costUsd": 0.7377,
      "activeMin": 3.09
     },
     "implement": {
      "calls": 10,
      "costUsd": 0.49,
      "activeMin": 2.08
     },
     "validate": {
      "calls": 10,
      "costUsd": 0.6155,
      "activeMin": 30.63
     },
     "review": {
      "calls": 1,
      "costUsd": 0.0738,
      "activeMin": 0.71
     },
     "deliver": {
      "calls": 0,
      "costUsd": 0,
      "activeMin": 0
     }
    },
    "onboarding": {
     "calls": 1,
     "costUsd": 0.0556,
     "minutes": 0.41
    },
    "campaign": "c1-f0ac3a8a"
   }
  },
  {
   "order": 6,
   "instanceId": "django__django-12193",
   "band": "2-half-plus",
   "panelSolvedOf11": 10,
   "status": "not-started",
   "goldResolvedOnHost": null,
   "opus": null,
   "sonnet": {
    "model": "claude-sonnet-5-5",
    "resolved": true,
    "outcome": "resolved",
    "stopReason": "in_review",
    "costUsd": 2.0633,
    "costEstimated": true,
    "calls": 43,
    "workerMinutes": 5.74,
    "inputTokens": 4076373,
    "cachedTokens": 3862815,
    "outputTokens": 36215,
    "patchFiles": [
     "django/forms/widgets.py",
     "tests/forms_tests/widget_tests/test_checkboxinput.py",
     "tests/postgres_tests/test_array.py"
    ],
    "platform": "f0ac3a8a",
    "pipelineStages": {
     "research": {
      "calls": 10,
      "costUsd": 0.4855,
      "activeMin": 0.66
     },
     "plan": {
      "calls": 1,
      "costUsd": 0.0692,
      "activeMin": 0.3
     },
     "implement": {
      "calls": 9,
      "costUsd": 0.4392,
      "activeMin": 1.02
     },
     "validate": {
      "calls": 6,
      "costUsd": 0.4918,
      "activeMin": 1.46
     },
     "review": {
      "calls": 4,
      "costUsd": 0.1391,
      "activeMin": 0.75
     },
     "deliver": {
      "calls": 8,
      "costUsd": 0.3517,
      "activeMin": 1.33
     }
    },
    "onboarding": {
     "calls": 1,
     "costUsd": 0.0597,
     "minutes": 0.41
    },
    "campaign": "c1-f0ac3a8a"
   }
  },
  {
   "order": 7,
   "instanceId": "sympy__sympy-18189",
   "band": "3-all",
   "panelSolvedOf11": 11,
   "status": "not-started",
   "goldResolvedOnHost": null,
   "opus": null,
   "sonnet": {
    "model": "claude-sonnet-5-5",
    "resolved": false,
    "outcome": "empty_patch",
    "stopReason": "escalation",
    "costUsd": 0.8898,
    "costEstimated": true,
    "calls": 13,
    "workerMinutes": 1.63,
    "inputTokens": 996024,
    "cachedTokens": 846144,
    "outputTokens": 12113,
    "patchFiles": [],
    "platform": "f0ac3a8a",
    "pipelineStages": {
     "research": {
      "calls": 5,
      "costUsd": 0.6128,
      "activeMin": 0.51
     },
     "plan": {
      "calls": 4,
      "costUsd": 0.2136,
      "activeMin": 0.92
     },
     "implement": {
      "calls": 0,
      "costUsd": 0,
      "activeMin": 0
     },
     "validate": {
      "calls": 0,
      "costUsd": 0,
      "activeMin": 0
     },
     "review": {
      "calls": 0,
      "costUsd": 0,
      "activeMin": 0
     },
     "deliver": {
      "calls": 0,
      "costUsd": 0,
      "activeMin": 0
     }
    },
    "onboarding": {
     "calls": 1,
     "costUsd": 0.0497,
     "minutes": 0.3
    },
    "campaign": "c1-f0ac3a8a"
   }
  },
  {
   "order": 8,
   "instanceId": "django__django-11333",
   "band": "3-all",
   "panelSolvedOf11": 11,
   "status": "not-started",
   "goldResolvedOnHost": null,
   "opus": null,
   "sonnet": {
    "model": "claude-sonnet-5-5",
    "resolved": true,
    "outcome": "resolved",
    "stopReason": "in_review",
    "costUsd": 2.5122,
    "costEstimated": true,
    "calls": 46,
    "workerMinutes": 7.99,
    "inputTokens": 4637892,
    "cachedTokens": 4396072,
    "outputTokens": 55592,
    "patchFiles": [
     "django/urls/base.py",
     "django/urls/resolvers.py",
     "tests/urlpatterns/test_resolvers.py"
    ],
    "platform": "f0ac3a8a",
    "pipelineStages": {
     "research": {
      "calls": 8,
      "costUsd": 0.4405,
      "activeMin": 0.77
     },
     "plan": {
      "calls": 5,
      "costUsd": 0.3926,
      "activeMin": 1.7
     },
     "implement": {
      "calls": 11,
      "costUsd": 0.5269,
      "activeMin": 1.38
     },
     "validate": {
      "calls": 5,
      "costUsd": 0.4497,
      "activeMin": 1.44
     },
     "review": {
      "calls": 4,
      "costUsd": 0.1884,
      "activeMin": 0.88
     },
     "deliver": {
      "calls": 9,
      "costUsd": 0.4403,
      "activeMin": 1.58
     }
    },
    "onboarding": {
     "calls": 1,
     "costUsd": 0.0697,
     "minutes": 0.5
    },
    "campaign": "c1-f0ac3a8a"
   }
  }
 ],
 "protocolNote": "The run protocol was declared before the first run. Its public summary is the Method section of the study page."
}