{
 "date": "2026-10-20",
 "note": "CodeTwig local-model test. Qwen3.8-27B-MXFP4 on vLLM 0.27.1 (2x Radeon AI PRO R9700, 210 W cap each) vs claude-sonnet-5-5 and claude-haiku-5-5. Model outputs omitted; per-run metrics and grades only.",
 "overall": {
  "qwen-off": {
   "passes": 12,
   "n": 18,
   "cost_usd": 0,
   "total_request_time_s": 68.4,
   "errors": 0
  },
  "qwen-on": {
   "passes": 15,
   "n": 18,
   "cost_usd": 0,
   "total_request_time_s": 236.5,
   "errors": 0
  },
  "sonnet-off": {
   "passes": 16,
   "n": 18,
   "cost_usd": 0.2166,
   "total_request_time_s": 101.5,
   "errors": 0
  },
  "sonnet-adaptive": {
   "passes": 18,
   "n": 18,
   "cost_usd": 0.3018,
   "total_request_time_s": 156.2,
   "errors": 0
  },
  "haiku-off": {
   "passes": 14,
   "n": 18,
   "cost_usd": 0.0102,
   "total_request_time_s": 60.2,
   "errors": 0
  }
 },
 "total_api_spend_usd": 0.5286,
 "gpu": {
  "gpu0": {
   "peak_power_W": 217.0,
   "median_power_W": 210.0,
   "peak_vram_GB": 33.88,
   "samples": 307,
   "cap_W": 210
  },
  "gpu1": {
   "peak_power_W": 223.0,
   "median_power_W": 210.0,
   "peak_vram_GB": 33.88,
   "samples": 307,
   "cap_W": 210
  },
  "qwen_window_s": 308.8
 },
 "table": [
  {
   "task": "01-subtle-bug",
   "config": "qwen-off",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.18,
   "median_ttft_answer_s": 0.18,
   "median_total_s": 2.2,
   "median_output_tokens": 438,
   "median_output_tok_s": 218.32,
   "cost_usd_total": 0,
   "ttft_by_repeat": [
    0.19,
    0.18,
    0.18
   ]
  },
  {
   "task": "02-multifile-diff",
   "config": "qwen-off",
   "n": 3,
   "passes": 0,
   "statuses": [
    "logic_right_patch_failed",
    "logic_right_patch_failed",
    "logic_right_patch_failed"
   ],
   "median_ttft_s": 0.28,
   "median_ttft_answer_s": 0.28,
   "median_total_s": 3.16,
   "median_output_tokens": 783,
   "median_output_tok_s": 258.59,
   "cost_usd_total": 0,
   "ttft_by_repeat": [
    0.3,
    0.22,
    0.28
   ]
  },
  {
   "task": "03-algorithm",
   "config": "qwen-off",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.18,
   "median_ttft_answer_s": 0.18,
   "median_total_s": 3.28,
   "median_output_tokens": 691,
   "median_output_tok_s": 224.97,
   "cost_usd_total": 0,
   "ttft_by_repeat": [
    0.18,
    0.18,
    0.18
   ]
  },
  {
   "task": "04-long-context",
   "config": "qwen-off",
   "n": 3,
   "passes": 0,
   "statuses": [
    "3/4 correct",
    "3/4 correct",
    "3/4 correct"
   ],
   "median_ttft_s": 0.49,
   "median_ttft_answer_s": 0.49,
   "median_total_s": 1.17,
   "median_output_tokens": 118,
   "median_output_tok_s": 207.97,
   "cost_usd_total": 0,
   "ttft_by_repeat": [
    1.73,
    0.49,
    0.48
   ]
  },
  {
   "task": "05-logic",
   "config": "qwen-off",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.15,
   "median_ttft_answer_s": 0.15,
   "median_total_s": 11.65,
   "median_output_tokens": 2687,
   "median_output_tok_s": 237.37,
   "cost_usd_total": 0,
   "ttft_by_repeat": [
    0.15,
    0.16,
    0.14
   ]
  },
  {
   "task": "06-tool-json",
   "config": "qwen-off",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.2,
   "median_ttft_answer_s": 0.2,
   "median_total_s": 0.88,
   "median_output_tokens": 217,
   "median_output_tok_s": 315.29,
   "cost_usd_total": 0,
   "ttft_by_repeat": [
    0.33,
    0.2,
    0.19
   ]
  },
  {
   "task": "01-subtle-bug",
   "config": "qwen-on",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.18,
   "median_ttft_answer_s": 4.99,
   "median_total_s": 7.11,
   "median_output_tokens": 1594,
   "median_output_tok_s": 229.79,
   "cost_usd_total": 0,
   "ttft_by_repeat": [
    0.18,
    0.18,
    0.18
   ]
  },
  {
   "task": "02-multifile-diff",
   "config": "qwen-on",
   "n": 3,
   "passes": 1,
   "statuses": [
    "logic_right_patch_failed",
    "pass",
    "logic_right_patch_failed"
   ],
   "median_ttft_s": 0.22,
   "median_ttft_answer_s": 22.16,
   "median_total_s": 24.77,
   "median_output_tokens": 5906,
   "median_output_tok_s": 244.91,
   "cost_usd_total": 0,
   "ttft_by_repeat": [
    0.22,
    0.23,
    0.21
   ]
  },
  {
   "task": "03-algorithm",
   "config": "qwen-on",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.18,
   "median_ttft_answer_s": 15.59,
   "median_total_s": 18.28,
   "median_output_tokens": 3498,
   "median_output_tok_s": 193.99,
   "cost_usd_total": 0,
   "ttft_by_repeat": [
    0.18,
    0.18,
    0.18
   ]
  },
  {
   "task": "04-long-context",
   "config": "qwen-on",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.49,
   "median_ttft_answer_s": 7.94,
   "median_total_s": 8.77,
   "median_output_tokens": 1951,
   "median_output_tok_s": 235.53,
   "cost_usd_total": 0,
   "ttft_by_repeat": [
    0.49,
    0.48,
    0.49
   ]
  },
  {
   "task": "05-logic",
   "config": "qwen-on",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.14,
   "median_ttft_answer_s": 7.74,
   "median_total_s": 9.99,
   "median_output_tokens": 2343,
   "median_output_tok_s": 239.66,
   "cost_usd_total": 0,
   "ttft_by_repeat": [
    0.14,
    0.14,
    0.14
   ]
  },
  {
   "task": "06-tool-json",
   "config": "qwen-on",
   "n": 3,
   "passes": 2,
   "statuses": [
    "pass",
    "wrong:start,end",
    "pass"
   ],
   "median_ttft_s": 0.19,
   "median_ttft_answer_s": 4.74,
   "median_total_s": 5.38,
   "median_output_tokens": 1315,
   "median_output_tok_s": 252.1,
   "cost_usd_total": 0,
   "ttft_by_repeat": [
    0.19,
    0.2,
    0.18
   ]
  },
  {
   "task": "01-subtle-bug",
   "config": "sonnet-off",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.52,
   "median_ttft_answer_s": 0.52,
   "median_total_s": 4.16,
   "median_output_tokens": 625,
   "median_output_tok_s": 175.55,
   "cost_usd_total": 0.0219,
   "ttft_by_repeat": [
    0.52,
    0.51,
    0.53
   ]
  },
  {
   "task": "02-multifile-diff",
   "config": "sonnet-off",
   "n": 3,
   "passes": 1,
   "statuses": [
    "logic_right_patch_failed",
    "pass",
    "logic_right_patch_failed"
   ],
   "median_ttft_s": 1.39,
   "median_ttft_answer_s": 1.54,
   "median_total_s": 7.49,
   "median_output_tokens": 1280,
   "median_output_tok_s": 210.08,
   "cost_usd_total": 0.0647,
   "ttft_by_repeat": [
    1.54,
    1.39,
    0.94
   ]
  },
  {
   "task": "03-algorithm",
   "config": "sonnet-off",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.79,
   "median_ttft_answer_s": 0.79,
   "median_total_s": 3.91,
   "median_output_tokens": 554,
   "median_output_tok_s": 180.16,
   "cost_usd_total": 0.0185,
   "ttft_by_repeat": [
    0.79,
    0.74,
    0.87
   ]
  },
  {
   "task": "04-long-context",
   "config": "sonnet-off",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 1.45,
   "median_ttft_answer_s": 1.45,
   "median_total_s": 4.75,
   "median_output_tokens": 650,
   "median_output_tok_s": 193.38,
   "cost_usd_total": 0.0675,
   "ttft_by_repeat": [
    1.21,
    1.8,
    1.45
   ]
  },
  {
   "task": "05-logic",
   "config": "sonnet-off",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.83,
   "median_ttft_answer_s": 0.83,
   "median_total_s": 6.92,
   "median_output_tokens": 943,
   "median_output_tok_s": 157.01,
   "cost_usd_total": 0.0297,
   "ttft_by_repeat": [
    0.83,
    0.82,
    0.96
   ]
  },
  {
   "task": "06-tool-json",
   "config": "sonnet-off",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.72,
   "median_ttft_answer_s": 0.72,
   "median_total_s": 2.55,
   "median_output_tokens": 374,
   "median_output_tok_s": 209.48,
   "cost_usd_total": 0.0143,
   "ttft_by_repeat": [
    2.1,
    0.64,
    0.72
   ]
  },
  {
   "task": "01-subtle-bug",
   "config": "sonnet-adaptive",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.98,
   "median_ttft_answer_s": 0.98,
   "median_total_s": 4.54,
   "median_output_tokens": 637,
   "median_output_tok_s": 178.48,
   "cost_usd_total": 0.0212,
   "ttft_by_repeat": [
    0.87,
    1.08,
    0.98
   ]
  },
  {
   "task": "02-multifile-diff",
   "config": "sonnet-adaptive",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.57,
   "median_ttft_answer_s": 17.05,
   "median_total_s": 21.83,
   "median_output_tokens": 3662,
   "median_output_tok_s": 169.74,
   "cost_usd_total": 0.1169,
   "ttft_by_repeat": [
    0.54,
    0.57,
    0.57
   ]
  },
  {
   "task": "03-algorithm",
   "config": "sonnet-adaptive",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.59,
   "median_ttft_answer_s": 0.59,
   "median_total_s": 6.19,
   "median_output_tokens": 767,
   "median_output_tok_s": 135.83,
   "cost_usd_total": 0.026,
   "ttft_by_repeat": [
    0.7,
    0.59,
    0.55
   ]
  },
  {
   "task": "04-long-context",
   "config": "sonnet-adaptive",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.64,
   "median_ttft_answer_s": 4.49,
   "median_total_s": 6.06,
   "median_output_tokens": 993,
   "median_output_tok_s": 183.48,
   "cost_usd_total": 0.0774,
   "ttft_by_repeat": [
    0.64,
    0.67,
    0.52
   ]
  },
  {
   "task": "05-logic",
   "config": "sonnet-adaptive",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.5,
   "median_ttft_answer_s": 5.25,
   "median_total_s": 9.6,
   "median_output_tokens": 1435,
   "median_output_tok_s": 158.58,
   "cost_usd_total": 0.0438,
   "ttft_by_repeat": [
    0.61,
    0.5,
    0.49
   ]
  },
  {
   "task": "06-tool-json",
   "config": "sonnet-adaptive",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 1.03,
   "median_ttft_answer_s": 2.14,
   "median_total_s": 3.22,
   "median_output_tokens": 442,
   "median_output_tok_s": 202.47,
   "cost_usd_total": 0.0164,
   "ttft_by_repeat": [
    1.13,
    1.01,
    1.03
   ]
  },
  {
   "task": "01-subtle-bug",
   "config": "haiku-off",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.59,
   "median_ttft_answer_s": 0.59,
   "median_total_s": 3.88,
   "median_output_tokens": 837,
   "median_output_tok_s": 256.84,
   "cost_usd_total": 0.0014,
   "ttft_by_repeat": [
    0.59,
    0.57,
    0.71
   ]
  },
  {
   "task": "02-multifile-diff",
   "config": "haiku-off",
   "n": 3,
   "passes": 0,
   "statuses": [
    "logic_right_patch_failed",
    "logic_right_patch_failed",
    "logic_right_patch_failed"
   ],
   "median_ttft_s": 0.57,
   "median_ttft_answer_s": 0.57,
   "median_total_s": 3.69,
   "median_output_tokens": 1108,
   "median_output_tok_s": 367.1,
   "cost_usd_total": 0.002,
   "ttft_by_repeat": [
    0.56,
    0.57,
    0.71
   ]
  },
  {
   "task": "03-algorithm",
   "config": "haiku-off",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.78,
   "median_ttft_answer_s": 0.78,
   "median_total_s": 2.09,
   "median_output_tokens": 363,
   "median_output_tok_s": 291.84,
   "cost_usd_total": 0.0007,
   "ttft_by_repeat": [
    0.78,
    0.7,
    1.14
   ]
  },
  {
   "task": "04-long-context",
   "config": "haiku-off",
   "n": 3,
   "passes": 2,
   "statuses": [
    "3/4 correct",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.6,
   "median_ttft_answer_s": 0.6,
   "median_total_s": 3.54,
   "median_output_tokens": 813,
   "median_output_tok_s": 284.59,
   "cost_usd_total": 0.0036,
   "ttft_by_repeat": [
    0.67,
    0.6,
    0.59
   ]
  },
  {
   "task": "05-logic",
   "config": "haiku-off",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.68,
   "median_ttft_answer_s": 0.68,
   "median_total_s": 5.62,
   "median_output_tokens": 1316,
   "median_output_tok_s": 266.79,
   "cost_usd_total": 0.002,
   "ttft_by_repeat": [
    0.68,
    0.86,
    0.59
   ]
  },
  {
   "task": "06-tool-json",
   "config": "haiku-off",
   "n": 3,
   "passes": 3,
   "statuses": [
    "pass",
    "pass",
    "pass"
   ],
   "median_ttft_s": 0.58,
   "median_ttft_answer_s": 0.58,
   "median_total_s": 1.28,
   "median_output_tokens": 240,
   "median_output_tok_s": 339.23,
   "cost_usd_total": 0.0005,
   "ttft_by_repeat": [
    0.59,
    0.54,
    0.58
   ]
  }
 ],
 "runs": [
  {
   "config": "qwen-off",
   "task": "01-subtle-bug",
   "repeat": 1,
   "ttft_s": 0.18540072441101074,
   "ttft_answer_s": 0.18540072441101074,
   "total_s": 1.9510750770568848,
   "input_tokens": 293,
   "output_tokens": 436,
   "thinking_tokens": null,
   "output_tok_s": 246.94922415000312,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-off",
   "task": "01-subtle-bug",
   "repeat": 2,
   "ttft_s": 0.17954349517822266,
   "ttft_answer_s": 0.17954349517822266,
   "total_s": 2.594719171524048,
   "input_tokens": 293,
   "output_tokens": 504,
   "thinking_tokens": null,
   "output_tok_s": 208.69052537970063,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-off",
   "task": "01-subtle-bug",
   "repeat": 3,
   "ttft_s": 0.18376851081848145,
   "ttft_answer_s": 0.18376851081848145,
   "total_s": 2.202669858932495,
   "input_tokens": 293,
   "output_tokens": 438,
   "thinking_tokens": null,
   "output_tok_s": 218.31989485117782,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-off",
   "task": "02-multifile-diff",
   "repeat": 1,
   "ttft_s": 0.2984604835510254,
   "ttft_answer_s": 0.2984604835510254,
   "total_s": 3.7974369525909424,
   "input_tokens": 673,
   "output_tokens": 893,
   "thinking_tokens": null,
   "output_tok_s": 256.99372911483186,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": false,
   "grade_status": "logic_right_patch_failed"
  },
  {
   "config": "qwen-off",
   "task": "02-multifile-diff",
   "repeat": 2,
   "ttft_s": 0.22363519668579102,
   "ttft_answer_s": 0.22363519668579102,
   "total_s": 3.1279115676879883,
   "input_tokens": 673,
   "output_tokens": 751,
   "thinking_tokens": null,
   "output_tok_s": 258.59339815294084,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": false,
   "grade_status": "logic_right_patch_failed"
  },
  {
   "config": "qwen-off",
   "task": "02-multifile-diff",
   "repeat": 3,
   "ttft_s": 0.2806706428527832,
   "ttft_answer_s": 0.2806706428527832,
   "total_s": 3.164618492126465,
   "input_tokens": 673,
   "output_tokens": 783,
   "thinking_tokens": null,
   "output_tok_s": 271.513760423464,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": false,
   "grade_status": "logic_right_patch_failed"
  },
  {
   "config": "qwen-off",
   "task": "03-algorithm",
   "repeat": 1,
   "ttft_s": 0.1783754825592041,
   "ttft_answer_s": 0.1783754825592041,
   "total_s": 1.455815315246582,
   "input_tokens": 278,
   "output_tokens": 306,
   "thinking_tokens": null,
   "output_tok_s": 244.04374033941374,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-off",
   "task": "03-algorithm",
   "repeat": 2,
   "ttft_s": 0.17927098274230957,
   "ttft_answer_s": 0.17927098274230957,
   "total_s": 4.420336484909058,
   "input_tokens": 278,
   "output_tokens": 800,
   "thinking_tokens": null,
   "output_tok_s": 188.6380568360261,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-off",
   "task": "03-algorithm",
   "repeat": 3,
   "ttft_s": 0.1790318489074707,
   "ttft_answer_s": 0.1790318489074707,
   "total_s": 3.2821617126464844,
   "input_tokens": 278,
   "output_tokens": 691,
   "thinking_tokens": null,
   "output_tok_s": 224.96915020950186,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-off",
   "task": "04-long-context",
   "repeat": 1,
   "ttft_s": 1.7340481281280518,
   "ttft_answer_s": 1.7340481281280518,
   "total_s": 2.281248092651367,
   "input_tokens": 8249,
   "output_tokens": 118,
   "thinking_tokens": null,
   "output_tok_s": 215.699583878835,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": false,
   "grade_status": "3/4 correct"
  },
  {
   "config": "qwen-off",
   "task": "04-long-context",
   "repeat": 2,
   "ttft_s": 0.4945049285888672,
   "ttft_answer_s": 0.4945049285888672,
   "total_s": 1.1657311916351318,
   "input_tokens": 8249,
   "output_tokens": 133,
   "thinking_tokens": null,
   "output_tok_s": 198.16986817981427,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": false,
   "grade_status": "3/4 correct"
  },
  {
   "config": "qwen-off",
   "task": "04-long-context",
   "repeat": 3,
   "ttft_s": 0.480135440826416,
   "ttft_answer_s": 0.480135440826416,
   "total_s": 0.9851181507110596,
   "input_tokens": 8249,
   "output_tokens": 105,
   "thinking_tokens": null,
   "output_tok_s": 207.97386454273575,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": false,
   "grade_status": "3/4 correct"
  },
  {
   "config": "qwen-off",
   "task": "05-logic",
   "repeat": 1,
   "ttft_s": 0.15302538871765137,
   "ttft_answer_s": 0.15302538871765137,
   "total_s": 10.178741216659546,
   "input_tokens": 184,
   "output_tokens": 2483,
   "thinking_tokens": null,
   "output_tok_s": 247.66630011410842,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-off",
   "task": "05-logic",
   "repeat": 2,
   "ttft_s": 0.16149020195007324,
   "ttft_answer_s": 0.16149020195007324,
   "total_s": 13.314084529876709,
   "input_tokens": 184,
   "output_tokens": 3122,
   "thinking_tokens": null,
   "output_tok_s": 237.36979198257228,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-off",
   "task": "05-logic",
   "repeat": 3,
   "ttft_s": 0.14331722259521484,
   "ttft_answer_s": 0.14331722259521484,
   "total_s": 11.648086547851562,
   "input_tokens": 184,
   "output_tokens": 2687,
   "thinking_tokens": null,
   "output_tok_s": 233.5574280656947,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-off",
   "task": "06-tool-json",
   "repeat": 1,
   "ttft_s": 0.3263564109802246,
   "ttft_answer_s": 0.3263564109802246,
   "total_s": 1.0626778602600098,
   "input_tokens": 412,
   "output_tokens": 225,
   "thinking_tokens": null,
   "output_tok_s": 305.6330337624484,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-off",
   "task": "06-tool-json",
   "repeat": 2,
   "ttft_s": 0.1975407600402832,
   "ttft_answer_s": 0.1975407600402832,
   "total_s": 0.851891040802002,
   "input_tokens": 412,
   "output_tokens": 211,
   "thinking_tokens": null,
   "output_tok_s": 322.50814343880404,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-off",
   "task": "06-tool-json",
   "repeat": 3,
   "ttft_s": 0.19238972663879395,
   "ttft_answer_s": 0.19238972663879395,
   "total_s": 0.8807485103607178,
   "input_tokens": 412,
   "output_tokens": 217,
   "thinking_tokens": null,
   "output_tok_s": 315.2894296263939,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-on",
   "task": "01-subtle-bug",
   "repeat": 1,
   "ttft_s": 0.18486666679382324,
   "ttft_answer_s": 4.69279408454895,
   "total_s": 6.5125250816345215,
   "input_tokens": 291,
   "output_tokens": 1573,
   "thinking_tokens": null,
   "output_tok_s": 248.59483102542256,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-on",
   "task": "01-subtle-bug",
   "repeat": 2,
   "ttft_s": 0.17645478248596191,
   "ttft_answer_s": 4.993905067443848,
   "total_s": 7.113226413726807,
   "input_tokens": 291,
   "output_tokens": 1594,
   "thinking_tokens": null,
   "output_tok_s": 229.79350873916903,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-on",
   "task": "01-subtle-bug",
   "repeat": 3,
   "ttft_s": 0.18323564529418945,
   "ttft_answer_s": 6.150966644287109,
   "total_s": 8.563784122467041,
   "input_tokens": 291,
   "output_tokens": 1854,
   "thinking_tokens": null,
   "output_tok_s": 221.2300763149121,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-on",
   "task": "02-multifile-diff",
   "repeat": 1,
   "ttft_s": 0.21574759483337402,
   "ttft_answer_s": 19.921764135360718,
   "total_s": 22.056316137313843,
   "input_tokens": 671,
   "output_tokens": 5405,
   "thinking_tokens": null,
   "output_tok_s": 247.47704188964227,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": false,
   "grade_status": "logic_right_patch_failed"
  },
  {
   "config": "qwen-on",
   "task": "02-multifile-diff",
   "repeat": 2,
   "ttft_s": 0.23353290557861328,
   "ttft_answer_s": 36.71666884422302,
   "total_s": 39.1661651134491,
   "input_tokens": 671,
   "output_tokens": 9535,
   "thinking_tokens": null,
   "output_tok_s": 244.91111273489648,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-on",
   "task": "02-multifile-diff",
   "repeat": 3,
   "ttft_s": 0.2129974365234375,
   "ttft_answer_s": 22.161948919296265,
   "total_s": 24.769688367843628,
   "input_tokens": 671,
   "output_tokens": 5906,
   "thinking_tokens": null,
   "output_tok_s": 240.5062743622276,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": false,
   "grade_status": "logic_right_patch_failed"
  },
  {
   "config": "qwen-on",
   "task": "03-algorithm",
   "repeat": 1,
   "ttft_s": 0.18033838272094727,
   "ttft_answer_s": 20.25069522857666,
   "total_s": 22.7027690410614,
   "input_tokens": 276,
   "output_tokens": 4877,
   "thinking_tokens": null,
   "output_tok_s": 216.5410333602596,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-on",
   "task": "03-algorithm",
   "repeat": 2,
   "ttft_s": 0.17890667915344238,
   "ttft_answer_s": 15.589589834213257,
   "total_s": 18.277692794799805,
   "input_tokens": 276,
   "output_tokens": 3498,
   "thinking_tokens": null,
   "output_tok_s": 193.27382196360298,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-on",
   "task": "03-algorithm",
   "repeat": 3,
   "ttft_s": 0.17977237701416016,
   "ttft_answer_s": 14.515186071395874,
   "total_s": 15.356197357177734,
   "input_tokens": 276,
   "output_tokens": 2944,
   "thinking_tokens": null,
   "output_tok_s": 193.98674606320455,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-on",
   "task": "04-long-context",
   "repeat": 1,
   "ttft_s": 0.48853254318237305,
   "ttft_answer_s": 10.18054461479187,
   "total_s": 11.018583297729492,
   "input_tokens": 8247,
   "output_tokens": 2638,
   "thinking_tokens": null,
   "output_tok_s": 250.52476832132817,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-on",
   "task": "04-long-context",
   "repeat": 2,
   "ttft_s": 0.4825599193572998,
   "ttft_answer_s": 7.942788362503052,
   "total_s": 8.766088962554932,
   "input_tokens": 8247,
   "output_tokens": 1951,
   "thinking_tokens": null,
   "output_tok_s": 235.53197304585018,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-on",
   "task": "04-long-context",
   "repeat": 3,
   "ttft_s": 0.49333977699279785,
   "ttft_answer_s": 7.296817064285278,
   "total_s": 8.19286847114563,
   "input_tokens": 8247,
   "output_tokens": 1724,
   "thinking_tokens": null,
   "output_tok_s": 223.91362255088575,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-on",
   "task": "05-logic",
   "repeat": 1,
   "ttft_s": 0.13922953605651855,
   "ttft_answer_s": 7.74162745475769,
   "total_s": 9.994667291641235,
   "input_tokens": 182,
   "output_tokens": 2343,
   "thinking_tokens": null,
   "output_tok_s": 237.74114720791897,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-on",
   "task": "05-logic",
   "repeat": 2,
   "ttft_s": 0.14080071449279785,
   "ttft_answer_s": 6.092222690582275,
   "total_s": 8.742876529693604,
   "input_tokens": 182,
   "output_tokens": 2106,
   "thinking_tokens": null,
   "output_tok_s": 244.82829097478682,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-on",
   "task": "05-logic",
   "repeat": 3,
   "ttft_s": 0.1389937400817871,
   "ttft_answer_s": 7.821655750274658,
   "total_s": 10.407912492752075,
   "input_tokens": 182,
   "output_tokens": 2455,
   "thinking_tokens": null,
   "output_tok_s": 239.662331043794,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-on",
   "task": "06-tool-json",
   "repeat": 1,
   "ttft_s": 0.1900317668914795,
   "ttft_answer_s": 4.736461162567139,
   "total_s": 5.558318376541138,
   "input_tokens": 410,
   "output_tokens": 1318,
   "thinking_tokens": null,
   "output_tok_s": 245.52157902193625,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "qwen-on",
   "task": "06-tool-json",
   "repeat": 2,
   "ttft_s": 0.196580171585083,
   "ttft_answer_s": 4.779012680053711,
   "total_s": 5.384188413619995,
   "input_tokens": 410,
   "output_tokens": 1315,
   "thinking_tokens": null,
   "output_tok_s": 253.4957584634365,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": false,
   "grade_status": "wrong:start,end"
  },
  {
   "config": "qwen-on",
   "task": "06-tool-json",
   "repeat": 3,
   "ttft_s": 0.18473505973815918,
   "ttft_answer_s": 2.995288848876953,
   "total_s": 3.925387382507324,
   "input_tokens": 410,
   "output_tokens": 943,
   "thinking_tokens": null,
   "output_tok_s": 252.10204118184978,
   "stop": "stop",
   "cost_usd": 0.0,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-off",
   "task": "01-subtle-bug",
   "repeat": 1,
   "ttft_s": 0.5220799446105957,
   "ttft_answer_s": 0.5221116542816162,
   "total_s": 4.082315683364868,
   "input_tokens": 378,
   "output_tokens": 625,
   "thinking_tokens": 0,
   "output_tok_s": 175.55017303957734,
   "stop": "end_turn",
   "cost_usd": 0.007006,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-off",
   "task": "01-subtle-bug",
   "repeat": 2,
   "ttft_s": 0.5061113834381104,
   "ttft_answer_s": 0.5061342716217041,
   "total_s": 4.547643423080444,
   "input_tokens": 378,
   "output_tokens": 723,
   "thinking_tokens": 0,
   "output_tok_s": 178.8925568097151,
   "stop": "end_turn",
   "cost_usd": 0.007986,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-off",
   "task": "01-subtle-bug",
   "repeat": 3,
   "ttft_s": 0.5319015979766846,
   "ttft_answer_s": 0.5319221019744873,
   "total_s": 4.1649580001831055,
   "input_tokens": 378,
   "output_tokens": 618,
   "thinking_tokens": 0,
   "output_tok_s": 170.10470842805452,
   "stop": "end_turn",
   "cost_usd": 0.006936,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-off",
   "task": "02-multifile-diff",
   "repeat": 1,
   "ttft_s": 1.5411646366119385,
   "ttft_answer_s": 1.541182279586792,
   "total_s": 7.49123740196228,
   "input_tokens": 908,
   "output_tokens": 1250,
   "thinking_tokens": 0,
   "output_tok_s": 210.08146442834294,
   "stop": "end_turn",
   "cost_usd": 0.014316,
   "error": null,
   "grade_pass": false,
   "grade_status": "logic_right_patch_failed"
  },
  {
   "config": "sonnet-off",
   "task": "02-multifile-diff",
   "repeat": 2,
   "ttft_s": 1.3914673328399658,
   "ttft_answer_s": 14.714698314666748,
   "total_s": 19.245744705200195,
   "input_tokens": 908,
   "output_tokens": 3397,
   "thinking_tokens": 2311,
   "output_tok_s": 190.2625308856696,
   "stop": "end_turn",
   "cost_usd": 0.035786,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-off",
   "task": "02-multifile-diff",
   "repeat": 3,
   "ttft_s": 0.9358201026916504,
   "ttft_answer_s": 0.9358439445495605,
   "total_s": 6.921815395355225,
   "input_tokens": 908,
   "output_tokens": 1280,
   "thinking_tokens": 0,
   "output_tok_s": 213.83244346495994,
   "stop": "end_turn",
   "cost_usd": 0.014616,
   "error": null,
   "grade_pass": false,
   "grade_status": "logic_right_patch_failed"
  },
  {
   "config": "sonnet-off",
   "task": "03-algorithm",
   "repeat": 1,
   "ttft_s": 0.7877135276794434,
   "ttft_answer_s": 0.7877345085144043,
   "total_s": 3.475100040435791,
   "input_tokens": 349,
   "output_tokens": 516,
   "thinking_tokens": 0,
   "output_tok_s": 192.00810808221215,
   "stop": "end_turn",
   "cost_usd": 0.005858,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-off",
   "task": "03-algorithm",
   "repeat": 2,
   "ttft_s": 0.7398555278778076,
   "ttft_answer_s": 0.7398748397827148,
   "total_s": 3.9148802757263184,
   "input_tokens": 349,
   "output_tokens": 572,
   "thinking_tokens": 0,
   "output_tok_s": 180.1560760707783,
   "stop": "end_turn",
   "cost_usd": 0.006418,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-off",
   "task": "03-algorithm",
   "repeat": 3,
   "ttft_s": 0.8660941123962402,
   "ttft_answer_s": 0.8661143779754639,
   "total_s": 4.0112597942352295,
   "input_tokens": 349,
   "output_tokens": 554,
   "thinking_tokens": 0,
   "output_tok_s": 176.143343798688,
   "stop": "end_turn",
   "cost_usd": 0.006238,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-off",
   "task": "04-long-context",
   "repeat": 1,
   "ttft_s": 1.2090177536010742,
   "ttft_answer_s": 1.2090346813201904,
   "total_s": 4.5702056884765625,
   "input_tokens": 7968,
   "output_tokens": 650,
   "thinking_tokens": 0,
   "output_tok_s": 193.3840096400556,
   "stop": "end_turn",
   "cost_usd": 0.022436,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-off",
   "task": "04-long-context",
   "repeat": 2,
   "ttft_s": 1.797318696975708,
   "ttft_answer_s": 1.7973487377166748,
   "total_s": 4.751826524734497,
   "input_tokens": 7968,
   "output_tokens": 545,
   "thinking_tokens": 0,
   "output_tok_s": 184.46388764974859,
   "stop": "end_turn",
   "cost_usd": 0.021386,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-off",
   "task": "04-long-context",
   "repeat": 3,
   "ttft_s": 1.4537768363952637,
   "ttft_answer_s": 1.4537944793701172,
   "total_s": 5.356764793395996,
   "input_tokens": 7968,
   "output_tokens": 775,
   "thinking_tokens": 0,
   "output_tok_s": 198.5658189413303,
   "stop": "end_turn",
   "cost_usd": 0.023686,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-off",
   "task": "05-logic",
   "repeat": 1,
   "ttft_s": 0.8273336887359619,
   "ttft_answer_s": 0.8273518085479736,
   "total_s": 6.236851215362549,
   "input_tokens": 259,
   "output_tokens": 943,
   "thinking_tokens": 0,
   "output_tok_s": 174.32238556551297,
   "stop": "end_turn",
   "cost_usd": 0.009948,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-off",
   "task": "05-logic",
   "repeat": 2,
   "ttft_s": 0.8153564929962158,
   "ttft_answer_s": 0.8153805732727051,
   "total_s": 6.923057317733765,
   "input_tokens": 259,
   "output_tokens": 959,
   "thinking_tokens": 0,
   "output_tok_s": 157.01489439624095,
   "stop": "end_turn",
   "cost_usd": 0.010108,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-off",
   "task": "05-logic",
   "repeat": 3,
   "ttft_s": 0.9577305316925049,
   "ttft_answer_s": 0.9577524662017822,
   "total_s": 7.044195652008057,
   "input_tokens": 259,
   "output_tokens": 909,
   "thinking_tokens": 0,
   "output_tok_s": 149.34777116620904,
   "stop": "end_turn",
   "cost_usd": 0.009608,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-off",
   "task": "06-tool-json",
   "repeat": 1,
   "ttft_s": 2.0957627296447754,
   "ttft_answer_s": 2.0957868099212646,
   "total_s": 3.7531304359436035,
   "input_tokens": 530,
   "output_tokens": 354,
   "thinking_tokens": 0,
   "output_tok_s": 213.59170849934057,
   "stop": "end_turn",
   "cost_usd": 0.0046,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-off",
   "task": "06-tool-json",
   "repeat": 2,
   "ttft_s": 0.6397585868835449,
   "ttft_answer_s": 0.6397762298583984,
   "total_s": 2.4638333320617676,
   "input_tokens": 530,
   "output_tokens": 374,
   "thinking_tokens": 0,
   "output_tok_s": 205.03545755931071,
   "stop": "end_turn",
   "cost_usd": 0.0048,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-off",
   "task": "06-tool-json",
   "repeat": 3,
   "ttft_s": 0.7178382873535156,
   "ttft_answer_s": 0.7178559303283691,
   "total_s": 2.546144723892212,
   "input_tokens": 530,
   "output_tokens": 383,
   "thinking_tokens": 0,
   "output_tok_s": 209.48348282637235,
   "stop": "end_turn",
   "cost_usd": 0.00489,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-adaptive",
   "task": "01-subtle-bug",
   "repeat": 1,
   "ttft_s": 0.8740618228912354,
   "ttft_answer_s": 0.8740804195404053,
   "total_s": 4.336615800857544,
   "input_tokens": 380,
   "output_tokens": 618,
   "thinking_tokens": 0,
   "output_tok_s": 178.48097211844052,
   "stop": "end_turn",
   "cost_usd": 0.00694,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-adaptive",
   "task": "01-subtle-bug",
   "repeat": 2,
   "ttft_s": 1.0810775756835938,
   "ttft_answer_s": 1.081099271774292,
   "total_s": 4.705185413360596,
   "input_tokens": 380,
   "output_tokens": 638,
   "thinking_tokens": 0,
   "output_tok_s": 176.04332668228446,
   "stop": "end_turn",
   "cost_usd": 0.00714,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-adaptive",
   "task": "01-subtle-bug",
   "repeat": 3,
   "ttft_s": 0.9803318977355957,
   "ttft_answer_s": 0.9803507328033447,
   "total_s": 4.536604404449463,
   "input_tokens": 380,
   "output_tokens": 637,
   "thinking_tokens": 0,
   "output_tok_s": 179.12013176645243,
   "stop": "end_turn",
   "cost_usd": 0.00713,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-adaptive",
   "task": "02-multifile-diff",
   "repeat": 1,
   "ttft_s": 0.5389401912689209,
   "ttft_answer_s": 16.909852266311646,
   "total_s": 21.83322501182556,
   "input_tokens": 910,
   "output_tokens": 3592,
   "thinking_tokens": 2491,
   "output_tok_s": 168.68375858917923,
   "stop": "end_turn",
   "cost_usd": 0.03774,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-adaptive",
   "task": "02-multifile-diff",
   "repeat": 2,
   "ttft_s": 0.5676159858703613,
   "ttft_answer_s": 18.41523814201355,
   "total_s": 23.514769077301025,
   "input_tokens": 910,
   "output_tokens": 3895,
   "thinking_tokens": 2753,
   "output_tok_s": 169.7378312891694,
   "stop": "end_turn",
   "cost_usd": 0.04077,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-adaptive",
   "task": "02-multifile-diff",
   "repeat": 3,
   "ttft_s": 0.5711164474487305,
   "ttft_answer_s": 17.04623770713806,
   "total_s": 21.817976236343384,
   "input_tokens": 910,
   "output_tokens": 3662,
   "thinking_tokens": 2597,
   "output_tok_s": 172.35488144530706,
   "stop": "end_turn",
   "cost_usd": 0.03844,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-adaptive",
   "task": "03-algorithm",
   "repeat": 1,
   "ttft_s": 0.7043306827545166,
   "ttft_answer_s": 0.7043485641479492,
   "total_s": 7.053584337234497,
   "input_tokens": 351,
   "output_tokens": 861,
   "thinking_tokens": 0,
   "output_tok_s": 135.60648965291938,
   "stop": "end_turn",
   "cost_usd": 0.009312,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-adaptive",
   "task": "03-algorithm",
   "repeat": 2,
   "ttft_s": 0.5939743518829346,
   "ttft_answer_s": 0.5939900875091553,
   "total_s": 6.192605257034302,
   "input_tokens": 351,
   "output_tokens": 767,
   "thinking_tokens": 0,
   "output_tok_s": 136.997779098864,
   "stop": "end_turn",
   "cost_usd": 0.008372,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-adaptive",
   "task": "03-algorithm",
   "repeat": 3,
   "ttft_s": 0.546006441116333,
   "ttft_answer_s": 0.5460236072540283,
   "total_s": 6.177900314331055,
   "input_tokens": 351,
   "output_tokens": 765,
   "thinking_tokens": 0,
   "output_tok_s": 135.83352549279005,
   "stop": "end_turn",
   "cost_usd": 0.008352,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-adaptive",
   "task": "04-long-context",
   "repeat": 1,
   "ttft_s": 0.6376492977142334,
   "ttft_answer_s": 4.392914533615112,
   "total_s": 6.055891752243042,
   "input_tokens": 7970,
   "output_tokens": 1015,
   "thinking_tokens": 744,
   "output_tok_s": 187.33011830277505,
   "stop": "end_turn",
   "cost_usd": 0.02609,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-adaptive",
   "task": "04-long-context",
   "repeat": 2,
   "ttft_s": 0.6650748252868652,
   "ttft_answer_s": 4.48752236366272,
   "total_s": 6.077045202255249,
   "input_tokens": 7970,
   "output_tokens": 993,
   "thinking_tokens": 738,
   "output_tok_s": 183.48215729817935,
   "stop": "end_turn",
   "cost_usd": 0.02587,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-adaptive",
   "task": "04-long-context",
   "repeat": 3,
   "ttft_s": 0.5248754024505615,
   "ttft_answer_s": 4.76784086227417,
   "total_s": 5.9519665241241455,
   "input_tokens": 7970,
   "output_tokens": 954,
   "thinking_tokens": 753,
   "output_tok_s": 175.78477652422563,
   "stop": "end_turn",
   "cost_usd": 0.02548,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-adaptive",
   "task": "05-logic",
   "repeat": 1,
   "ttft_s": 0.6056728363037109,
   "ttft_answer_s": 5.464419603347778,
   "total_s": 9.600677967071533,
   "input_tokens": 261,
   "output_tokens": 1435,
   "thinking_tokens": 701,
   "output_tok_s": 159.53298293199606,
   "stop": "end_turn",
   "cost_usd": 0.014872,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-adaptive",
   "task": "05-logic",
   "repeat": 2,
   "ttft_s": 0.5014870166778564,
   "ttft_answer_s": 4.795050859451294,
   "total_s": 8.786690950393677,
   "input_tokens": 261,
   "output_tokens": 1305,
   "thinking_tokens": 617,
   "output_tok_s": 157.50970168512464,
   "stop": "end_turn",
   "cost_usd": 0.013572,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-adaptive",
   "task": "05-logic",
   "repeat": 3,
   "ttft_s": 0.4942023754119873,
   "ttft_answer_s": 5.247727155685425,
   "total_s": 9.82099962234497,
   "input_tokens": 261,
   "output_tokens": 1479,
   "thinking_tokens": 715,
   "output_tok_s": 158.57533522414172,
   "stop": "end_turn",
   "cost_usd": 0.015312,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-adaptive",
   "task": "06-tool-json",
   "repeat": 1,
   "ttft_s": 1.1270811557769775,
   "ttft_answer_s": 2.3089983463287354,
   "total_s": 3.3742029666900635,
   "input_tokens": 532,
   "output_tokens": 435,
   "thinking_tokens": 175,
   "output_tok_s": 193.58096115993104,
   "stop": "end_turn",
   "cost_usd": 0.005414,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-adaptive",
   "task": "06-tool-json",
   "repeat": 2,
   "ttft_s": 1.0099141597747803,
   "ttft_answer_s": 2.07578444480896,
   "total_s": 3.1693873405456543,
   "input_tokens": 532,
   "output_tokens": 442,
   "thinking_tokens": 174,
   "output_tok_s": 204.67955051991794,
   "stop": "end_turn",
   "cost_usd": 0.005484,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "sonnet-adaptive",
   "task": "06-tool-json",
   "repeat": 3,
   "ttft_s": 1.0327680110931396,
   "ttft_answer_s": 2.138043165206909,
   "total_s": 3.2157790660858154,
   "input_tokens": 532,
   "output_tokens": 442,
   "thinking_tokens": 182,
   "output_tok_s": 202.47263475332375,
   "stop": "end_turn",
   "cost_usd": 0.005484,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "haiku-off",
   "task": "01-subtle-bug",
   "repeat": 1,
   "ttft_s": 0.5877888202667236,
   "ttft_answer_s": 0.5878100395202637,
   "total_s": 3.8842315673828125,
   "input_tokens": 378,
   "output_tokens": 837,
   "thinking_tokens": 0,
   "output_tok_s": 253.91006736951644,
   "stop": "end_turn",
   "cost_usd": 0.00045630000000000003,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "haiku-off",
   "task": "01-subtle-bug",
   "repeat": 2,
   "ttft_s": 0.5743598937988281,
   "ttft_answer_s": 0.5743775367736816,
   "total_s": 3.926607847213745,
   "input_tokens": 378,
   "output_tokens": 861,
   "thinking_tokens": 0,
   "output_tok_s": 256.84257607582515,
   "stop": "end_turn",
   "cost_usd": 0.0004683,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "haiku-off",
   "task": "01-subtle-bug",
   "repeat": 3,
   "ttft_s": 0.7115871906280518,
   "ttft_answer_s": 0.71160888671875,
   "total_s": 3.5977842807769775,
   "input_tokens": 378,
   "output_tokens": 800,
   "thinking_tokens": 0,
   "output_tok_s": 277.18134798574016,
   "stop": "end_turn",
   "cost_usd": 0.0004378,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "haiku-off",
   "task": "02-multifile-diff",
   "repeat": 1,
   "ttft_s": 0.5593271255493164,
   "ttft_answer_s": 0.5593559741973877,
   "total_s": 3.845219612121582,
   "input_tokens": 908,
   "output_tokens": 1188,
   "thinking_tokens": 0,
   "output_tok_s": 361.5456089493915,
   "stop": "end_turn",
   "cost_usd": 0.0006848,
   "error": null,
   "grade_pass": false,
   "grade_status": "logic_right_patch_failed"
  },
  {
   "config": "haiku-off",
   "task": "02-multifile-diff",
   "repeat": 2,
   "ttft_s": 0.5726172924041748,
   "ttft_answer_s": 0.5726351737976074,
   "total_s": 3.5799646377563477,
   "input_tokens": 908,
   "output_tokens": 1104,
   "thinking_tokens": 0,
   "output_tok_s": 367.1009275686833,
   "stop": "end_turn",
   "cost_usd": 0.0006427999999999999,
   "error": null,
   "grade_pass": false,
   "grade_status": "logic_right_patch_failed"
  },
  {
   "config": "haiku-off",
   "task": "02-multifile-diff",
   "repeat": 3,
   "ttft_s": 0.7075896263122559,
   "ttft_answer_s": 0.7076098918914795,
   "total_s": 3.6903951168060303,
   "input_tokens": 908,
   "output_tokens": 1108,
   "thinking_tokens": 0,
   "output_tok_s": 371.4623710903058,
   "stop": "end_turn",
   "cost_usd": 0.0006448,
   "error": null,
   "grade_pass": false,
   "grade_status": "logic_right_patch_failed"
  },
  {
   "config": "haiku-off",
   "task": "03-algorithm",
   "repeat": 1,
   "ttft_s": 0.7815327644348145,
   "ttft_answer_s": 0.7815556526184082,
   "total_s": 2.08864164352417,
   "input_tokens": 349,
   "output_tokens": 374,
   "thinking_tokens": 0,
   "output_tok_s": 286.1276562213858,
   "stop": "end_turn",
   "cost_usd": 0.0002219,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "haiku-off",
   "task": "03-algorithm",
   "repeat": 2,
   "ttft_s": 0.7033147811889648,
   "ttft_answer_s": 0.703331470489502,
   "total_s": 1.9471635818481445,
   "input_tokens": 349,
   "output_tokens": 363,
   "thinking_tokens": 0,
   "output_tok_s": 291.8361136881167,
   "stop": "end_turn",
   "cost_usd": 0.0002164,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "haiku-off",
   "task": "03-algorithm",
   "repeat": 3,
   "ttft_s": 1.1365857124328613,
   "ttft_answer_s": 1.136603593826294,
   "total_s": 2.340811252593994,
   "input_tokens": 349,
   "output_tokens": 363,
   "thinking_tokens": 0,
   "output_tok_s": 301.4385494194288,
   "stop": "end_turn",
   "cost_usd": 0.0002164,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "haiku-off",
   "task": "04-long-context",
   "repeat": 1,
   "ttft_s": 0.6734268665313721,
   "ttft_answer_s": 0.6734440326690674,
   "total_s": 3.8059372901916504,
   "input_tokens": 7968,
   "output_tokens": 813,
   "thinking_tokens": 0,
   "output_tok_s": 259.53624730481346,
   "stop": "end_turn",
   "cost_usd": 0.0012033000000000002,
   "error": null,
   "grade_pass": false,
   "grade_status": "3/4 correct"
  },
  {
   "config": "haiku-off",
   "task": "04-long-context",
   "repeat": 2,
   "ttft_s": 0.6027171611785889,
   "ttft_answer_s": 0.6027348041534424,
   "total_s": 3.307250738143921,
   "input_tokens": 7968,
   "output_tokens": 793,
   "thinking_tokens": 0,
   "output_tok_s": 293.211371733014,
   "stop": "end_turn",
   "cost_usd": 0.0011933000000000002,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "haiku-off",
   "task": "04-long-context",
   "repeat": 3,
   "ttft_s": 0.5913746356964111,
   "ttft_answer_s": 0.5914027690887451,
   "total_s": 3.5359578132629395,
   "input_tokens": 7968,
   "output_tokens": 838,
   "thinking_tokens": 0,
   "output_tok_s": 284.59036456648596,
   "stop": "end_turn",
   "cost_usd": 0.0012158000000000002,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "haiku-off",
   "task": "05-logic",
   "repeat": 1,
   "ttft_s": 0.675121545791626,
   "ttft_answer_s": 0.6751511096954346,
   "total_s": 5.622869253158569,
   "input_tokens": 259,
   "output_tokens": 1320,
   "thinking_tokens": 0,
   "output_tok_s": 266.7880575306189,
   "stop": "end_turn",
   "cost_usd": 0.0006859,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "haiku-off",
   "task": "05-logic",
   "repeat": 2,
   "ttft_s": 0.8615741729736328,
   "ttft_answer_s": 0.8615922927856445,
   "total_s": 5.8494203090667725,
   "input_tokens": 259,
   "output_tokens": 1316,
   "thinking_tokens": 0,
   "output_tok_s": 263.841338343847,
   "stop": "end_turn",
   "cost_usd": 0.0006839,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "haiku-off",
   "task": "05-logic",
   "repeat": 3,
   "ttft_s": 0.5948600769042969,
   "ttft_answer_s": 0.5948774814605713,
   "total_s": 5.2857911586761475,
   "input_tokens": 259,
   "output_tokens": 1298,
   "thinking_tokens": 0,
   "output_tok_s": 276.7041291746545,
   "stop": "end_turn",
   "cost_usd": 0.0006749,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "haiku-off",
   "task": "06-tool-json",
   "repeat": 1,
   "ttft_s": 0.5851659774780273,
   "ttft_answer_s": 0.5851879119873047,
   "total_s": 1.3583664894104004,
   "input_tokens": 530,
   "output_tokens": 249,
   "thinking_tokens": 0,
   "output_tok_s": 322.03806924248187,
   "stop": "end_turn",
   "cost_usd": 0.0001775,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "haiku-off",
   "task": "06-tool-json",
   "repeat": 2,
   "ttft_s": 0.5352833271026611,
   "ttft_answer_s": 0.5353059768676758,
   "total_s": 1.237555980682373,
   "input_tokens": 530,
   "output_tokens": 240,
   "thinking_tokens": 0,
   "output_tok_s": 341.74760867683233,
   "stop": "end_turn",
   "cost_usd": 0.000173,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  },
  {
   "config": "haiku-off",
   "task": "06-tool-json",
   "repeat": 3,
   "ttft_s": 0.5750689506530762,
   "ttft_answer_s": 0.575087308883667,
   "total_s": 1.276653528213501,
   "input_tokens": 530,
   "output_tokens": 238,
   "thinking_tokens": 0,
   "output_tok_s": 339.2320863545521,
   "stop": "end_turn",
   "cost_usd": 0.000172,
   "error": null,
   "grade_pass": true,
   "grade_status": "pass"
  }
 ]
}