{ "params": { "bootstrap_draws": 10000, "seed": 1234, "ci": "95%" }, "model": [ "claude-sonnet-4-6" ], "claude_versions": [ "2.1.206 (Claude Code)" ], "tasks": [ "code-bugfix-py", "code-comprehension-django", "code-debug-cascade-py", "code-debug-ledger-py", "code-debug-pipeline-py", "code-feature-js", "code-feature-validate-py", "code-iterate-tests", "code-migration-py", "code-migration-py-xl", "code-overview-cobra", "code-refactor-split-py", "code-settings-inventory-django", "log-needle-zh", "report-pdf", "seo-audit", "web-research-oss-inventory" ], "task_control_tokens": { "code-bugfix-py": 116597, "code-comprehension-django": 95346, "code-debug-cascade-py": 689671, "code-debug-ledger-py": 373018, "code-debug-pipeline-py": 463241, "code-feature-js": 193747, "code-feature-validate-py": 240760, "code-iterate-tests": 146833, "code-migration-py": 169042, "code-migration-py-xl": 150740, "code-overview-cobra": 213742, "code-refactor-split-py": 145496, "code-settings-inventory-django": 412738, "log-needle-zh": 101135, "report-pdf": 197564, "seo-audit": 161533, "web-research-oss-inventory": 1253740 }, "headline_min_tokens": 200000, "headline_n_tasks": 7, "headline": [ { "competitor": "tokenade", "cost_ratio": 0.6107890918361051, "cost_reduction_pct": 38.9, "ci_lo_pct": 22.3, "ci_hi_pct": 53.2, "adoption": 5, "n_runs": 70, "tasks_compared": 7, "no_adoption": false, "tokens": { "input": 32895, "output": 8102, "cache": 368853 } }, { "competitor": "edgee", "cost_ratio": 0.8525732643594099, "cost_reduction_pct": 14.7, "ci_lo_pct": -11.1, "ci_hi_pct": 37.1, "adoption": 0, "n_runs": 70, "tasks_compared": 7, "no_adoption": false, "tokens": { "input": null, "output": null, "cache": null } }, { "competitor": "caveman", "cost_ratio": 0.8635234221969219, "cost_reduction_pct": 13.6, "ci_lo_pct": -3.6, "ci_hi_pct": 30.2, "adoption": 0, "n_runs": 70, "tasks_compared": 7, "no_adoption": false, "tokens": { "input": 31562, "output": 8152, "cache": 438883 } }, { "competitor": "claude-token-efficient", "cost_ratio": 0.8836469088131867, "cost_reduction_pct": 11.6, "ci_lo_pct": -1.4, "ci_hi_pct": 24.2, "adoption": 0, "n_runs": 70, "tasks_compared": 7, "no_adoption": false, "tokens": { "input": 32956, "output": 9576, "cache": 444948 } }, { "competitor": "code-review-graph", "cost_ratio": 0.9151766471570281, "cost_reduction_pct": 8.5, "ci_lo_pct": -13.6, "ci_hi_pct": 26.3, "adoption": 0, "n_runs": 70, "tasks_compared": 7, "no_adoption": true, "tokens": { "input": 34507, "output": 10128, "cache": 459802 } }, { "competitor": "codegraph", "cost_ratio": 0.9243248360269913, "cost_reduction_pct": 7.6, "ci_lo_pct": -8.7, "ci_hi_pct": 21.6, "adoption": 4, "n_runs": 70, "tasks_compared": 7, "no_adoption": false, "tokens": { "input": 35312, "output": 10277, "cache": 442697 } }, { "competitor": "squeez", "cost_ratio": 0.9306371790348722, "cost_reduction_pct": 6.9, "ci_lo_pct": -18.4, "ci_hi_pct": 26.1, "adoption": 0, "n_runs": 70, "tasks_compared": 7, "no_adoption": false, "tokens": { "input": 31331, "output": 8634, "cache": 512527 } }, { "competitor": "ponytail", "cost_ratio": 0.9648070827312065, "cost_reduction_pct": 3.5, "ci_lo_pct": -12.4, "ci_hi_pct": 18.1, "adoption": 0, "n_runs": 70, "tasks_compared": 7, "no_adoption": false, "tokens": { "input": 34646, "output": 9646, "cache": 488468 } }, { "competitor": "graphify", "cost_ratio": 0.9671176882309961, "cost_reduction_pct": 3.3, "ci_lo_pct": -12.2, "ci_hi_pct": 16.7, "adoption": 0, "n_runs": 70, "tasks_compared": 7, "no_adoption": true, "tokens": { "input": 35075, "output": 10266, "cache": 489272 } }, { "competitor": "control", "cost_ratio": 1.0, "cost_reduction_pct": 0.0, "ci_lo_pct": null, "ci_hi_pct": null, "adoption": null, "n_runs": 70, "tasks_compared": 7, "tokens": { "input": 34530, "output": 11080, "cache": 471813 } }, { "competitor": "rtk", "cost_ratio": 1.0706952639327463, "cost_reduction_pct": -7.1, "ci_lo_pct": -26.0, "ci_hi_pct": 7.2, "adoption": 0, "n_runs": 70, "tasks_compared": 7, "no_adoption": false, "tokens": { "input": 35574, "output": 12302, "cache": 542739 } }, { "competitor": "lean-ctx", "cost_ratio": 1.1025453613092588, "cost_reduction_pct": -10.3, "ci_lo_pct": -46.4, "ci_hi_pct": 12.6, "adoption": 60, "n_runs": 70, "tasks_compared": 7, "no_adoption": false, "tokens": { "input": 35689, "output": 10937, "cache": 611410 } }, { "competitor": "headroom", "cost_ratio": 1.528455659507268, "cost_reduction_pct": -52.8, "ci_lo_pct": -120.2, "ci_hi_pct": -10.2, "adoption": 0, "n_runs": 70, "tasks_compared": 7, "no_adoption": false, "tokens": { "input": 244493, "output": 11574, "cache": 471049 } } ], "adoption_na": [ "caveman", "claude-token-efficient", "edgee", "headroom", "ponytail", "rtk", "squeez" ], "no_adoption": [ "code-review-graph", "graphify" ], "opt_in_adoption": [ "code-review-graph", "codegraph", "graphify" ], "no_token_accounting": [ "edgee" ], "token_bands": [ { "label": "0\u2013200k", "lo": 0, "hi": 200000, "tasks": [ "code-bugfix-py", "code-comprehension-django", "code-feature-js", "code-iterate-tests", "code-migration-py", "code-migration-py-xl", "code-refactor-split-py", "log-needle-zh", "report-pdf", "seo-audit" ], "n_tasks": 10, "ranking": [ { "competitor": "tokenade", "aggregate_cost_ratio": 0.9012262027025272, "no_adoption": false, "tasks_compared": 10, "tokens": { "input": 13041, "output": 2534, "cache": 136993 } }, { "competitor": "ponytail", "aggregate_cost_ratio": 0.9526162121167375, "no_adoption": false, "tasks_compared": 10, "tokens": { "input": 13998, "output": 2351, "cache": 142330 } }, { "competitor": "control", "aggregate_cost_ratio": 1.0, "tasks_compared": 10, "tokens": { "input": 12976, "output": 2928, "cache": 131891 } }, { "competitor": "claude-token-efficient", "aggregate_cost_ratio": 1.0033935267407126, "no_adoption": false, "tasks_compared": 10, "tokens": { "input": 15055, "output": 3140, "cache": 139409 } }, { "competitor": "graphify", "aggregate_cost_ratio": 1.0083886228578394, "no_adoption": true, "tasks_compared": 10, "tokens": { "input": 14783, "output": 3259, "cache": 146437 } }, { "competitor": "rtk", "aggregate_cost_ratio": 1.0393643635988516, "no_adoption": false, "tasks_compared": 10, "tokens": { "input": 14610, "output": 3245, "cache": 161338 } }, { "competitor": "squeez", "aggregate_cost_ratio": 1.070136520941322, "no_adoption": false, "tasks_compared": 10, "tokens": { "input": 16083, "output": 2940, "cache": 184699 } }, { "competitor": "caveman", "aggregate_cost_ratio": 1.0720854165755143, "no_adoption": false, "tasks_compared": 10, "tokens": { "input": 15625, "output": 2713, "cache": 149558 } }, { "competitor": "code-review-graph", "aggregate_cost_ratio": 1.0846191063775785, "no_adoption": true, "tasks_compared": 10, "tokens": { "input": 15114, "output": 3155, "cache": 144030 } }, { "competitor": "edgee", "aggregate_cost_ratio": 1.135770279646499, "no_adoption": false, "tasks_compared": 10, "tokens": { "input": null, "output": null, "cache": null } }, { "competitor": "codegraph", "aggregate_cost_ratio": 1.1469219237211266, "no_adoption": false, "tasks_compared": 10, "tokens": { "input": 16121, "output": 3173, "cache": 137430 } }, { "competitor": "lean-ctx", "aggregate_cost_ratio": 1.3357379240937846, "no_adoption": false, "tasks_compared": 10, "tokens": { "input": 17505, "output": 3925, "cache": 238764 } }, { "competitor": "headroom", "aggregate_cost_ratio": 1.5771221294693907, "no_adoption": false, "tasks_compared": 10, "tokens": { "input": 29010, "output": 4129, "cache": 200979 } } ] }, { "label": "200k\u2013400k", "lo": 200000, "hi": 400000, "tasks": [ "code-debug-ledger-py", "code-feature-validate-py", "code-overview-cobra" ], "n_tasks": 3, "ranking": [ { "competitor": "tokenade", "aggregate_cost_ratio": 0.6383296752828613, "no_adoption": false, "tasks_compared": 3, "tokens": { "input": 16319, "output": 3080, "cache": 183028 } }, { "competitor": "caveman", "aggregate_cost_ratio": 0.7709163395558996, "no_adoption": false, "tasks_compared": 3, "tokens": { "input": 15320, "output": 2927, "cache": 203656 } }, { "competitor": "edgee", "aggregate_cost_ratio": 0.8005104726919906, "no_adoption": false, "tasks_compared": 3, "tokens": { "input": null, "output": null, "cache": null } }, { "competitor": "claude-token-efficient", "aggregate_cost_ratio": 0.8203417602599249, "no_adoption": false, "tasks_compared": 3, "tokens": { "input": 18177, "output": 4541, "cache": 264147 } }, { "competitor": "code-review-graph", "aggregate_cost_ratio": 0.8787323096271802, "no_adoption": true, "tasks_compared": 3, "tokens": { "input": 19770, "output": 5659, "cache": 261586 } }, { "competitor": "codegraph", "aggregate_cost_ratio": 0.8822513791170323, "no_adoption": false, "tasks_compared": 3, "tokens": { "input": 20686, "output": 5757, "cache": 243390 } }, { "competitor": "graphify", "aggregate_cost_ratio": 0.9158933996996147, "no_adoption": true, "tasks_compared": 3, "tokens": { "input": 19438, "output": 5668, "cache": 272743 } }, { "competitor": "ponytail", "aggregate_cost_ratio": 0.9881065300678216, "no_adoption": false, "tasks_compared": 3, "tokens": { "input": 20816, "output": 5454, "cache": 317760 } }, { "competitor": "control", "aggregate_cost_ratio": 1.0, "tasks_compared": 3, "tokens": { "input": 19163, "output": 6247, "cache": 250421 } }, { "competitor": "squeez", "aggregate_cost_ratio": 1.0422942839886178, "no_adoption": false, "tasks_compared": 3, "tokens": { "input": 18948, "output": 4879, "cache": 360669 } }, { "competitor": "lean-ctx", "aggregate_cost_ratio": 1.0461083743082142, "no_adoption": false, "tasks_compared": 3, "tokens": { "input": 22082, "output": 5076, "cache": 392014 } }, { "competitor": "rtk", "aggregate_cost_ratio": 1.080004918267092, "no_adoption": false, "tasks_compared": 3, "tokens": { "input": 20760, "output": 7313, "cache": 349710 } }, { "competitor": "headroom", "aggregate_cost_ratio": 1.1687990938658137, "no_adoption": false, "tasks_compared": 3, "tokens": { "input": 28254, "output": 4474, "cache": 285439 } } ] }, { "label": "400k+", "lo": 400000, "hi": null, "tasks": [ "code-debug-cascade-py", "code-debug-pipeline-py", "code-settings-inventory-django", "web-research-oss-inventory" ], "n_tasks": 4, "ranking": [ { "competitor": "tokenade", "aggregate_cost_ratio": 0.5909162736512314, "no_adoption": false, "tasks_compared": 4, "tokens": { "input": 45327, "output": 11870, "cache": 508223 } }, { "competitor": "squeez", "aggregate_cost_ratio": 0.8548165670815981, "no_adoption": false, "tasks_compared": 4, "tokens": { "input": 40619, "output": 11451, "cache": 626419 } }, { "competitor": "edgee", "aggregate_cost_ratio": 0.8938306201733583, "no_adoption": false, "tasks_compared": 4, "tokens": { "input": null, "output": null, "cache": null } }, { "competitor": "claude-token-efficient", "aggregate_cost_ratio": 0.9343115534129184, "no_adoption": false, "tasks_compared": 4, "tokens": { "input": 44040, "output": 13352, "cache": 580548 } }, { "competitor": "caveman", "aggregate_cost_ratio": 0.9402087211509262, "no_adoption": false, "tasks_compared": 4, "tokens": { "input": 43743, "output": 12071, "cache": 615303 } }, { "competitor": "code-review-graph", "aggregate_cost_ratio": 0.9434984270422698, "no_adoption": true, "tasks_compared": 4, "tokens": { "input": 45561, "output": 13479, "cache": 608464 } }, { "competitor": "ponytail", "aggregate_cost_ratio": 0.9476937537347878, "no_adoption": false, "tasks_compared": 4, "tokens": { "input": 45019, "output": 12789, "cache": 616499 } }, { "competitor": "codegraph", "aggregate_cost_ratio": 0.9571914921271453, "no_adoption": false, "tasks_compared": 4, "tokens": { "input": 46282, "output": 13668, "cache": 592176 } }, { "competitor": "control", "aggregate_cost_ratio": 1.0, "tasks_compared": 4, "tokens": { "input": 46055, "output": 14704, "cache": 637858 } }, { "competitor": "graphify", "aggregate_cost_ratio": 1.0074073712624922, "no_adoption": true, "tasks_compared": 4, "tokens": { "input": 46803, "output": 13715, "cache": 651669 } }, { "competitor": "rtk", "aggregate_cost_ratio": 1.0637657247327694, "no_adoption": false, "tasks_compared": 4, "tokens": { "input": 46685, "output": 16044, "cache": 687510 } }, { "competitor": "lean-ctx", "aggregate_cost_ratio": 1.1468623804056592, "no_adoption": false, "tasks_compared": 4, "tokens": { "input": 45894, "output": 15334, "cache": 775958 } }, { "competitor": "headroom", "aggregate_cost_ratio": 1.8691216931333225, "no_adoption": false, "tasks_compared": 4, "tokens": { "input": 406672, "output": 16900, "cache": 610256 } } ] } ], "control_per_task": { "code-bugfix-py": { "n": 10, "mean_input_tokens": 8210, "mean_output_tokens": 687, "mean_cache_tokens": 107694 }, "code-comprehension-django": { "n": 10, "mean_input_tokens": 13124, "mean_output_tokens": 2387, "mean_cache_tokens": 79829 }, "code-debug-cascade-py": { "n": 10, "mean_input_tokens": 34480, "mean_output_tokens": 17680, "mean_cache_tokens": 637493 }, "code-debug-ledger-py": { "n": 10, "mean_input_tokens": 22458, "mean_output_tokens": 10501, "mean_cache_tokens": 340047 }, "code-debug-pipeline-py": { "n": 10, "mean_input_tokens": 26606, "mean_output_tokens": 6824, "mean_cache_tokens": 429798 }, "code-feature-js": { "n": 10, "mean_input_tokens": 9468, "mean_output_tokens": 1946, "mean_cache_tokens": 182324 }, "code-feature-validate-py": { "n": 10, "mean_input_tokens": 16952, "mean_output_tokens": 4786, "mean_cache_tokens": 219013 }, "code-iterate-tests": { "n": 10, "mean_input_tokens": 10487, "mean_output_tokens": 1658, "mean_cache_tokens": 134682 }, "code-migration-py": { "n": 10, "mean_input_tokens": 17041, "mean_output_tokens": 2509, "mean_cache_tokens": 149470 }, "code-migration-py-xl": { "n": 10, "mean_input_tokens": 13538, "mean_output_tokens": 1650, "mean_cache_tokens": 135546 }, "code-overview-cobra": { "n": 10, "mean_input_tokens": 18078, "mean_output_tokens": 3454, "mean_cache_tokens": 192202 }, "code-refactor-split-py": { "n": 10, "mean_input_tokens": 9304, "mean_output_tokens": 1490, "mean_cache_tokens": 134695 }, "code-settings-inventory-django": { "n": 10, "mean_input_tokens": 20889, "mean_output_tokens": 7244, "mean_cache_tokens": 384590 }, "log-needle-zh": { "n": 10, "mean_input_tokens": 12010, "mean_output_tokens": 1586, "mean_cache_tokens": 87535 }, "report-pdf": { "n": 10, "mean_input_tokens": 17559, "mean_output_tokens": 8030, "mean_cache_tokens": 171967 }, "seo-audit": { "n": 10, "mean_input_tokens": 19020, "mean_output_tokens": 7337, "mean_cache_tokens": 135169 }, "web-research-oss-inventory": { "n": 10, "mean_input_tokens": 102246, "mean_output_tokens": 27069, "mean_cache_tokens": 1099548 } }, "control_noise_floor": { "code-bugfix-py": { "n": 10, "successes": 10, "mean_cost": 0.09189006, "cv": 0.01696953154728839, "costs": [ 0.0926382, 0.0932049, 0.0901737, 0.0918696, 0.093567, 0.09009389999999999, 0.09355979999999998, 0.0933687, 0.09015329999999999, 0.09027149999999999 ] }, "code-comprehension-django": { "n": 10, "successes": 10, "mean_cost": 0.38451606, "cv": 0.41985957468688473, "costs": [ 0.3418983000000001, 0.1870887, 0.5949943499999999, 0.4467025499999999, 0.46477319999999994, 0.47628555000000006, 0.3978163500000001, 0.1554645, 0.19252650000000002, 0.5876106 ] }, "code-debug-cascade-py": { "n": 10, "successes": 10, "mean_cost": 0.66338619, "cv": 0.5727310483770449, "costs": [ 0.32487629999999995, 0.9427632, 0.3787844999999999, 0.8173254, 0.3629577, 0.3723108, 0.7240217999999999, 0.37599899999999997, 1.5083904000000004, 0.8264328 ] }, "code-debug-ledger-py": { "n": 10, "successes": 10, "mean_cost": 0.39431511, "cv": 0.39146529712031847, "costs": [ 0.22760820000000007, 0.2654184, 0.48324330000000004, 0.2345367, 0.5547789000000002, 0.42272429999999994, 0.5300760000000001, 0.38376, 0.6327906, 0.2082147 ] }, "code-debug-pipeline-py": { "n": 10, "successes": 10, "mean_cost": 0.39097032000000004, "cv": 0.13518357189555064, "costs": [ 0.4046517, 0.2816232000000001, 0.3922602, 0.3582336, 0.4086516, 0.47352330000000004, 0.39411090000000004, 0.41385630000000007, 0.4375578000000001, 0.34523460000000006 ] }, "code-feature-js": { "n": 10, "successes": 10, "mean_cost": 0.14072904, "cv": 0.17261778326619898, "costs": [ 0.1300488, 0.12602850000000002, 0.15055680000000002, 0.1904589, 0.1472892, 0.1497228, 0.1514016, 0.09795000000000001, 0.12222000000000001, 0.1416138 ] }, "code-feature-validate-py": { "n": 10, "successes": 10, "mean_cost": 0.23922906, "cv": 0.20059197621835678, "costs": [ 0.19933620000000002, 0.3056985, 0.2075493, 0.206835, 0.294381, 0.19167209999999998, 0.2560575, 0.31173120000000004, 0.21468749999999998, 0.20434229999999998 ] }, "code-iterate-tests": { "n": 10, "successes": 10, "mean_cost": 0.1282083, "cv": 0.013344194861575781, "costs": [ 0.12733229999999998, 0.12828359999999997, 0.1277535, 0.1273119, 0.1324422, 0.1270866, 0.1269198, 0.1298832, 0.1275723, 0.1274976 ] }, "code-migration-py": { "n": 10, "successes": 10, "mean_cost": 0.18478824, "cv": 0.5555406985874879, "costs": [ 0.1404966, 0.1291977, 0.1178133, 0.1298346, 0.15145530000000001, 0.2987097, 0.43773150000000005, 0.1513521, 0.1499781, 0.14131349999999998 ] }, "code-migration-py-xl": { "n": 10, "successes": 10, "mean_cost": 0.14666625, "cv": 0.08938586658868856, "costs": [ 0.12716999999999998, 0.15390360000000003, 0.12717420000000002, 0.1503201, 0.1572369, 0.161373, 0.1518447, 0.1560783, 0.1509873, 0.1305744 ] }, "code-overview-cobra": { "n": 10, "successes": 10, "mean_cost": 0.32157825, "cv": 0.47167518826230814, "costs": [ 0.699654, 0.2155926, 0.2131287, 0.2668779, 0.2507391, 0.38338140000000004, 0.22885049999999996, 0.43585260000000003, 0.2493414, 0.27236429999999995 ] }, "code-refactor-split-py": { "n": 10, "successes": 10, "mean_cost": 0.11860302, "cv": 0.07176569750624245, "costs": [ 0.1075338, 0.1062132, 0.120045, 0.1167378, 0.1345014, 0.115821, 0.1285155, 0.11650529999999999, 0.118869, 0.1212882 ] }, "code-settings-inventory-django": { "n": 10, "successes": 10, "mean_cost": 0.34941159000000005, "cv": 0.4623678895934021, "costs": [ 0.1940286, 0.2714727, 0.22516680000000003, 0.3051099, 0.5701863000000001, 0.3855090000000001, 0.3509322, 0.2796333, 0.22190940000000003, 0.6901676999999999 ] }, "log-needle-zh": { "n": 10, "successes": 10, "mean_cost": 0.12212064, "cv": 0.07799471587647969, "costs": [ 0.11245680000000002, 0.13527660000000002, 0.1169256, 0.1177452, 0.1127613, 0.1163544, 0.1196064, 0.12207509999999999, 0.12713370000000002, 0.1408713 ] }, "report-pdf": { "n": 10, "successes": 10, "mean_cost": 0.27741402, "cv": 0.26894396373746016, "costs": [ 0.19520999999999997, 0.3669651, 0.21364499999999997, 0.2180355, 0.23841870000000004, 0.43272000000000005, 0.27071520000000004, 0.29468849999999996, 0.2428557, 0.3008865 ] }, "seo-audit": { "n": 10, "successes": 10, "mean_cost": 0.26475024, "cv": 0.06573240834286322, "costs": [ 0.2624034, 0.30297090000000004, 0.2522505, 0.2673954, 0.2672658, 0.2577852, 0.2510364, 0.2796132, 0.26706959999999996, 0.23971199999999998 ] }, "web-research-oss-inventory": { "n": 10, "successes": 10, "mean_cost": 5.006345, "cv": 0.13344237093088737, "costs": [ 4.6242195, 5.081650299999999, 4.5112307, 4.071590799999999, 5.528334699999999, 6.523181200000001, 5.2613541, 4.8700748, 4.7772955999999995, 4.8145183000000005 ] } }, "competitors": { "caveman": { "version": "v1.9.0", "aggregate_cost_ratio": 0.9807129853494566, "aggregate_ci_lo": 0.8613692671665824, "aggregate_ci_hi": 1.1113763247130377, "n_success": 170, "n_runs": 170, "tasks_compared": 17, "per_task": { "code-bugfix-py": { "n": 10, "successes": 10, "mean_cost": 0.10468665, "ratio": 1.1392597850083024, "ci_lo": 1.1162107452573122, "ci_hi": 1.170495743667926, "adoption": 0, "mean_wall_s": 16.9471, "mean_input_tokens": 10034, "mean_output_tokens": 612, "mean_cache_tokens": 117638, "costs": [ 0.10313130000000001, 0.1032054, 0.1030092, 0.1030761, 0.1030113, 0.11569860000000001, 0.10300799999999999, 0.1063875, 0.10326779999999999, 0.1030713 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 4, 4, 4, 4, 4, 5, 4, 5, 4, 4 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-comprehension-django": { "n": 10, "successes": 10, "mean_cost": 0.281312685, "ratio": 0.7316019128043703, "ci_lo": 0.5539290115792377, "ci_hi": 1.0026167409708338, "adoption": 0, "mean_wall_s": 48.0276, "mean_input_tokens": 16902, "mean_output_tokens": 2348, "mean_cache_tokens": 84260, "costs": [ 0.34193049999999997, 0.32361210000000007, 0.28488785, 0.1548825, 0.32547135000000005, 0.35066305, 0.1860147, 0.34233709999999995, 0.1838481, 0.31947960000000003 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 29, 32, 29, 12, 26, 31, 16, 30, 16, 27 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-cascade-py": { "n": 10, "successes": 10, "mean_cost": 0.56869116, "ratio": 0.8572550477723994, "ci_lo": 0.5453177981851504, "ci_hi": 1.3362096251151774, "adoption": 0, "mean_wall_s": 191.1069, "mean_input_tokens": 30836, "mean_output_tokens": 12151, "mean_cache_tokens": 671212, "costs": [ 0.35634450000000006, 0.34997849999999997, 0.8515446000000001, 0.9173217000000002, 0.9804332999999998, 0.3963891, 0.3384765, 0.8163183, 0.2854407, 0.3946644 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 24, 25, 34, 36, 38, 27, 25, 35, 26, 26 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-ledger-py": { "n": 10, "successes": 10, "mean_cost": 0.20416775999999998, "ratio": 0.5177781799941675, "ci_lo": 0.41663838742878506, "ci_hi": 0.671588482756322, "adoption": 0, "mean_wall_s": 45.484, "mean_input_tokens": 15290, "mean_output_tokens": 2505, "mean_cache_tokens": 249393, "costs": [ 0.21732269999999998, 0.22642410000000002, 0.1951068, 0.21269999999999997, 0.1656405, 0.1711326, 0.21972419999999998, 0.19629539999999998, 0.2226606, 0.2146707 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 13, 14, 13, 13, 14, 13, 14, 13, 14, 13 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-pipeline-py": { "n": 10, "successes": 10, "mean_cost": 0.37000302, "ratio": 0.9463711209587469, "ci_lo": 0.8355214309023065, "ci_hi": 1.0654432755458982, "adoption": 0, "mean_wall_s": 89.8299, "mean_input_tokens": 26224, "mean_output_tokens": 5400, "mean_cache_tokens": 438755, "costs": [ 0.4133559000000001, 0.3696204, 0.3646884, 0.41261580000000003, 0.36235560000000006, 0.29344410000000004, 0.25956840000000003, 0.36797159999999995, 0.4213944, 0.4350156 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 20, 23, 23, 24, 23, 23, 21, 23, 26, 23 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-js": { "n": 10, "successes": 10, "mean_cost": 0.21695678999999998, "ratio": 1.5416632558567869, "ci_lo": 1.2466621870516768, "ci_hi": 1.873510169559377, "adoption": 0, "mean_wall_s": 57.6689, "mean_input_tokens": 13147, "mean_output_tokens": 3188, "mean_cache_tokens": 300718, "costs": [ 0.18939809999999999, 0.1843356, 0.23111130000000002, 0.14819909999999997, 0.32388269999999997, 0.1370409, 0.2309823, 0.15816449999999999, 0.29024609999999995, 0.27620729999999993 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 11, 11, 14, 7, 18, 8, 14, 8, 17, 16 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-validate-py": { "n": 10, "successes": 10, "mean_cost": 0.2415549, "ratio": 1.0097222302340694, "ci_lo": 0.8802097790405708, "ci_hi": 1.1702817669376488, "adoption": 0, "mean_wall_s": 53.9323, "mean_input_tokens": 18276, "mean_output_tokens": 4328, "mean_cache_tokens": 223160, "costs": [ 0.22359779999999996, 0.22487880000000002, 0.22109099999999998, 0.2776389, 0.3209028, 0.2341383, 0.2158431, 0.22500989999999998, 0.2403402, 0.2321082 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 23, 23, 23, 16, 40, 23, 23, 23, 24, 23 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-iterate-tests": { "n": 10, "successes": 10, "mean_cost": 0.13977335999999999, "ratio": 1.0902052363224533, "ci_lo": 1.0764826422793694, "ci_hi": 1.101042993817729, "adoption": 0, "mean_wall_s": 26.2151, "mean_input_tokens": 12480, "mean_output_tokens": 1426, "mean_cache_tokens": 144954, "costs": [ 0.1396989, 0.1404987, 0.1402908, 0.14036939999999998, 0.1410957, 0.1343271, 0.1400532, 0.1405413, 0.13993319999999998, 0.1409253 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 10, 10, 10, 10, 10, 10, 10, 10, 10, 10 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py": { "n": 10, "successes": 10, "mean_cost": 0.27235443, "ratio": 1.4738731750462042, "ci_lo": 0.890379139295692, "ci_hi": 2.284291274303857, "adoption": 0, "mean_wall_s": 42.798, "mean_input_tokens": 29053, "mean_output_tokens": 3491, "mean_cache_tokens": 152173, "costs": [ 0.15808679999999997, 0.2918037, 0.5204127000000001, 0.4589517, 0.45422039999999997, 0.1273464, 0.1295175, 0.146019, 0.2995062, 0.13767990000000002 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 8, 27, 42, 40, 41, 6, 7, 8, 28, 7 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py-xl": { "n": 10, "successes": 10, "mean_cost": 0.15146826000000002, "ratio": 1.0327410702871316, "ci_lo": 0.9694880273741604, "ci_hi": 1.1041239955338822, "adoption": 0, "mean_wall_s": 27.0308, "mean_input_tokens": 14465, "mean_output_tokens": 1382, "mean_cache_tokens": 146459, "costs": [ 0.1490991, 0.1597173, 0.139173, 0.16445910000000002, 0.1523982, 0.16475369999999998, 0.14139480000000004, 0.1425429, 0.15874860000000002, 0.1423959 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 6, 6, 5, 7, 6, 6, 7, 6, 6, 6 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-overview-cobra": { "n": 10, "successes": 10, "mean_cost": 0.28181412499999997, "ratio": 0.8763469699831999, "ci_lo": 0.6432260467509257, "ci_hi": 1.1787939823641738, "adoption": 0, "mean_wall_s": 118.7754, "mean_input_tokens": 12395, "mean_output_tokens": 1946, "mean_cache_tokens": 138416, "costs": [ 0.24656505000000004, 0.34253390000000006, 0.3581619, 0.25256809999999996, 0.20907204999999995, 0.35282669999999994, 0.2160466, 0.37541814999999995, 0.21442185000000002, 0.25052695 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 36, 35, 49, 32, 23, 28, 28, 38, 26, 22 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-refactor-split-py": { "n": 10, "successes": 10, "mean_cost": 0.13193654999999999, "ratio": 1.1124215049498738, "ci_lo": 1.0497428229413657, "ci_hi": 1.180009376445382, "adoption": 0, "mean_wall_s": 23.7105, "mean_input_tokens": 11105, "mean_output_tokens": 1412, "mean_cache_tokens": 147024, "costs": [ 0.12003959999999998, 0.13889519999999997, 0.14055959999999998, 0.11936189999999999, 0.1291842, 0.1492212, 0.13033979999999998, 0.1294044, 0.129933, 0.1324266 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 10, 12, 12, 10, 11, 13, 11, 11, 11, 11 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-settings-inventory-django": { "n": 10, "successes": 10, "mean_cost": 0.37500666, "ratio": 1.0732519204643438, "ci_lo": 0.6825025515444991, "ci_hi": 1.6976430187453906, "adoption": 0, "mean_wall_s": 116.3221, "mean_input_tokens": 23445, "mean_output_tokens": 7358, "mean_cache_tokens": 413087, "costs": [ 0.5940834, 0.32819159999999997, 0.2274366, 0.24513930000000003, 0.3405969, 0.24809640000000005, 0.257814, 0.23134110000000002, 1.0032524999999999, 0.2741148 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.9944, 1.0 ], "tool_calls": [ 20, 9, 5, 8, 14, 7, 9, 10, 32, 8 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "log-needle-zh": { "n": 10, "successes": 10, "mean_cost": 0.12923694, "ratio": 1.0582727047614555, "ci_lo": 0.992476964405052, "ci_hi": 1.1241733094522084, "adoption": 0, "mean_wall_s": 27.0805, "mean_input_tokens": 13883, "mean_output_tokens": 1332, "mean_cache_tokens": 86464, "costs": [ 0.11962409999999998, 0.11514059999999998, 0.14155469999999998, 0.1273824, 0.1394094, 0.1239525, 0.1358385, 0.13287120000000002, 0.1379187, 0.1186773 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 7, 7, 7, 7, 6, 6, 6, 7, 8, 7 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "report-pdf": { "n": 10, "successes": 10, "mean_cost": 0.23192868000000003, "ratio": 0.8360380632528955, "ci_lo": 0.6977832429059143, "ci_hi": 1.0053262980740312, "adoption": 0, "mean_wall_s": 81.5818, "mean_input_tokens": 15394, "mean_output_tokens": 5988, "mean_cache_tokens": 165757, "costs": [ 0.1917174, 0.2232438, 0.2903526, 0.2241855, 0.2370819, 0.19264140000000002, 0.28520760000000006, 0.18661380000000002, 0.24568230000000002, 0.24256049999999998 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 5, 5, 9, 5, 6, 5, 7, 5, 6, 6 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "seo-audit": { "n": 10, "successes": 10, "mean_cost": 0.25304064, "ratio": 0.9557711449100101, "ci_lo": 0.9082325860825149, "ci_hi": 1.0054868188697392, "adoption": 0, "mean_wall_s": 103.7172, "mean_input_tokens": 19784, "mean_output_tokens": 5952, "mean_cache_tokens": 150131, "costs": [ 0.2442432, 0.26640240000000004, 0.2756562, 0.2427822, 0.24908219999999998, 0.2352342, 0.2435844, 0.2454462, 0.2519382, 0.2760372 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "web-research-oss-inventory": { "n": 10, "successes": 10, "mean_cost": 4.493085379999999, "ratio": 0.8974781761944092, "ci_lo": 0.7913070665491001, "ci_hi": 1.0020933098844622, "adoption": 0, "mean_wall_s": 845.5323, "mean_input_tokens": 94469, "mean_output_tokens": 23376, "mean_cache_tokens": 938157, "costs": [ 3.8689507999999995, 4.970288499999998, 3.6791365999999996, 5.155421999999999, 5.076106299999999, 3.1603358999999998, 4.696428299999999, 4.960633000000001, 4.5320697999999995, 4.8314826 ], "scores": [ 0.5263, 0.5263, 0.6667, 0.7018, 0.4912, 0.7018, 0.4912, 0.614, 0.5789, 0.5088 ], "tool_calls": [ 43, 55, 48, 58, 57, 41, 53, 60, 51, 53 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] } } }, "claude-token-efficient": { "version": "b32fa8b", "aggregate_cost_ratio": 0.9522367932207415, "aggregate_ci_lo": 0.8247980689646457, "aggregate_ci_hi": 1.0759563303036992, "n_success": 170, "n_runs": 170, "tasks_compared": 17, "per_task": { "code-bugfix-py": { "n": 10, "successes": 10, "mean_cost": 0.09731187, "ratio": 1.059003226246669, "ci_lo": 1.0226741246468374, "ci_hi": 1.096346744678348, "adoption": 0, "mean_wall_s": 20.246299999999998, "mean_input_tokens": 8496, "mean_output_tokens": 731, "mean_cache_tokens": 117865, "costs": [ 0.091779, 0.0958008, 0.0917463, 0.0918255, 0.1035297, 0.10338809999999998, 0.09587129999999999, 0.10366140000000001, 0.1036068, 0.09190979999999999 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 4, 5, 4, 4, 5, 5, 5, 5, 5, 4 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-comprehension-django": { "n": 10, "successes": 10, "mean_cost": 0.17102177999999998, "ratio": 0.4447714875680355, "ci_lo": 0.35466486869513725, "ci_hi": 0.5941229906536517, "adoption": 0, "mean_wall_s": 49.483, "mean_input_tokens": 15152, "mean_output_tokens": 3105, "mean_cache_tokens": 111722, "costs": [ 0.16310159999999999, 0.17371740000000002, 0.1631154, 0.15755819999999998, 0.155859, 0.19115159999999998, 0.1882566, 0.1818261, 0.160767, 0.1748649 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 14, 13, 14, 13, 12, 13, 14, 14, 11, 15 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-cascade-py": { "n": 10, "successes": 10, "mean_cost": 0.51866631, "ratio": 0.7818467098327747, "ci_lo": 0.5283944403025828, "ci_hi": 1.1947919894158656, "adoption": 0, "mean_wall_s": 180.7577, "mean_input_tokens": 29451, "mean_output_tokens": 12350, "mean_cache_tokens": 522207, "costs": [ 0.3571668, 0.8248271999999999, 0.3526767, 0.6514268999999998, 0.3706569, 0.38733059999999986, 0.40914839999999997, 0.3181074, 0.7471326000000001, 0.7681896 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 23, 33, 26, 32, 27, 26, 26, 33, 33, 33 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-ledger-py": { "n": 10, "successes": 10, "mean_cost": 0.267225, "ratio": 0.6776940401802, "ci_lo": 0.4975208310123229, "ci_hi": 0.9476939922264281, "adoption": 0, "mean_wall_s": 90.21979999999999, "mean_input_tokens": 16735, "mean_output_tokens": 5232, "mean_cache_tokens": 294327, "costs": [ 0.22969470000000003, 0.2119239, 0.48410519999999996, 0.21905940000000002, 0.23170769999999996, 0.1923051, 0.2298192, 0.4275015, 0.21996659999999998, 0.22616670000000003 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 17, 14, 24, 15, 17, 14, 17, 17, 17, 17 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-pipeline-py": { "n": 10, "successes": 10, "mean_cost": 0.39876671999999996, "ratio": 1.0199411556355478, "ci_lo": 0.8687785835040043, "ci_hi": 1.2148892720467086, "adoption": 0, "mean_wall_s": 114.3191, "mean_input_tokens": 26237, "mean_output_tokens": 6995, "mean_cache_tokens": 454565, "costs": [ 0.38337540000000003, 0.37886429999999993, 0.3656244, 0.3813078, 0.33508380000000004, 0.4065069, 0.6744842999999999, 0.3973965, 0.2743599, 0.39066389999999995 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 24, 24, 24, 24, 23, 24, 31, 24, 22, 23 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-js": { "n": 10, "successes": 10, "mean_cost": 0.16351104, "ratio": 1.1618855639177244, "ci_lo": 1.0270647843640544, "ci_hi": 1.3118834982973817, "adoption": 0, "mean_wall_s": 52.6209, "mean_input_tokens": 10202, "mean_output_tokens": 2586, "mean_cache_tokens": 211615, "costs": [ 0.1752456, 0.1736214, 0.17381219999999994, 0.1266891, 0.1577682, 0.1416258, 0.1670493, 0.18200429999999998, 0.15762659999999998, 0.17966790000000002 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 10, 11, 11, 8, 10, 8, 10, 12, 10, 11 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-validate-py": { "n": 10, "successes": 10, "mean_cost": 0.23199993, "ratio": 0.969781555802627, "ci_lo": 0.8395047965842559, "ci_hi": 1.1258745686138425, "adoption": 0, "mean_wall_s": 55.961800000000004, "mean_input_tokens": 16971, "mean_output_tokens": 4574, "mean_cache_tokens": 205118, "costs": [ 0.2949168, 0.26306640000000003, 0.22719509999999998, 0.203283, 0.23088869999999997, 0.20487389999999997, 0.20350680000000002, 0.21051000000000003, 0.2014134, 0.28034519999999996 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 40, 33, 24, 23, 23, 23, 23, 23, 23, 32 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-iterate-tests": { "n": 10, "successes": 10, "mean_cost": 0.13060206, "ratio": 1.018670866082773, "ci_lo": 1.0032860225960205, "ci_hi": 1.0331719450380592, "adoption": 0, "mean_wall_s": 31.4788, "mean_input_tokens": 10848, "mean_output_tokens": 1691, "mean_cache_tokens": 133769, "costs": [ 0.13151580000000002, 0.1299534, 0.1311699, 0.12473549999999999, 0.1317228, 0.1290336, 0.1298913, 0.1292016, 0.13487159999999998, 0.1339251 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 10, 10, 10, 10, 10, 10, 10, 10, 10, 10 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py": { "n": 10, "successes": 10, "mean_cost": 0.26298771, "ratio": 1.4231842351006752, "ci_lo": 0.9504658960896146, "ci_hi": 2.1104807897462954, "adoption": 0, "mean_wall_s": 44.7929, "mean_input_tokens": 26835, "mean_output_tokens": 4278, "mean_cache_tokens": 125807, "costs": [ 0.2598435, 0.2459469, 0.2593893, 0.2608983, 0.13577309999999998, 0.5250471, 0.27164640000000007, 0.2879493, 0.135852, 0.2475312 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 27, 27, 27, 27, 7, 52, 28, 29, 7, 27 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py-xl": { "n": 10, "successes": 10, "mean_cost": 0.18264249, "ratio": 1.2452932423103473, "ci_lo": 1.01361746940155, "ci_hi": 1.6322848168790451, "adoption": 0, "mean_wall_s": 37.4273, "mean_input_tokens": 16932, "mean_output_tokens": 2479, "mean_cache_tokens": 146166, "costs": [ 0.1626783, 0.4118805, 0.17309159999999998, 0.15670800000000001, 0.1368579, 0.1628799, 0.1657671, 0.1405998, 0.1411119, 0.17484989999999997 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 7, 53, 8, 6, 6, 6, 7, 7, 7, 9 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-overview-cobra": { "n": 10, "successes": 10, "mean_cost": 0.27012423, "ratio": 0.839995335505433, "ci_lo": 0.6285203241170657, "ci_hi": 1.115034115840565, "adoption": 0, "mean_wall_s": 76.1979, "mean_input_tokens": 20826, "mean_output_tokens": 3816, "mean_cache_tokens": 292996, "costs": [ 0.24535679999999999, 0.21693780000000001, 0.27226500000000003, 0.3088371, 0.22064160000000002, 0.2475846, 0.3694395, 0.29729910000000004, 0.2397453, 0.2831355 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 14, 10, 18, 20, 14, 13, 22, 16, 10, 19 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-refactor-split-py": { "n": 10, "successes": 10, "mean_cost": 0.12087189, "ratio": 1.0191299513283893, "ci_lo": 0.9609384998876194, "ci_hi": 1.076334504412977, "adoption": 0, "mean_wall_s": 26.2417, "mean_input_tokens": 9666, "mean_output_tokens": 1523, "mean_cache_tokens": 133372, "costs": [ 0.1083468, 0.1296648, 0.12001349999999998, 0.1210272, 0.1255536, 0.11925150000000001, 0.132246, 0.1077504, 0.1225767, 0.12228839999999999 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 10, 12, 11, 11, 12, 11, 11, 10, 11, 11 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-settings-inventory-django": { "n": 10, "successes": 10, "mean_cost": 0.34950645, "ratio": 1.0002714849842274, "ci_lo": 0.7313975140911714, "ci_hi": 1.3818002502085072, "adoption": 0, "mean_wall_s": 125.25540000000001, "mean_input_tokens": 21530, "mean_output_tokens": 7700, "mean_cache_tokens": 349280, "costs": [ 0.4260030000000001, 0.26717070000000004, 0.35142629999999997, 0.2345955, 0.41732759999999997, 0.3069897000000001, 0.22306289999999998, 0.5491158, 0.3666588, 0.3527141999999999 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 15, 10, 11, 8, 13, 7, 8, 18, 11, 12 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "log-needle-zh": { "n": 10, "successes": 10, "mean_cost": 0.1290324, "ratio": 1.056597803614524, "ci_lo": 0.970412590569658, "ci_hi": 1.1443538861357343, "adoption": 0, "mean_wall_s": 30.5596, "mean_input_tokens": 12775, "mean_output_tokens": 1503, "mean_cache_tokens": 99423, "costs": [ 0.1281147, 0.1404549, 0.12546870000000002, 0.15355049999999998, 0.14963040000000002, 0.104094, 0.11873369999999998, 0.12436800000000002, 0.1201593, 0.1257498 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 8, 9, 7, 9, 8, 7, 7, 7, 8, 7 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "report-pdf": { "n": 10, "successes": 10, "mean_cost": 0.27773658, "ratio": 1.0011627386388042, "ci_lo": 0.8002416243006163, "ci_hi": 1.2458810785655674, "adoption": 0, "mean_wall_s": 88.7192, "mean_input_tokens": 20888, "mean_output_tokens": 6705, "mean_cache_tokens": 172724, "costs": [ 0.19914060000000003, 0.3366621, 0.24357749999999997, 0.2700741, 0.37111170000000004, 0.1960416, 0.42104339999999996, 0.2507892, 0.2671731, 0.22175250000000002 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 5, 5, 7, 6, 5, 5, 7, 7, 6, 5 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "seo-audit": { "n": 10, "successes": 10, "mean_cost": 0.25712313, "ratio": 0.9711913009030698, "ci_lo": 0.9248124200255389, "ci_hi": 1.0191467327109387, "adoption": 0, "mean_wall_s": 118.5126, "mean_input_tokens": 18756, "mean_output_tokens": 6805, "mean_cache_tokens": 141626, "costs": [ 0.2666742, 0.2485434, 0.2855964, 0.24528509999999998, 0.25343039999999994, 0.2562543, 0.26304690000000003, 0.2386194, 0.2583768, 0.25540440000000003 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "web-research-oss-inventory": { "n": 10, "successes": 10, "mean_cost": 4.78269883, "ratio": 0.9553274554590226, "ci_lo": 0.8610062338806772, "ci_hi": 1.0541045819377044, "adoption": 0, "mean_wall_s": 953.1229, "mean_input_tokens": 98942, "mean_output_tokens": 26362, "mean_cache_tokens": 996140, "costs": [ 5.103012199999999, 4.1308109, 5.309648099999998, 5.534078700000001, 4.381007599999999, 4.413191899999999, 3.9686279, 5.003748400000002, 4.8911262, 5.091736400000001 ], "scores": [ 0.5965, 0.6491, 0.6491, 0.5789, 0.6316, 0.6842, 0.5789, 0.6491, 0.6842, 0.5439 ], "tool_calls": [ 57, 49, 63, 63, 48, 57, 45, 61, 56, 58 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] } } }, "code-review-graph": { "version": "2.3.6", "aggregate_cost_ratio": 1.0113475169386419, "aggregate_ci_lo": 0.9021437384639193, "aggregate_ci_hi": 1.1283570512884105, "n_success": 170, "n_runs": 170, "tasks_compared": 17, "per_task": { "code-bugfix-py": { "n": 10, "successes": 10, "mean_cost": 0.09885525, "ratio": 1.0757991669610403, "ci_lo": 1.0569099979151813, "ci_hi": 1.093513313961797, "adoption": 0, "mean_wall_s": 19.2157, "mean_input_tokens": 9158, "mean_output_tokens": 727, "mean_cache_tokens": 109944, "costs": [ 0.0998421, 0.0998583, 0.0997383, 0.1003401, 0.09586739999999999, 0.1016991, 0.0955752, 0.0999921, 0.1002075, 0.09543239999999999 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 5, 5, 5, 5, 4, 5, 4, 5, 5, 4 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-comprehension-django": { "n": 10, "successes": 10, "mean_cost": 0.339324195, "ratio": 0.8824707997892207, "ci_lo": 0.5942666314275414, "ci_hi": 1.2992012897929281, "adoption": 0, "mean_wall_s": 49.0235, "mean_input_tokens": 15415, "mean_output_tokens": 2788, "mean_cache_tokens": 102362, "costs": [ 0.20600490000000002, 0.5590965000000001, 0.194058, 0.5101625999999999, 0.1585542, 0.1715568, 0.48998939999999985, 0.1959672, 0.44086964999999995, 0.46698270000000003 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 17, 26, 16, 25, 12, 13, 25, 15, 24, 29 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-cascade-py": { "n": 10, "successes": 10, "mean_cost": 0.38520023999999997, "ratio": 0.5806576106144145, "ci_lo": 0.410764855648436, "ci_hi": 0.8601780611350359, "adoption": 0, "mean_wall_s": 114.057, "mean_input_tokens": 23492, "mean_output_tokens": 7493, "mean_cache_tokens": 439372, "costs": [ 0.36318719999999993, 0.3594647999999999, 0.6454896000000001, 0.3680757, 0.36639420000000006, 0.4147778999999999, 0.38083949999999994, 0.2694879, 0.33367349999999996, 0.3506120999999999 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 26, 26, 24, 25, 24, 24, 26, 22, 26, 26 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-ledger-py": { "n": 10, "successes": 10, "mean_cost": 0.37849809, "ratio": 0.9598873601369219, "ci_lo": 0.6060390398699511, "ci_hi": 1.434384970100357, "adoption": 0, "mean_wall_s": 140.6305, "mean_input_tokens": 22707, "mean_output_tokens": 9071, "mean_cache_tokens": 353853, "costs": [ 0.6519033, 0.22095059999999997, 0.23021429999999998, 0.19969979999999998, 0.572952, 0.7543764, 0.5410185000000001, 0.23243729999999999, 0.19240590000000002, 0.18902280000000002 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 25, 14, 14, 12, 24, 30, 21, 17, 14, 14 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-pipeline-py": { "n": 10, "successes": 10, "mean_cost": 0.39353253, "ratio": 1.0065534642118101, "ci_lo": 0.9107788892136891, "ci_hi": 1.1269153782764552, "adoption": 0, "mean_wall_s": 103.90990000000001, "mean_input_tokens": 27262, "mean_output_tokens": 6558, "mean_cache_tokens": 438495, "costs": [ 0.412428, 0.3792759, 0.3740697, 0.38494530000000005, 0.38773620000000003, 0.5102289, 0.3312993, 0.3756327, 0.4169664000000001, 0.3627429 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 24, 23, 23, 23, 24, 24, 26, 23, 23, 23 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-js": { "n": 10, "successes": 10, "mean_cost": 0.16641501, "ratio": 1.182520750514606, "ci_lo": 1.03509032436067, "ci_hi": 1.3559440951139123, "adoption": 0, "mean_wall_s": 47.5306, "mean_input_tokens": 10795, "mean_output_tokens": 2414, "mean_cache_tokens": 218047, "costs": [ 0.2083473, 0.1614099, 0.1737, 0.15723300000000004, 0.1312443, 0.193833, 0.1395003, 0.17892329999999998, 0.1448067, 0.17515229999999998 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 13, 10, 11, 10, 8, 12, 8, 11, 9, 11 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-validate-py": { "n": 10, "successes": 10, "mean_cost": 0.21705372, "ratio": 0.907304990455591, "ci_lo": 0.8030638408213003, "ci_hi": 1.0294300022165719, "adoption": 0, "mean_wall_s": 51.63, "mean_input_tokens": 16731, "mean_output_tokens": 4192, "mean_cache_tokens": 179194, "costs": [ 0.2229303, 0.20698979999999997, 0.2070225, 0.2079135, 0.2085327, 0.21281670000000003, 0.2079837, 0.2140434, 0.22482480000000002, 0.2574798 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-iterate-tests": { "n": 10, "successes": 10, "mean_cost": 0.1358448, "ratio": 1.0595632264057786, "ci_lo": 1.0324972105549508, "ci_hi": 1.0846630662963366, "adoption": 0, "mean_wall_s": 30.6848, "mean_input_tokens": 11543, "mean_output_tokens": 1775, "mean_cache_tokens": 133128, "costs": [ 0.1389438, 0.12684990000000002, 0.1370757, 0.1361283, 0.1446261, 0.12690089999999998, 0.13948409999999997, 0.1348689, 0.13596239999999998, 0.1376079 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 10, 10, 10, 10, 10, 10, 10, 10, 10, 10 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py": { "n": 10, "successes": 10, "mean_cost": 0.26934738, "ratio": 1.457600223910353, "ci_lo": 0.9096778064677721, "ci_hi": 2.2675607221170626, "adoption": 0, "mean_wall_s": 46.5229, "mean_input_tokens": 27644, "mean_output_tokens": 3649, "mean_cache_tokens": 162419, "costs": [ 0.25545, 0.16051650000000003, 0.2666379, 0.13512390000000002, 0.1499568, 0.5522538, 0.1777824, 0.40482, 0.4419258, 0.1490067 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 27, 8, 27, 6, 8, 52, 8, 41, 42, 7 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py-xl": { "n": 10, "successes": 10, "mean_cost": 0.15830445, "ratio": 1.0793515890670144, "ci_lo": 1.0055819671755766, "ci_hi": 1.1632010370040986, "adoption": 0, "mean_wall_s": 34.015100000000004, "mean_input_tokens": 14895, "mean_output_tokens": 1797, "mean_cache_tokens": 139852, "costs": [ 0.1425588, 0.14639639999999998, 0.16407180000000002, 0.18093360000000003, 0.1478721, 0.15031710000000004, 0.1494261, 0.17088930000000002, 0.17569649999999998, 0.1548828 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 7, 6, 7, 9, 6, 7, 7, 8, 8, 8 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-overview-cobra": { "n": 10, "successes": 10, "mean_cost": 0.25054335, "ratio": 0.7791053965869893, "ci_lo": 0.5755219690586952, "ci_hi": 1.058924675836774, "adoption": 0, "mean_wall_s": 74.0835, "mean_input_tokens": 19871, "mean_output_tokens": 3713, "mean_cache_tokens": 251710, "costs": [ 0.4338729, 0.2386269, 0.24462899999999999, 0.21091260000000003, 0.22886849999999997, 0.2202696, 0.2315688, 0.24422760000000004, 0.223998, 0.22845959999999998 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 27, 9, 14, 14, 15, 13, 12, 14, 14, 10 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-refactor-split-py": { "n": 10, "successes": 10, "mean_cost": 0.12622878, "ratio": 1.0642965077955013, "ci_lo": 0.9975701160806543, "ci_hi": 1.1338504571712786, "adoption": 0, "mean_wall_s": 27.8894, "mean_input_tokens": 10416, "mean_output_tokens": 1582, "mean_cache_tokens": 133297, "costs": [ 0.13779780000000003, 0.1216788, 0.11790810000000002, 0.1176939, 0.14527020000000002, 0.11702189999999998, 0.1276872, 0.1278927, 0.1356825, 0.1136547 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 12, 11, 11, 11, 12, 11, 12, 12, 11, 10 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-settings-inventory-django": { "n": 10, "successes": 10, "mean_cost": 0.49762145999999996, "ratio": 1.4241698737010982, "ci_lo": 0.9767229148957315, "ci_hi": 2.036571004100051, "adoption": 0, "mean_wall_s": 168.0852, "mean_input_tokens": 27716, "mean_output_tokens": 10043, "mean_cache_tokens": 602051, "costs": [ 0.46222469999999993, 0.39385259999999994, 0.6043392000000001, 0.7020216, 0.21864960000000003, 0.7718816999999998, 0.2571618, 0.6179372999999999, 0.6955946999999999, 0.25255140000000004 ], "scores": [ 1.0, 1.0, 1.0, 0.9758, 1.0, 1.0, 1.0, 1.0, 0.9944, 1.0 ], "tool_calls": [ 18, 12, 22, 28, 6, 33, 8, 21, 22, 9 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "log-needle-zh": { "n": 10, "successes": 10, "mean_cost": 0.12789282, "ratio": 1.0472662115101918, "ci_lo": 0.975157212509818, "ci_hi": 1.1260727635700964, "adoption": 0, "mean_wall_s": 32.605000000000004, "mean_input_tokens": 12521, "mean_output_tokens": 1753, "mean_cache_tokens": 88173, "costs": [ 0.12430440000000001, 0.1140021, 0.1340391, 0.15300660000000002, 0.1420386, 0.12298769999999999, 0.1171479, 0.1317942, 0.1177914, 0.12181619999999999 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 6, 7, 7, 8, 8, 7, 7, 8, 6, 7 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "report-pdf": { "n": 10, "successes": 10, "mean_cost": 0.29999328, "ratio": 1.0813919209995226, "ci_lo": 0.8698558266155959, "ci_hi": 1.3365058911882153, "adoption": 0, "mean_wall_s": 111.0344, "mean_input_tokens": 18902, "mean_output_tokens": 8071, "mean_cache_tokens": 218290, "costs": [ 0.2702706, 0.28001699999999996, 0.4373664, 0.2725635, 0.388005, 0.27996659999999995, 0.3710892, 0.222828, 0.2185494, 0.2592771 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 6, 11, 7, 9, 8, 9, 10, 5, 5, 5 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "seo-audit": { "n": 10, "successes": 10, "mean_cost": 0.26444628, "ratio": 0.9988518990577684, "ci_lo": 0.9493456367907677, "ci_hi": 1.0471219813918724, "adoption": 0, "mean_wall_s": 120.4487, "mean_input_tokens": 19854, "mean_output_tokens": 6991, "mean_cache_tokens": 134794, "costs": [ 0.2715681, 0.25275960000000003, 0.25319939999999996, 0.23928839999999998, 0.27654090000000003, 0.25823250000000003, 0.26226989999999994, 0.2752059, 0.27403350000000004, 0.28136459999999996 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "web-research-oss-inventory": { "n": 10, "successes": 10, "mean_cost": 4.7661393, "ratio": 0.9520197469411318, "ci_lo": 0.8429240882492005, "ci_hi": 1.0720543398362374, "adoption": 0, "mean_wall_s": 923.5484, "mean_input_tokens": 103772, "mean_output_tokens": 29822, "mean_cache_tokens": 953936, "costs": [ 4.933855199999999, 6.2960864999999995, 4.562604799999999, 4.2992802999999995, 5.5555579999999996, 5.175538300000001, 4.454418399999999, 4.3992028, 4.141359699999999, 3.843488999999999 ], "scores": [ 0.7018, 0.6842, 0.6491, 0.6316, 0.6667, 0.6491, 0.7193, 0.5614, 0.6842, 0.5439 ], "tool_calls": [ 58, 77, 51, 49, 65, 63, 46, 52, 49, 48 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] } } }, "codegraph": { "version": "0.9.9", "aggregate_cost_ratio": 1.0494160451367467, "aggregate_ci_lo": 0.9459574187229038, "aggregate_ci_hi": 1.1661736916132155, "n_success": 170, "n_runs": 170, "tasks_compared": 17, "per_task": { "code-bugfix-py": { "n": 10, "successes": 10, "mean_cost": 0.09888585, "ratio": 1.0761321735996254, "ci_lo": 1.058097922524019, "ci_hi": 1.0944633720429229, "adoption": 0, "mean_wall_s": 20.5447, "mean_input_tokens": 9240, "mean_output_tokens": 693, "mean_cache_tokens": 110108, "costs": [ 0.10132830000000001, 0.10114650000000001, 0.10039050000000002, 0.10122990000000001, 0.0965895, 0.09596070000000001, 0.09622679999999999, 0.0999462, 0.0998511, 0.09618899999999998 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 5, 5, 4, 5, 4, 4, 4, 4, 4, 4 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-comprehension-django": { "n": 10, "successes": 10, "mean_cost": 0.42023241, "ratio": 1.0928864973806296, "ci_lo": 0.7947627983045724, "ci_hi": 1.5114098366896815, "adoption": 0, "mean_wall_s": 50.7619, "mean_input_tokens": 15127, "mean_output_tokens": 2565, "mean_cache_tokens": 70505, "costs": [ 0.37540365, 0.1574427, 0.49817055000000005, 0.5026899, 0.5014665, 0.5268873, 0.47407769999999994, 0.2022291, 0.48667859999999996, 0.4772781 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 27, 13, 28, 31, 26, 23, 27, 12, 28, 26 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-cascade-py": { "n": 10, "successes": 10, "mean_cost": 0.46169817, "ratio": 0.6959719345378595, "ci_lo": 0.4625563975077405, "ci_hi": 1.1040861337167565, "adoption": 0, "mean_wall_s": 140.0814, "mean_input_tokens": 25946, "mean_output_tokens": 9264, "mean_cache_tokens": 556717, "costs": [ 0.4040589000000001, 0.37237500000000007, 0.42436560000000006, 0.3508829999999999, 0.4067154, 0.4540869, 1.0820607000000002, 0.38043180000000004, 0.3595803, 0.38242410000000004 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 27, 25, 26, 25, 23, 23, 43, 26, 26, 26 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-ledger-py": { "n": 10, "successes": 10, "mean_cost": 0.37913421, "ratio": 0.9615005876898808, "ci_lo": 0.6707988360302343, "ci_hi": 1.3612439159027556, "adoption": 0, "mean_wall_s": 155.5508, "mean_input_tokens": 23266, "mean_output_tokens": 9843, "mean_cache_tokens": 306202, "costs": [ 0.39043380000000005, 0.5883342, 0.185673, 0.496662, 0.1938009, 0.5947905000000001, 0.43745490000000004, 0.18843809999999997, 0.22947030000000004, 0.4862844 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 18, 30, 14, 22, 15, 23, 18, 14, 17, 19 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-pipeline-py": { "n": 10, "successes": 10, "mean_cost": 0.39995967000000004, "ratio": 1.022992410267869, "ci_lo": 0.9397559532211682, "ci_hi": 1.1260942500511129, "adoption": 0, "mean_wall_s": 109.0281, "mean_input_tokens": 28731, "mean_output_tokens": 7227, "mean_cache_tokens": 397127, "costs": [ 0.3828957, 0.42114750000000006, 0.3900684000000001, 0.4559226, 0.396573, 0.4138164, 0.4004124, 0.3776394, 0.3527442, 0.40837710000000005 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 23, 23, 23, 23, 24, 18, 23, 23, 24, 23 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-js": { "n": 10, "successes": 10, "mean_cost": 0.1517313, "ratio": 1.0781804523074983, "ci_lo": 0.9404856270603011, "ci_hi": 1.2347337276084076, "adoption": 0, "mean_wall_s": 43.2745, "mean_input_tokens": 10468, "mean_output_tokens": 2136, "mean_cache_tokens": 189538, "costs": [ 0.1902348, 0.1337202, 0.1563972, 0.1573704, 0.12958350000000002, 0.17720490000000003, 0.1661346, 0.1394628, 0.1528731, 0.1143315 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 11, 8, 10, 10, 8, 11, 9, 8, 10, 7 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-validate-py": { "n": 10, "successes": 10, "mean_cost": 0.21060645, "ratio": 0.8803547946892405, "ci_lo": 0.7827982303801413, "ci_hi": 0.9949196245662741, "adoption": 0, "mean_wall_s": 53.4433, "mean_input_tokens": 16546, "mean_output_tokens": 3835, "mean_cache_tokens": 179266, "costs": [ 0.2080386, 0.20809350000000001, 0.2088447, 0.2111172, 0.20760479999999998, 0.20770229999999998, 0.2089734, 0.20939249999999998, 0.2101422, 0.2261553 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-iterate-tests": { "n": 10, "successes": 10, "mean_cost": 0.13798047, "ratio": 1.0762210402914631, "ci_lo": 1.0491263229622962, "ci_hi": 1.1069276097131835, "adoption": 0, "mean_wall_s": 35.4728, "mean_input_tokens": 11672, "mean_output_tokens": 1698, "mean_cache_tokens": 141559, "costs": [ 0.13750260000000003, 0.13923359999999999, 0.1368339, 0.13857689999999998, 0.12763649999999999, 0.13636500000000001, 0.1398483, 0.1332183, 0.15195060000000002, 0.138639 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 10, 10, 10, 10, 10, 10, 10, 10, 10, 10 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py": { "n": 10, "successes": 10, "mean_cost": 0.28063431, "ratio": 1.5186805718805483, "ci_lo": 0.9717852466272205, "ci_hi": 2.275407100174849, "adoption": 0, "mean_wall_s": 49.2442, "mean_input_tokens": 29418, "mean_output_tokens": 4094, "mean_cache_tokens": 142295, "costs": [ 0.2768637, 0.4698306000000001, 0.1385835, 0.15478920000000002, 0.40397970000000005, 0.26828520000000006, 0.3985161, 0.14490389999999997, 0.13495079999999998, 0.41564039999999997 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 29, 41, 6, 8, 41, 27, 40, 6, 7, 47 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py-xl": { "n": 10, "successes": 10, "mean_cost": 0.16089444, "ratio": 1.0970106619621078, "ci_lo": 1.0186216090366924, "ci_hi": 1.1853728745624807, "adoption": 0, "mean_wall_s": 39.6455, "mean_input_tokens": 15186, "mean_output_tokens": 1848, "mean_cache_tokens": 140032, "costs": [ 0.163437, 0.1638993, 0.1416402, 0.1889784, 0.1754766, 0.1415628, 0.16495559999999998, 0.1610481, 0.1448136, 0.1631328 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 7, 7, 6, 9, 7, 8, 8, 6, 8, 7 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-overview-cobra": { "n": 10, "successes": 10, "mean_cost": 0.26088939, "ratio": 0.8112780948338391, "ci_lo": 0.6126125223457074, "ci_hi": 1.0718579663342023, "adoption": 4, "mean_wall_s": 78.2245, "mean_input_tokens": 22246, "mean_output_tokens": 3593, "mean_cache_tokens": 244703, "costs": [ 0.2269779, 0.22891830000000002, 0.2191194, 0.27079050000000005, 0.3201735, 0.2366238, 0.2757114, 0.27567179999999997, 0.3285342, 0.2263731 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 12, 10, 15, 18, 10, 11, 13, 11, 19, 11 ], "competitor_tool_calls": [ 0, 2, 0, 0, 4, 0, 2, 2, 0, 0 ] }, "code-refactor-split-py": { "n": 10, "successes": 10, "mean_cost": 0.12516939, "ratio": 1.0553642731862982, "ci_lo": 1.0016137746256362, "ci_hi": 1.1094051176646098, "adoption": 0, "mean_wall_s": 29.2598, "mean_input_tokens": 10561, "mean_output_tokens": 1549, "mean_cache_tokens": 128522, "costs": [ 0.1249551, 0.1228848, 0.12505290000000002, 0.1133211, 0.1284273, 0.13156410000000002, 0.11824440000000001, 0.1288278, 0.1258257, 0.1325907 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 11, 11, 11, 10, 11, 12, 11, 12, 11, 12 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-settings-inventory-django": { "n": 10, "successes": 10, "mean_cost": 0.42421992, "ratio": 1.2140980211904244, "ci_lo": 0.7949115315517383, "ci_hi": 1.818496200988756, "adoption": 0, "mean_wall_s": 147.9567, "mean_input_tokens": 24300, "mean_output_tokens": 8578, "mean_cache_tokens": 498976, "costs": [ 0.9016614000000001, 0.21459899999999998, 0.47423789999999993, 0.431826, 0.1905942, 0.4129674, 0.6101154, 0.3279279, 0.44753610000000005, 0.2307339 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 34, 6, 15, 15, 7, 13, 24, 11, 20, 8 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "log-needle-zh": { "n": 10, "successes": 10, "mean_cost": 0.18406356, "ratio": 1.5072272795163864, "ci_lo": 1.1780623017749894, "ci_hi": 1.8817292968529742, "adoption": 0, "mean_wall_s": 35.9109, "mean_input_tokens": 20880, "mean_output_tokens": 1767, "mean_cache_tokens": 107550, "costs": [ 0.2879913, 0.15435629999999997, 0.1295589, 0.1568538, 0.1338624, 0.28977569999999997, 0.1277166, 0.2891097, 0.1512255, 0.1201854 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 6, 9, 7, 9, 7, 6, 8, 7, 9, 7 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "report-pdf": { "n": 10, "successes": 10, "mean_cost": 0.2733951, "ratio": 0.9855129167588574, "ci_lo": 0.7819124865821283, "ci_hi": 1.234949142664848, "adoption": 0, "mean_wall_s": 101.4592, "mean_input_tokens": 17819, "mean_output_tokens": 7353, "mean_cache_tokens": 187046, "costs": [ 0.2244507, 0.2881506, 0.2871279, 0.20488499999999998, 0.39567420000000003, 0.20336520000000002, 0.24957480000000004, 0.208587, 0.2582871, 0.41384850000000006 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 6, 8, 6, 5, 5, 5, 5, 5, 6, 14 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "seo-audit": { "n": 10, "successes": 10, "mean_cost": 0.29256378, "ratio": 1.1050557687879714, "ci_lo": 1.0232757503529817, "ci_hi": 1.1993075142431415, "adoption": 0, "mean_wall_s": 143.1482, "mean_input_tokens": 20836, "mean_output_tokens": 8026, "mean_cache_tokens": 157146, "costs": [ 0.2703303, 0.2798469, 0.28490190000000004, 0.2786613, 0.3350223, 0.27783630000000004, 0.2865114, 0.2988759, 0.24605280000000002, 0.36759869999999994 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 14, 14, 14, 14, 16, 14, 14, 14, 14, 18 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "web-research-oss-inventory": { "n": 10, "successes": 10, "mean_cost": 4.86181248, "ratio": 0.9711301318626664, "ci_lo": 0.8843201154839782, "ci_hi": 1.061445478649347, "adoption": 0, "mean_wall_s": 954.4095, "mean_input_tokens": 106151, "mean_output_tokens": 29602, "mean_cache_tokens": 915886, "costs": [ 4.650261299999999, 4.7901899, 4.4573483, 5.272515700000001, 4.7143745, 4.6468171, 5.7857045000000005, 4.7752105, 5.0185902, 4.5071128 ], "scores": [ 0.5088, 0.614, 0.6316, 0.6842, 0.5965, 0.5439, 0.5965, 0.5965, 0.5439, 0.6667 ], "tool_calls": [ 53, 46, 55, 62, 54, 58, 66, 50, 59, 51 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] } } }, "edgee": { "version": "v0.2.13", "aggregate_cost_ratio": 1.0092565725594083, "aggregate_ci_lo": 0.8513202313728729, "aggregate_ci_hi": 1.1695418779070588, "n_success": 169, "n_runs": 170, "tasks_compared": 17, "per_task": { "code-bugfix-py": { "n": 10, "successes": 10, "mean_cost": 0.12002049, "ratio": 1.3061313704659676, "ci_lo": 1.2824963706315586, "ci_hi": 1.3388645072958265, "adoption": 0, "mean_wall_s": 20.6508, "mean_input_tokens": 0, "mean_output_tokens": 0, "mean_cache_tokens": 0, "costs": [ 0.11847809999999998, 0.1185336, 0.11854050000000001, 0.11858549999999998, 0.13060349999999998, 0.12147599999999997, 0.1185714, 0.11855639999999998, 0.1185675, 0.11829239999999998 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 5, 5, 5, 5, 6, 5, 5, 5, 5, 5 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-comprehension-django": { "n": 10, "successes": 10, "mean_cost": 0.23782747499999998, "ratio": 0.6185111617964669, "ci_lo": 0.4139811940185404, "ci_hi": 0.9345930299906395, "adoption": 0, "mean_wall_s": 43.7915, "mean_input_tokens": 0, "mean_output_tokens": 0, "mean_cache_tokens": 0, "costs": [ 0.5012911499999999, 0.1638732, 0.20003129999999997, 0.15172049999999998, 0.1832064, 0.18963390000000002, 0.19976520000000003, 0.46213409999999994, 0.15259140000000002, 0.17402759999999998 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 22, 14, 17, 11, 15, 15, 16, 28, 12, 14 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-cascade-py": { "n": 10, "successes": 10, "mean_cost": 0.32687868, "ratio": 0.49274266622885227, "ci_lo": 0.3547254393957709, "ci_hi": 0.7180696874179576, "adoption": 0, "mean_wall_s": 81.78999999999999, "mean_input_tokens": 0, "mean_output_tokens": 0, "mean_cache_tokens": 0, "costs": [ 0.4010151, 0.3953235, 0.3506510999999999, 0.26606339999999995, 0.2459796, 0.35823330000000003, 0.2872614, 0.36705390000000004, 0.31744710000000004, 0.2797584 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 26, 25, 24, 25, 21, 25, 25, 26, 24, 24 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-ledger-py": { "n": 10, "successes": 10, "mean_cost": 0.18990192, "ratio": 0.4815993990187188, "ci_lo": 0.37982792936608706, "ci_hi": 0.6340132706144461, "adoption": 0, "mean_wall_s": 39.9253, "mean_input_tokens": 0, "mean_output_tokens": 0, "mean_cache_tokens": 0, "costs": [ 0.2699979, 0.18006509999999998, 0.1981203, 0.1770375, 0.15697979999999997, 0.21200129999999998, 0.1510617, 0.1955943, 0.1663077, 0.1918536 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 15, 13, 14, 13, 11, 13, 11, 11, 13, 11 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-pipeline-py": { "n": 10, "successes": 10, "mean_cost": 0.37486155, "ratio": 0.9587979721836685, "ci_lo": 0.877411883564395, "ci_hi": 1.0544112637954837, "adoption": 0, "mean_wall_s": 84.8744, "mean_input_tokens": 0, "mean_output_tokens": 0, "mean_cache_tokens": 0, "costs": [ 0.38149980000000006, 0.3923598, 0.4199585999999999, 0.37693679999999996, 0.3222021, 0.3457593, 0.36570179999999997, 0.3660590999999999, 0.41353710000000005, 0.36460109999999996 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 22, 23, 22, 22, 19, 23, 23, 23, 17, 23 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-js": { "n": 10, "successes": 10, "mean_cost": 0.17786364, "ratio": 1.2638730428346558, "ci_lo": 1.1016715248897895, "ci_hi": 1.4615150214730457, "adoption": 0, "mean_wall_s": 39.7371, "mean_input_tokens": 0, "mean_output_tokens": 0, "mean_cache_tokens": 0, "costs": [ 0.2004876, 0.181893, 0.2338539, 0.16476960000000002, 0.1359963, 0.1551888, 0.19968899999999998, 0.14979689999999998, 0.1574028, 0.19955849999999997 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 10, 9, 11, 8, 6, 7, 10, 7, 8, 10 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-validate-py": { "n": 10, "successes": 10, "mean_cost": 0.23840160000000002, "ratio": 0.9965411392746351, "ci_lo": 0.8557748450990061, "ci_hi": 1.1756786239581634, "adoption": 0, "mean_wall_s": 50.1349, "mean_input_tokens": 0, "mean_output_tokens": 0, "mean_cache_tokens": 0, "costs": [ 0.2198661, 0.3228804, 0.2166324, 0.21632459999999998, 0.2178747, 0.2174751, 0.21903540000000002, 0.21667140000000001, 0.3218229000000001, 0.215433 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 16, 40, 16, 16, 16, 16, 16, 16, 40, 16 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-iterate-tests": { "n": 10, "successes": 10, "mean_cost": 0.1436916, "ratio": 1.1207667522305498, "ci_lo": 1.0796067258199298, "ci_hi": 1.1675420238408245, "adoption": 0, "mean_wall_s": 26.2028, "mean_input_tokens": 0, "mean_output_tokens": 0, "mean_cache_tokens": 0, "costs": [ 0.1355826, 0.1386786, 0.1532433, 0.1356066, 0.1369257, 0.16167120000000001, 0.15303419999999998, 0.1365642, 0.1493046, 0.13630499999999998 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 9, 10, 10, 9, 9, 10, 10, 9, 10, 9 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py": { "n": 10, "successes": 9, "mean_cost": 0.22504816666666666, "ratio": 1.2178706105251431, "ci_lo": 0.8098114722999932, "ci_hi": 1.8204969558479807, "adoption": 0, "mean_wall_s": 40.2128, "mean_input_tokens": 0, "mean_output_tokens": 0, "mean_cache_tokens": 0, "costs": [ 0.21389760000000002, 0.1829538, 0.2071845, 0.14168909999999998, 0.3758274, 0.1811568, 0.39324239999999994, 0.17502120000000002, 0.15446069999999998 ], "scores": [ 1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 7, 7, 19, 8, 6, 30, 7, 40, 7, 7 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py-xl": { "n": 10, "successes": 10, "mean_cost": 0.20005902, "ratio": 1.3640426478484315, "ci_lo": 1.2446219760493653, "ci_hi": 1.494432468108322, "adoption": 0, "mean_wall_s": 41.983000000000004, "mean_input_tokens": 0, "mean_output_tokens": 0, "mean_cache_tokens": 0, "costs": [ 0.19546080000000002, 0.20935769999999998, 0.2389767, 0.1730592, 0.15771690000000002, 0.241188, 0.1889232, 0.1991964, 0.1915959, 0.20511539999999998 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 9, 9, 10, 8, 7, 12, 8, 9, 9, 10 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-overview-cobra": { "n": 10, "successes": 10, "mean_cost": 0.343721385, "ratio": 1.0688576886030072, "ci_lo": 0.7549205330734761, "ci_hi": 1.5004567542132443, "adoption": 0, "mean_wall_s": 102.9756, "mean_input_tokens": 0, "mean_output_tokens": 0, "mean_cache_tokens": 0, "costs": [ 0.52107405, 0.23796749999999997, 0.58782345, 0.2389605, 0.40794929999999996, 0.2650578, 0.33448215, 0.29932695000000004, 0.32508285000000003, 0.21948929999999997 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 38, 17, 52, 11, 31, 18, 21, 13, 13, 13 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-refactor-split-py": { "n": 10, "successes": 10, "mean_cost": 0.14236748999999999, "ratio": 1.2003698556748386, "ci_lo": 1.0986994764460074, "ci_hi": 1.3522815588451529, "adoption": 0, "mean_wall_s": 25.9579, "mean_input_tokens": 0, "mean_output_tokens": 0, "mean_cache_tokens": 0, "costs": [ 0.20954309999999998, 0.1338309, 0.1272135, 0.13356869999999998, 0.1331853, 0.13309769999999999, 0.1337904, 0.1505103, 0.1341501, 0.1347849 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 14, 10, 5, 10, 10, 10, 10, 11, 10, 10 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-settings-inventory-django": { "n": 10, "successes": 10, "mean_cost": 0.42269784000000005, "ratio": 1.2097418978002419, "ci_lo": 0.8682910254545599, "ci_hi": 1.6786317129738906, "adoption": 0, "mean_wall_s": 113.5405, "mean_input_tokens": 0, "mean_output_tokens": 0, "mean_cache_tokens": 0, "costs": [ 0.4055001000000001, 0.49435290000000015, 0.273612, 0.42092909999999994, 0.6185364000000001, 0.5187135, 0.6087066000000001, 0.31771320000000003, 0.2458404, 0.32307420000000003 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 14, 18, 8, 17, 24, 17, 21, 8, 8, 7 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "log-needle-zh": { "n": 10, "successes": 10, "mean_cost": 0.13939194000000002, "ratio": 1.1414281811821492, "ci_lo": 1.0312515767858983, "ci_hi": 1.262547525733567, "adoption": 0, "mean_wall_s": 26.6065, "mean_input_tokens": 0, "mean_output_tokens": 0, "mean_cache_tokens": 0, "costs": [ 0.18251130000000002, 0.1123428, 0.1456257, 0.1247211, 0.1207845, 0.1236555, 0.13689780000000001, 0.14365740000000002, 0.16807320000000003, 0.1356501 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 12, 6, 8, 7, 7, 7, 8, 8, 10, 8 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "report-pdf": { "n": 10, "successes": 10, "mean_cost": 0.29300481, "ratio": 1.0562004400498577, "ci_lo": 0.8499608623519341, "ci_hi": 1.317064655010319, "adoption": 0, "mean_wall_s": 104.3947, "mean_input_tokens": 0, "mean_output_tokens": 0, "mean_cache_tokens": 0, "costs": [ 0.3292383, 0.3159276, 0.26114309999999996, 0.2643522000000001, 0.2368164, 0.2456229, 0.22439580000000006, 0.2491257, 0.31066980000000005, 0.49275630000000004 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 9, 8, 6, 7, 5, 7, 7, 6, 7, 10 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "seo-audit": { "n": 10, "successes": 10, "mean_cost": 0.343755105, "ratio": 1.2984128173028284, "ci_lo": 1.0710620098241113, "ci_hi": 1.5206959881980882, "adoption": 0, "mean_wall_s": 144.42680000000001, "mean_input_tokens": 0, "mean_output_tokens": 0, "mean_cache_tokens": 0, "costs": [ 0.40484024999999996, 0.39913380000000004, 0.38966025, 0.23332949999999997, 0.21869490000000003, 0.2335416, 0.4142920500000001, 0.48302085000000006, 0.41467754999999995, 0.2463603 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 15, 15, 15, 3, 3, 3, 15, 15, 15, 4 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "web-research-oss-inventory": { "n": 10, "successes": 10, "mean_cost": 5.591157030000001, "ratio": 1.116814168819768, "ci_lo": 0.9817313668728632, "ci_hi": 1.2676995149325172, "adoption": 0, "mean_wall_s": 891.5627000000001, "mean_input_tokens": 0, "mean_output_tokens": 0, "mean_cache_tokens": 0, "costs": [ 5.507349800000002, 4.2518231, 5.914498499999999, 5.656576799999999, 4.951341199999999, 4.8152912, 6.2609799000000015, 7.3222705999999995, 4.7148884, 6.516550800000005 ], "scores": [ 0.5614, 0.6491, 0.6316, 0.5965, 0.5789, 0.6842, 0.6667, 0.7018, 0.6316, 0.5789 ], "tool_calls": [ 65, 49, 71, 64, 57, 52, 74, 84, 56, 77 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] } } }, "graphify": { "version": "0.8.49", "aggregate_cost_ratio": 0.9911855904549622, "aggregate_ci_lo": 0.8707359930059344, "aggregate_ci_hi": 1.1120196999324352, "n_success": 170, "n_runs": 170, "tasks_compared": 17, "per_task": { "code-bugfix-py": { "n": 10, "successes": 10, "mean_cost": 0.09465764999999998, "ratio": 1.0301184915974588, "ci_lo": 1.015397494582351, "ci_hi": 1.0440821311671749, "adoption": 0, "mean_wall_s": 20.2247, "mean_input_tokens": 8547, "mean_output_tokens": 712, "mean_cache_tokens": 108942, "costs": [ 0.09548129999999999, 0.0953679, 0.09427529999999999, 0.09159840000000001, 0.09593249999999998, 0.09543089999999997, 0.09539129999999998, 0.09549239999999998, 0.09583229999999998, 0.0917742 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 5, 5, 4, 4, 5, 5, 5, 5, 5, 4 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-comprehension-django": { "n": 10, "successes": 10, "mean_cost": 0.18107346, "ratio": 0.47091260635511556, "ci_lo": 0.37096529167872894, "ci_hi": 0.6313514041138742, "adoption": 0, "mean_wall_s": 52.771300000000004, "mean_input_tokens": 16065, "mean_output_tokens": 3117, "mean_cache_tokens": 126381, "costs": [ 0.2090583, 0.18046079999999998, 0.1543269, 0.1780929, 0.20075340000000003, 0.2001138, 0.16370579999999998, 0.1320786, 0.22167630000000002, 0.1704678 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 16, 14, 12, 14, 13, 17, 13, 11, 16, 13 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-cascade-py": { "n": 10, "successes": 10, "mean_cost": 0.54445101, "ratio": 0.8207150197082034, "ci_lo": 0.5251178206041697, "ci_hi": 1.3000417163750015, "adoption": 0, "mean_wall_s": 183.7395, "mean_input_tokens": 29627, "mean_output_tokens": 12138, "mean_cache_tokens": 615242, "costs": [ 0.8625417, 0.3403595999999999, 0.4088901, 1.2011085000000001, 0.3946580999999999, 0.6199671000000001, 0.5045864999999999, 0.36323700000000003, 0.40431959999999995, 0.3448419 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 35, 26, 26, 58, 25, 36, 30, 26, 26, 25 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-ledger-py": { "n": 10, "successes": 10, "mean_cost": 0.34348238999999997, "ratio": 0.8710860458783838, "ci_lo": 0.6185151764821747, "ci_hi": 1.2263153864727898, "adoption": 0, "mean_wall_s": 127.9039, "mean_input_tokens": 21534, "mean_output_tokens": 8169, "mean_cache_tokens": 305689, "costs": [ 0.5077157999999999, 0.4291167, 0.297636, 0.19584780000000002, 0.2204754, 0.2552496, 0.5563431, 0.2112237, 0.5260992, 0.2351166 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 25, 28, 17, 17, 16, 18, 20, 14, 24, 17 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-pipeline-py": { "n": 10, "successes": 10, "mean_cost": 0.38606766000000003, "ratio": 0.9874602757569935, "ci_lo": 0.9086796018459035, "ci_hi": 1.0854737235053702, "adoption": 0, "mean_wall_s": 95.4896, "mean_input_tokens": 27198, "mean_output_tokens": 6156, "mean_cache_tokens": 434971, "costs": [ 0.38412, 0.4232424, 0.36693390000000004, 0.3579510000000001, 0.3764862, 0.3551127, 0.4073355, 0.4250004, 0.3898716, 0.3746229 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 24, 24, 23, 23, 24, 23, 24, 25, 24, 23 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-js": { "n": 10, "successes": 10, "mean_cost": 0.16810355999999999, "ratio": 1.1945193401447205, "ci_lo": 1.044147387444242, "ci_hi": 1.3648200925332221, "adoption": 0, "mean_wall_s": 52.9013, "mean_input_tokens": 10331, "mean_output_tokens": 2528, "mean_cache_tokens": 227200, "costs": [ 0.19153169999999997, 0.15003719999999998, 0.1588683, 0.1201863, 0.1821609, 0.20859389999999997, 0.1500837, 0.1844856, 0.17124509999999998, 0.16384289999999996 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 11, 10, 10, 7, 12, 13, 10, 12, 11, 11 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-validate-py": { "n": 10, "successes": 10, "mean_cost": 0.25082652, "ratio": 1.0484784749812586, "ci_lo": 0.8883729892562163, "ci_hi": 1.23119398532893, "adoption": 0, "mean_wall_s": 63.3031, "mean_input_tokens": 17677, "mean_output_tokens": 5035, "mean_cache_tokens": 230706, "costs": [ 0.29534610000000006, 0.211668, 0.27881429999999996, 0.2923122, 0.2019765, 0.20796869999999998, 0.3061878, 0.2057547, 0.3051617999999999, 0.2030751 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 40, 23, 26, 40, 23, 23, 40, 23, 40, 23 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-iterate-tests": { "n": 10, "successes": 10, "mean_cost": 0.13256484, "ratio": 1.0339801713305612, "ci_lo": 0.9949983634918409, "ci_hi": 1.0826862438478688, "adoption": 0, "mean_wall_s": 30.4271, "mean_input_tokens": 10841, "mean_output_tokens": 1719, "mean_cache_tokens": 139044, "costs": [ 0.1439895, 0.12217140000000001, 0.1297485, 0.1243995, 0.1296219, 0.129381, 0.12934919999999997, 0.1316712, 0.1548276, 0.1304886 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 10, 10, 10, 10, 10, 10, 10, 10, 11, 10 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py": { "n": 10, "successes": 10, "mean_cost": 0.25692792, "ratio": 1.3903910768347596, "ci_lo": 0.8816715712101157, "ci_hi": 2.136321285052673, "adoption": 0, "mean_wall_s": 48.5398, "mean_input_tokens": 26410, "mean_output_tokens": 3550, "mean_cache_tokens": 150634, "costs": [ 0.41644080000000006, 0.40177379999999996, 0.1304523, 0.3818262, 0.2575728, 0.1477653, 0.11507429999999999, 0.13208789999999998, 0.1751325, 0.4111533 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 43, 41, 6, 41, 27, 7, 5, 7, 8, 41 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py-xl": { "n": 10, "successes": 10, "mean_cost": 0.15274722, "ratio": 1.0414612768786273, "ci_lo": 0.9734378001136835, "ci_hi": 1.117428182420204, "adoption": 0, "mean_wall_s": 34.4242, "mean_input_tokens": 14485, "mean_output_tokens": 1687, "mean_cache_tokens": 135052, "costs": [ 0.14357609999999998, 0.1589292, 0.1570437, 0.13219799999999998, 0.1573872, 0.1551396, 0.1538556, 0.1645182, 0.1386795, 0.16614510000000002 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 6, 7, 7, 5, 7, 7, 6, 7, 7, 8 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-overview-cobra": { "n": 10, "successes": 10, "mean_cost": 0.270520935, "ratio": 0.8412289543835754, "ci_lo": 0.6027577759840294, "ci_hi": 1.1550949033502202, "adoption": 0, "mean_wall_s": 83.7572, "mean_input_tokens": 19103, "mean_output_tokens": 3799, "mean_cache_tokens": 281833, "costs": [ 0.4177647, 0.2355039, 0.2065893, 0.2138577, 0.26133330000000005, 0.41379960000000005, 0.2249469, 0.31045634999999994, 0.22964790000000002, 0.19130970000000003 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 26, 14, 15, 15, 12, 21, 14, 21, 13, 9 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-refactor-split-py": { "n": 10, "successes": 10, "mean_cost": 0.12124281, "ratio": 1.022257359045326, "ci_lo": 0.9659166953278068, "ci_hi": 1.0801470619168216, "adoption": 0, "mean_wall_s": 27.021900000000002, "mean_input_tokens": 9620, "mean_output_tokens": 1560, "mean_cache_tokens": 133645, "costs": [ 0.1248735, 0.1162467, 0.1235073, 0.117975, 0.11967300000000002, 0.12019559999999999, 0.10848630000000001, 0.1191258, 0.12421380000000001, 0.1381311 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 11, 11, 12, 11, 11, 12, 10, 11, 11, 13 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-settings-inventory-django": { "n": 10, "successes": 10, "mean_cost": 0.43705638, "ratio": 1.2508353829934489, "ci_lo": 0.9102193154761994, "ci_hi": 1.753330023347246, "adoption": 0, "mean_wall_s": 153.4327, "mean_input_tokens": 25076, "mean_output_tokens": 8864, "mean_cache_tokens": 511956, "costs": [ 0.47925540000000005, 0.7709652000000001, 0.35310660000000005, 0.5155434, 0.2687547, 0.4066557, 0.48631140000000006, 0.36573089999999997, 0.3552825, 0.36895800000000006 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 18, 30, 14, 20, 8, 13, 19, 13, 10, 14 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "log-needle-zh": { "n": 10, "successes": 10, "mean_cost": 0.13552464, "ratio": 1.109760315700933, "ci_lo": 0.9329401177063293, "ci_hi": 1.4053714001219084, "adoption": 0, "mean_wall_s": 33.4572, "mean_input_tokens": 13574, "mean_output_tokens": 1828, "mean_cache_tokens": 88812, "costs": [ 0.12471060000000002, 0.10464390000000001, 0.11359949999999999, 0.12059069999999998, 0.1165593, 0.131214, 0.1208583, 0.1271277, 0.2811693, 0.1147731 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 7, 7, 7, 7, 7, 7, 6, 7, 7, 6 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "report-pdf": { "n": 10, "successes": 10, "mean_cost": 0.31472661, "ratio": 1.1345014574245385, "ci_lo": 0.9078827409083696, "ci_hi": 1.416494607501456, "adoption": 0, "mean_wall_s": 129.2897, "mean_input_tokens": 19156, "mean_output_tokens": 9056, "mean_cache_tokens": 213094, "costs": [ 0.2638218, 0.37336860000000005, 0.5043033, 0.21366179999999999, 0.24335399999999996, 0.3063738000000001, 0.36394470000000007, 0.3383529, 0.27177599999999996, 0.26830919999999997 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 5, 13, 7, 5, 7, 6, 9, 8, 6, 6 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "seo-audit": { "n": 10, "successes": 10, "mean_cost": 0.25775568000000004, "ratio": 0.9735805338646719, "ci_lo": 0.9307748158724268, "ci_hi": 1.015404344869659, "adoption": 0, "mean_wall_s": 116.8721, "mean_input_tokens": 18802, "mean_output_tokens": 6830, "mean_cache_tokens": 141569, "costs": [ 0.25487760000000004, 0.2461698, 0.2520444, 0.2644383, 0.25608000000000003, 0.26349960000000006, 0.25098, 0.2688117, 0.27151800000000004, 0.2491374 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "web-research-oss-inventory": { "n": 10, "successes": 10, "mean_cost": 5.08661771, "ratio": 1.0160341946070437, "ci_lo": 0.9091031904962346, "ci_hi": 1.1295162602722622, "adoption": 0, "mean_wall_s": 991.7364, "mean_input_tokens": 105310, "mean_output_tokens": 27701, "mean_cache_tokens": 1044508, "costs": [ 5.625162699999999, 4.2797407, 6.1727762, 4.9166012, 4.4370168, 5.3235494999999995, 5.8631199999999986, 4.6177711000000015, 5.217225999999999, 4.4132129 ], "scores": [ 0.6667, 0.6491, 0.614, 0.5614, 0.4912, 0.6667, 0.5263, 0.4561, 0.5263, 0.5088 ], "tool_calls": [ 65, 54, 69, 56, 54, 59, 71, 51, 60, 55 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] } } }, "headroom": { "version": "0.27.0", "aggregate_cost_ratio": 1.5568980915065826, "aggregate_ci_lo": 1.3014624134696557, "aggregate_ci_hi": 1.897666681105211, "n_success": 170, "n_runs": 170, "tasks_compared": 17, "per_task": { "code-bugfix-py": { "n": 10, "successes": 10, "mean_cost": 0.12204522, "ratio": 1.328165636196124, "ci_lo": 1.296334846101126, "ci_hi": 1.376331704906677, "adoption": 0, "mean_wall_s": 27.240000000000002, "mean_input_tokens": 11419, "mean_output_tokens": 867, "mean_cache_tokens": 135017, "costs": [ 0.1196259, 0.1224237, 0.1195707, 0.11961269999999999, 0.1196586, 0.12162359999999998, 0.13944839999999997, 0.11961630000000001, 0.11941890000000002, 0.11945340000000002 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 6, 6, 6, 6, 6, 6, 8, 6, 6, 6 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-comprehension-django": { "n": 10, "successes": 10, "mean_cost": 0.3885963, "ratio": 1.0106113643211678, "ci_lo": 0.7870130960426592, "ci_hi": 1.3620780555494858, "adoption": 0, "mean_wall_s": 69.0812, "mean_input_tokens": 47768, "mean_output_tokens": 3972, "mean_cache_tokens": 141268, "costs": [ 0.5110983000000001, 0.33870900000000004, 0.4782873, 0.3695861999999999, 0.35173409999999994, 0.4077639, 0.45634620000000004, 0.4034349, 0.25825980000000004, 0.3107433 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 24, 22, 23, 22, 24, 20, 21, 21, 16, 20 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-cascade-py": { "n": 10, "successes": 10, "mean_cost": 0.91183809, "ratio": 1.3745207599211555, "ci_lo": 0.9114896590901944, "ci_hi": 2.1322077719437433, "adoption": 0, "mean_wall_s": 252.672, "mean_input_tokens": 77341, "mean_output_tokens": 16325, "mean_cache_tokens": 676190, "costs": [ 1.1693276999999997, 0.5144628, 0.6814743000000001, 1.4783691000000005, 0.6447543, 0.5092476, 1.0454246999999999, 0.7032324000000001, 1.6328664000000002, 0.7392216 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 42, 23, 33, 56, 30, 24, 33, 38, 53, 35 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-ledger-py": { "n": 10, "successes": 10, "mean_cost": 0.31330269, "ratio": 0.7945490346540359, "ci_lo": 0.529812897218746, "ci_hi": 1.267551523577653, "adoption": 0, "mean_wall_s": 95.8716, "mean_input_tokens": 22361, "mean_output_tokens": 5342, "mean_cache_tokens": 329872, "costs": [ 0.26470289999999996, 0.9191505, 0.2637381, 0.20592210000000002, 0.2359962, 0.26519789999999993, 0.21360869999999998, 0.2244624, 0.27063509999999996, 0.269613 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 18, 30, 18, 18, 19, 18, 14, 14, 18, 18 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-pipeline-py": { "n": 10, "successes": 10, "mean_cost": 0.6028787099999999, "ratio": 1.542006334393874, "ci_lo": 1.256257799641565, "ci_hi": 1.8643816033369782, "adoption": 0, "mean_wall_s": 130.6033, "mean_input_tokens": 61632, "mean_output_tokens": 7534, "mean_cache_tokens": 400080, "costs": [ 0.9265907999999999, 0.42225149999999995, 0.5369334, 0.8104719, 0.563118, 0.5753088, 0.37013909999999994, 0.8018919, 0.47721810000000003, 0.5448636 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 22, 23, 19, 25, 20, 19, 24, 24, 20, 20 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-js": { "n": 10, "successes": 10, "mean_cost": 0.1973169, "ratio": 1.4021050665875359, "ci_lo": 1.2363877597427615, "ci_hi": 1.5931241611734974, "adoption": 0, "mean_wall_s": 58.1406, "mean_input_tokens": 14392, "mean_output_tokens": 2679, "mean_cache_tokens": 235812, "costs": [ 0.15638549999999998, 0.2101905, 0.20384909999999998, 0.24160200000000004, 0.20902890000000002, 0.20844989999999997, 0.199572, 0.1731459, 0.20416020000000001, 0.166785 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 9, 11, 13, 14, 14, 14, 13, 12, 12, 11 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-validate-py": { "n": 10, "successes": 10, "mean_cost": 0.31914111, "ratio": 1.3340398946515946, "ci_lo": 1.1710081233916938, "ci_hi": 1.5197572517270679, "adoption": 0, "mean_wall_s": 63.7269, "mean_input_tokens": 30066, "mean_output_tokens": 4270, "mean_cache_tokens": 248856, "costs": [ 0.30348329999999996, 0.2930223, 0.30995189999999995, 0.3050913, 0.3586014, 0.3551541, 0.2866203, 0.3056679, 0.29863080000000003, 0.37518779999999996 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 24, 24, 24, 24, 24, 41, 25, 24, 24, 26 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-iterate-tests": { "n": 10, "successes": 10, "mean_cost": 0.16779518999999998, "ratio": 1.308770103027651, "ci_lo": 1.2883421111957407, "ci_hi": 1.3277986831723652, "adoption": 0, "mean_wall_s": 37.8703, "mean_input_tokens": 15896, "mean_output_tokens": 1731, "mean_cache_tokens": 154767, "costs": [ 0.16971540000000002, 0.1736898, 0.16285679999999997, 0.1697052, 0.1685643, 0.17003519999999997, 0.1618206, 0.16917120000000002, 0.16895099999999996, 0.1634424 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 11, 11, 11, 11, 11, 11, 11, 11, 11, 11 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py": { "n": 10, "successes": 10, "mean_cost": 0.49654737, "ratio": 2.6871156411252146, "ci_lo": 1.8736072351542867, "ci_hi": 3.850893496980862, "adoption": 0, "mean_wall_s": 86.8913, "mean_input_tokens": 53888, "mean_output_tokens": 6617, "mean_cache_tokens": 246446, "costs": [ 0.4133826, 0.6814506, 0.7174668, 0.3907755, 0.4536084, 0.5073825, 0.43762349999999994, 0.48612389999999994, 0.6498882000000001, 0.22777170000000002 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 34, 47, 45, 30, 34, 36, 34, 36, 49, 11 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py-xl": { "n": 10, "successes": 10, "mean_cost": 0.25693116, "ratio": 1.7518083403646032, "ci_lo": 1.5364119556660198, "ci_hi": 1.9690354295390298, "adoption": 0, "mean_wall_s": 62.0362, "mean_input_tokens": 23363, "mean_output_tokens": 3150, "mean_cache_tokens": 231579, "costs": [ 0.28210769999999996, 0.24625650000000002, 0.2943366, 0.2386875, 0.1706163, 0.32614229999999994, 0.26639219999999997, 0.30317669999999997, 0.18681090000000003, 0.25478490000000004 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 13, 16, 18, 14, 6, 19, 13, 15, 10, 15 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-overview-cobra": { "n": 10, "successes": 10, "mean_cost": 0.484414065, "ratio": 1.5063645162569297, "ci_lo": 1.0647364953090062, "ci_hi": 2.1036208629242004, "adoption": 0, "mean_wall_s": 124.6617, "mean_input_tokens": 32336, "mean_output_tokens": 3808, "mean_cache_tokens": 277588, "costs": [ 0.3334401, 0.4048044, 0.38136659999999994, 0.40981709999999993, 0.58533315, 0.37921560000000004, 0.8311436999999999, 0.37468110000000004, 0.7023426000000002, 0.4419963 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 21, 16, 16, 28, 26, 20, 65, 22, 44, 28 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-refactor-split-py": { "n": 10, "successes": 10, "mean_cost": 0.17014515, "ratio": 1.434576876710222, "ci_lo": 1.338312690628726, "ci_hi": 1.5401984686420007, "adoption": 0, "mean_wall_s": 32.3875, "mean_input_tokens": 17310, "mean_output_tokens": 1725, "mean_cache_tokens": 134622, "costs": [ 0.1466961, 0.1651137, 0.1718748, 0.1568964, 0.16539569999999998, 0.1634937, 0.174708, 0.18576389999999998, 0.1638402, 0.207669 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 11, 11, 15, 12, 11, 14, 13, 13, 11, 12 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-settings-inventory-django": { "n": 10, "successes": 10, "mean_cost": 0.51900876, "ratio": 1.4853793487502802, "ci_lo": 1.003840480741266, "ci_hi": 2.171873164472317, "adoption": 0, "mean_wall_s": 187.8493, "mean_input_tokens": 29329, "mean_output_tokens": 10818, "mean_cache_tokens": 556966, "costs": [ 0.43549949999999993, 0.32546070000000005, 0.474813, 0.3537038999999999, 0.6290382000000001, 0.2792442, 0.9724605, 0.7432542, 0.23841030000000002, 0.7382031000000001 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 0.9945, 1.0, 0.9758, 1.0, 1.0 ], "tool_calls": [ 21, 13, 18, 15, 26, 10, 31, 20, 7, 20 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "log-needle-zh": { "n": 10, "successes": 10, "mean_cost": 0.32920614, "ratio": 2.695745289248402, "ci_lo": 2.025873604800876, "ci_hi": 3.3957474168021324, "adoption": 0, "mean_wall_s": 59.1319, "mean_input_tokens": 37352, "mean_output_tokens": 2929, "mean_cache_tokens": 203761, "costs": [ 0.2161911, 0.4259328, 0.4183107, 0.5355825, 0.1570533, 0.4746162, 0.20320919999999998, 0.4208574, 0.23438040000000002, 0.2059278 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 10, 18, 16, 21, 12, 20, 11, 23, 15, 13 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "report-pdf": { "n": 10, "successes": 10, "mean_cost": 0.28243542, "ratio": 1.0181007434303428, "ci_lo": 0.8523531849384471, "ci_hi": 1.2166606833259965, "adoption": 0, "mean_wall_s": 105.6988, "mean_input_tokens": 19339, "mean_output_tokens": 7624, "mean_cache_tokens": 173402, "costs": [ 0.2301933, 0.2868147, 0.24028589999999994, 0.2569308, 0.3111165000000001, 0.23959620000000004, 0.33671399999999996, 0.300669, 0.3023754, 0.3196584 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 6, 9, 5, 7, 6, 7, 8, 6, 7, 8 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "seo-audit": { "n": 10, "successes": 10, "mean_cost": 0.55213197, "ratio": 2.0854824154267053, "ci_lo": 1.757860442878258, "ci_hi": 2.407392632568533, "adoption": 0, "mean_wall_s": 176.1842, "mean_input_tokens": 49378, "mean_output_tokens": 9992, "mean_cache_tokens": 353118, "costs": [ 0.3451263, 0.5296893, 0.6440291999999999, 0.6191118, 0.3919026, 0.7436928000000003, 0.7207941000000001, 0.5182167000000001, 0.6280644000000001, 0.38069250000000004 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 19, 25, 32, 22, 25, 40, 25, 20, 30, 19 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "web-research-oss-inventory": { "n": 10, "successes": 10, "mean_cost": 19.4086909, "ratio": 3.8768184973268927, "ci_lo": 3.4206870417426414, "ci_hi": 4.440804114232501, "adoption": 0, "mean_wall_s": 1799.6076, "mean_input_tokens": 1458388, "mean_output_tokens": 32924, "mean_cache_tokens": 807788, "costs": [ 18.818737900000002, 14.7389789, 17.677915000000002, 18.956084800000003, 28.3925219, 19.251713600000002, 19.426207100000006, 20.814038200000013, 19.34080759999998, 16.669904000000002 ], "scores": [ 0.5263, 0.4737, 0.4912, 0.5965, 0.5439, 0.5088, 0.5614, 0.5789, 0.6316, 0.6316 ], "tool_calls": [ 144, 127, 135, 149, 176, 148, 139, 157, 149, 119 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] } } }, "lean-ctx": { "version": "3.8.4", "aggregate_cost_ratio": 1.2342724885887937, "aggregate_ci_lo": 1.0417273386923387, "aggregate_ci_hi": 1.4559856011249679, "n_success": 170, "n_runs": 170, "tasks_compared": 17, "per_task": { "code-bugfix-py": { "n": 10, "successes": 10, "mean_cost": 0.15164826, "ratio": 1.6503227879054603, "ci_lo": 1.5096497484239642, "ci_hi": 1.7817072494751334, "adoption": 9, "mean_wall_s": 29.5335, "mean_input_tokens": 11493, "mean_output_tokens": 1158, "mean_cache_tokens": 217612, "costs": [ 0.14090850000000002, 0.16791209999999998, 0.1408965, 0.1756431, 0.1394397, 0.1085604, 0.1756638, 0.1643181, 0.138651, 0.16448940000000004 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 8, 10, 8, 10, 8, 5, 10, 9, 8, 9 ], "competitor_tool_calls": [ 4, 4, 4, 4, 4, 0, 4, 5, 4, 5 ] }, "code-comprehension-django": { "n": 10, "successes": 10, "mean_cost": 0.23645532, "ratio": 0.6149426372464131, "ci_lo": 0.49265385512764354, "ci_hi": 0.8287634703067023, "adoption": 10, "mean_wall_s": 66.3269, "mean_input_tokens": 19472, "mean_output_tokens": 3751, "mean_cache_tokens": 211023, "costs": [ 0.23559779999999997, 0.21525120000000003, 0.24877500000000002, 0.25046160000000006, 0.22343789999999997, 0.21670289999999998, 0.23708970000000001, 0.25014600000000003, 0.26440529999999995, 0.22268580000000002 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 20, 15, 17, 22, 20, 16, 19, 18, 20, 19 ], "competitor_tool_calls": [ 16, 13, 13, 20, 16, 12, 15, 14, 16, 15 ] }, "code-debug-cascade-py": { "n": 10, "successes": 10, "mean_cost": 0.57000507, "ratio": 0.8592356587947663, "ci_lo": 0.5631117476884974, "ci_hi": 1.3573407948732548, "adoption": 10, "mean_wall_s": 187.4643, "mean_input_tokens": 29048, "mean_output_tokens": 11462, "mean_cache_tokens": 745670, "costs": [ 0.36995160000000005, 0.6300363, 0.8134965000000001, 0.5727921, 0.3732825, 0.37770089999999995, 0.4291902000000001, 1.2601116, 0.5076881999999999, 0.3658008 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 32, 37, 48, 33, 29, 31, 37, 49, 35, 29 ], "competitor_tool_calls": [ 12, 17, 24, 13, 11, 11, 17, 27, 17, 11 ] }, "code-debug-ledger-py": { "n": 10, "successes": 10, "mean_cost": 0.32643744, "ratio": 0.8278593229663453, "ci_lo": 0.5585983533649332, "ci_hi": 1.296969670520467, "adoption": 10, "mean_wall_s": 90.6248, "mean_input_tokens": 19444, "mean_output_tokens": 5407, "mean_cache_tokens": 428672, "costs": [ 0.23778330000000006, 0.2775102, 0.28511430000000004, 0.237267, 0.2700921, 0.2725761, 0.281346, 0.9258798000000001, 0.22438439999999998, 0.2524212 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 14, 29, 18, 19, 18, 26, 26, 46, 21, 21 ], "competitor_tool_calls": [ 11, 17, 13, 9, 13, 22, 22, 26, 9, 9 ] }, "code-debug-pipeline-py": { "n": 10, "successes": 10, "mean_cost": 0.41346831, "ratio": 1.0575439844129344, "ci_lo": 0.9394948246699595, "ci_hi": 1.19216122180297, "adoption": 10, "mean_wall_s": 112.4627, "mean_input_tokens": 25098, "mean_output_tokens": 6898, "mean_cache_tokens": 531159, "costs": [ 0.5145198, 0.3139533, 0.4000863, 0.34930829999999996, 0.4692453, 0.4212195000000001, 0.44754810000000006, 0.43872539999999993, 0.4324089, 0.3476681999999999 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 33, 32, 28, 27, 29, 24, 27, 32, 26, 27 ], "competitor_tool_calls": [ 18, 17, 25, 10, 14, 20, 12, 17, 11, 10 ] }, "code-feature-js": { "n": 10, "successes": 10, "mean_cost": 0.21477996, "ratio": 1.5261950198764944, "ci_lo": 1.271745517072805, "ci_hi": 1.8009512179156837, "adoption": 5, "mean_wall_s": 60.6034, "mean_input_tokens": 12224, "mean_output_tokens": 3281, "mean_cache_tokens": 307258, "costs": [ 0.292329, 0.24443429999999997, 0.2529954, 0.282381, 0.21614640000000002, 0.21274860000000004, 0.1894293, 0.15012540000000002, 0.14109839999999998, 0.1661118 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 16, 16, 15, 18, 14, 12, 10, 8, 7, 8 ], "competitor_tool_calls": [ 8, 6, 7, 6, 6, 0, 0, 0, 0, 0 ] }, "code-feature-validate-py": { "n": 10, "successes": 10, "mean_cost": 0.31988514, "ratio": 1.337150010120008, "ci_lo": 1.1607249837580307, "ci_hi": 1.54094356146845, "adoption": 10, "mean_wall_s": 79.6361, "mean_input_tokens": 20489, "mean_output_tokens": 5790, "mean_cache_tokens": 366772, "costs": [ 0.4029039, 0.2782467, 0.32744639999999997, 0.3090612000000001, 0.2975562, 0.37285500000000005, 0.31259639999999994, 0.30326849999999994, 0.26021730000000004, 0.3346998 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 43, 25, 43, 25, 42, 37, 28, 34, 26, 27 ], "competitor_tool_calls": [ 22, 22, 14, 22, 14, 23, 23, 21, 22, 23 ] }, "code-iterate-tests": { "n": 10, "successes": 10, "mean_cost": 0.20928525, "ratio": 1.6323845648058668, "ci_lo": 1.5434347401335944, "ci_hi": 1.7161577137229422, "adoption": 10, "mean_wall_s": 44.0373, "mean_input_tokens": 14359, "mean_output_tokens": 2334, "mean_cache_tokens": 293548, "costs": [ 0.228342, 0.23641170000000003, 0.2029596, 0.22565970000000002, 0.2166807, 0.17705550000000003, 0.19436969999999998, 0.21986459999999997, 0.19591440000000002, 0.1955946 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 13, 15, 14, 13, 19, 15, 14, 16, 14, 14 ], "competitor_tool_calls": [ 9, 10, 6, 9, 7, 7, 10, 9, 10, 10 ] }, "code-migration-py": { "n": 10, "successes": 10, "mean_cost": 0.34596567, "ratio": 1.872227745661737, "ci_lo": 1.1459342928280805, "ci_hi": 2.925915779887378, "adoption": 10, "mean_wall_s": 66.0096, "mean_input_tokens": 32324, "mean_output_tokens": 4535, "mean_cache_tokens": 279787, "costs": [ 0.6388164000000001, 0.229854, 0.1602288, 0.26312519999999995, 0.17808659999999998, 0.168027, 0.5294256, 0.5893488, 0.5112351, 0.1915092 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 54, 12, 8, 15, 8, 9, 47, 47, 45, 10 ], "competitor_tool_calls": [ 40, 9, 7, 11, 5, 8, 7, 8, 31, 7 ] }, "code-migration-py-xl": { "n": 10, "successes": 10, "mean_cost": 0.19063341, "ratio": 1.2997769425481323, "ci_lo": 1.1904179679412987, "ci_hi": 1.431334637135199, "adoption": 10, "mean_wall_s": 46.3773, "mean_input_tokens": 14077, "mean_output_tokens": 2428, "mean_cache_tokens": 232366, "costs": [ 0.1668999, 0.17405729999999997, 0.1736949, 0.2449638, 0.18390510000000004, 0.17634059999999996, 0.2098443, 0.19066319999999998, 0.1724019, 0.21356309999999998 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 10, 8, 9, 13, 10, 9, 13, 11, 8, 11 ], "competitor_tool_calls": [ 7, 7, 8, 9, 7, 7, 10, 8, 7, 9 ] }, "code-overview-cobra": { "n": 10, "successes": 10, "mean_cost": 0.33256767, "ratio": 1.03417339325654, "ci_lo": 0.7839008543775615, "ci_hi": 1.3510217171044647, "adoption": 10, "mean_wall_s": 91.0277, "mean_input_tokens": 26314, "mean_output_tokens": 4030, "mean_cache_tokens": 380600, "costs": [ 0.31287960000000004, 0.3631824, 0.29595209999999994, 0.3289287, 0.32591220000000004, 0.32431709999999997, 0.310515, 0.3209358, 0.35368920000000004, 0.3893646 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 16, 15, 15, 17, 15, 17, 16, 17, 15, 16 ], "competitor_tool_calls": [ 12, 11, 11, 13, 10, 13, 12, 13, 11, 11 ] }, "code-refactor-split-py": { "n": 10, "successes": 10, "mean_cost": 0.14023428, "ratio": 1.1823837200772795, "ci_lo": 1.1307810158185192, "ci_hi": 1.2364392918752578, "adoption": 10, "mean_wall_s": 29.7117, "mean_input_tokens": 11315, "mean_output_tokens": 1707, "mean_cache_tokens": 155690, "costs": [ 0.14695680000000003, 0.14423640000000001, 0.14075189999999999, 0.1396641, 0.1387926, 0.13805099999999998, 0.13883759999999998, 0.137799, 0.14005379999999998, 0.1371996 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 13, 13, 12, 12, 12, 12, 12, 12, 12, 12 ], "competitor_tool_calls": [ 6, 6, 5, 5, 5, 5, 5, 5, 5, 5 ] }, "code-settings-inventory-django": { "n": 10, "successes": 10, "mean_cost": 0.7429319400000001, "ratio": 2.1262372550378195, "ci_lo": 1.3790361518353156, "ci_hi": 3.187528122363402, "adoption": 10, "mean_wall_s": 260.9767, "mean_input_tokens": 36290, "mean_output_tokens": 16389, "mean_cache_tokens": 930883, "costs": [ 0.5953917, 0.9386535000000001, 1.2449328000000002, 0.34847880000000003, 1.4851146000000002, 0.7505391000000002, 0.8294937000000001, 0.5513328, 0.2267655, 0.4586169 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 0.9758, 1.0, 0.9972, 1.0, 1.0, 1.0 ], "tool_calls": [ 18, 31, 31, 11, 36, 29, 33, 23, 6, 18 ], "competitor_tool_calls": [ 13, 22, 24, 6, 28, 26, 23, 18, 4, 13 ] }, "log-needle-zh": { "n": 10, "successes": 10, "mean_cost": 0.20773869, "ratio": 1.7010940165397102, "ci_lo": 1.5774514569625673, "ci_hi": 1.8443977689986428, "adoption": 10, "mean_wall_s": 55.388799999999996, "mean_input_tokens": 15721, "mean_output_tokens": 2741, "mean_cache_tokens": 240799, "costs": [ 0.19135049999999998, 0.21299160000000003, 0.2225319, 0.21413880000000002, 0.21644219999999997, 0.18273810000000001, 0.1926156, 0.25888769999999994, 0.18888299999999997, 0.19680749999999997 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 10, 12, 18, 16, 16, 14, 15, 17, 10, 16 ], "competitor_tool_calls": [ 7, 8, 15, 12, 13, 11, 12, 14, 7, 12 ] }, "report-pdf": { "n": 10, "successes": 10, "mean_cost": 0.29366091, "ratio": 1.058565497158363, "ci_lo": 0.8816842564385959, "ci_hi": 1.266809127239292, "adoption": 10, "mean_wall_s": 111.422, "mean_input_tokens": 17271, "mean_output_tokens": 7844, "mean_cache_tokens": 241103, "costs": [ 0.27718709999999996, 0.36414749999999996, 0.38293440000000006, 0.2899359, 0.28028580000000003, 0.2612058, 0.27735630000000006, 0.29611770000000004, 0.2811156, 0.226323 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 10, 12, 14, 9, 9, 9, 8, 10, 8, 9 ], "competitor_tool_calls": [ 6, 8, 8, 7, 7, 6, 6, 8, 6, 7 ] }, "seo-audit": { "n": 10, "successes": 10, "mean_cost": 0.36541040999999996, "ratio": 1.3802080406046089, "ci_lo": 1.276454307171404, "ci_hi": 1.4989406031209762, "adoption": 10, "mean_wall_s": 158.3381, "mean_input_tokens": 26792, "mean_output_tokens": 9472, "mean_cache_tokens": 208452, "costs": [ 0.34184220000000004, 0.3379743, 0.42849479999999995, 0.3597027, 0.3743418, 0.3562032, 0.3259806, 0.3398328, 0.33303059999999995, 0.45670109999999997 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 17, 17, 23, 20, 17, 17, 16, 17, 18, 19 ], "competitor_tool_calls": [ 13, 13, 19, 16, 12, 13, 13, 13, 14, 15 ] }, "web-research-oss-inventory": { "n": 10, "successes": 10, "mean_cost": 4.482739980000001, "ratio": 0.8954117185291867, "ci_lo": 0.7813716883952722, "ci_hi": 1.0235530372524968, "adoption": 0, "mean_wall_s": 914.0518, "mean_input_tokens": 93141, "mean_output_tokens": 26587, "mean_cache_tokens": 896119, "costs": [ 3.7291223000000002, 4.723339400000002, 3.9178272000000005, 3.7959511, 4.0965093, 5.8098447, 4.724133199999999, 6.022818700000002, 3.8534299000000005, 4.154424 ], "scores": [ 0.6316, 0.5965, 0.6316, 0.6491, 0.5789, 0.5789, 0.5965, 0.5088, 0.5088, 0.6842 ], "tool_calls": [ 48, 55, 50, 49, 47, 63, 52, 66, 42, 49 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] } } }, "ponytail": { "version": "main", "aggregate_cost_ratio": 0.9576172154231043, "aggregate_ci_lo": 0.8493117124279771, "aggregate_ci_hi": 1.0654785686813864, "n_success": 170, "n_runs": 170, "tasks_compared": 17, "per_task": { "code-bugfix-py": { "n": 10, "successes": 10, "mean_cost": 0.10500654000000001, "ratio": 1.1427410102899054, "ci_lo": 1.1318732549733663, "ci_hi": 1.1545410000072547, "adoption": 0, "mean_wall_s": 17.5171, "mean_input_tokens": 10153, "mean_output_tokens": 624, "mean_cache_tokens": 115713, "costs": [ 0.1050831, 0.10483830000000001, 0.1049325, 0.1049892, 0.10503330000000001, 0.10537800000000001, 0.1049133, 0.10501230000000002, 0.1048902, 0.10499520000000001 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-comprehension-django": { "n": 10, "successes": 10, "mean_cost": 0.21631479, "ratio": 0.5625637327085896, "ci_lo": 0.37903437246846544, "ci_hi": 0.8782853884878703, "adoption": 0, "mean_wall_s": 42.327799999999996, "mean_input_tokens": 16916, "mean_output_tokens": 2702, "mean_cache_tokens": 105131, "costs": [ 0.5972594999999999, 0.16496910000000004, 0.1708353, 0.14419530000000003, 0.19952729999999996, 0.1535694, 0.1830261, 0.1836816, 0.191838, 0.1742463 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 24, 12, 13, 11, 15, 11, 13, 13, 15, 13 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-cascade-py": { "n": 10, "successes": 10, "mean_cost": 0.49615743, "ratio": 0.7479164285889038, "ci_lo": 0.48547724415607, "ci_hi": 1.1903934731440975, "adoption": 0, "mean_wall_s": 153.2738, "mean_input_tokens": 28162, "mean_output_tokens": 9704, "mean_cache_tokens": 605266, "costs": [ 0.5703192, 0.34555409999999986, 0.34636680000000003, 0.37365929999999997, 0.36272489999999996, 0.38198430000000005, 1.1122650000000003, 0.3714051, 0.7322660999999998, 0.3650294999999999 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 28, 25, 26, 29, 26, 25, 37, 27, 38, 26 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-ledger-py": { "n": 10, "successes": 10, "mean_cost": 0.39059052, "ratio": 0.990554280306428, "ci_lo": 0.6782639336040095, "ci_hi": 1.4264676381783705, "adoption": 0, "mean_wall_s": 137.0292, "mean_input_tokens": 23560, "mean_output_tokens": 8525, "mean_cache_tokens": 404398, "costs": [ 0.49434930000000005, 0.3216213, 0.1863465, 0.18480780000000002, 0.21270869999999997, 0.5757801, 0.5431566, 0.46498169999999994, 0.22644840000000002, 0.6957048 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 23, 20, 14, 13, 14, 21, 21, 18, 15, 29 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-pipeline-py": { "n": 10, "successes": 10, "mean_cost": 0.39283281, "ratio": 1.0047637631419182, "ci_lo": 0.9250284040009217, "ci_hi": 1.1051008121923316, "adoption": 0, "mean_wall_s": 98.258, "mean_input_tokens": 28269, "mean_output_tokens": 6097, "mean_cache_tokens": 439076, "costs": [ 0.38270039999999994, 0.41057280000000007, 0.35756999999999994, 0.36319979999999996, 0.3848295, 0.3845964, 0.42171539999999996, 0.3981591, 0.4500642, 0.3749205 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 23, 24, 23, 23, 24, 23, 23, 23, 23, 23 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-js": { "n": 10, "successes": 10, "mean_cost": 0.15621018, "ratio": 1.1100067192954632, "ci_lo": 0.952395465626579, "ci_hi": 1.2918127770648227, "adoption": 0, "mean_wall_s": 38.2786, "mean_input_tokens": 11478, "mean_output_tokens": 1827, "mean_cache_tokens": 199707, "costs": [ 0.1660149, 0.1489578, 0.16649880000000003, 0.12159239999999998, 0.18842820000000002, 0.1651203, 0.13497779999999998, 0.10574489999999999, 0.15831330000000002, 0.2064534 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 9, 8, 9, 6, 11, 9, 7, 5, 8, 11 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-validate-py": { "n": 10, "successes": 10, "mean_cost": 0.27900345, "ratio": 1.1662606959204704, "ci_lo": 1.0037781142436626, "ci_hi": 1.3472577610749996, "adoption": 0, "mean_wall_s": 60.978, "mean_input_tokens": 19564, "mean_output_tokens": 4830, "mean_cache_tokens": 297142, "costs": [ 0.3487734, 0.3070347, 0.2097609, 0.2552343, 0.311223, 0.27863189999999993, 0.23875770000000002, 0.2920197, 0.23478870000000002, 0.31381020000000004 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 42, 35, 16, 24, 40, 16, 25, 27, 25, 40 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-iterate-tests": { "n": 10, "successes": 10, "mean_cost": 0.14731097999999998, "ratio": 1.1489972178088312, "ci_lo": 1.105195542020177, "ci_hi": 1.1942135136820906, "adoption": 0, "mean_wall_s": 32.5856, "mean_input_tokens": 12539, "mean_output_tokens": 1506, "mean_cache_tokens": 164858, "costs": [ 0.14271029999999998, 0.15645599999999998, 0.15733529999999998, 0.1432812, 0.14040929999999996, 0.1414479, 0.1320174, 0.15847109999999998, 0.1585383, 0.142443 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 10, 10, 10, 10, 10, 10, 10, 10, 10, 10 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py": { "n": 10, "successes": 10, "mean_cost": 0.13549131, "ratio": 0.7332247441720318, "ci_lo": 0.5350620955236893, "ci_hi": 1.0006033853022551, "adoption": 0, "mean_wall_s": 44.0681, "mean_input_tokens": 12901, "mean_output_tokens": 1232, "mean_cache_tokens": 128291, "costs": [ 0.14361809999999997, 0.1467042, 0.1346847, 0.1398879, 0.13426979999999997, 0.1205325, 0.12502080000000002, 0.1288434, 0.14692650000000002, 0.1344252 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 6, 6, 7, 6, 6, 5, 5, 5, 6, 6 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py-xl": { "n": 10, "successes": 10, "mean_cost": 0.14823618, "ratio": 1.0107040985911893, "ci_lo": 0.9472270042938105, "ci_hi": 1.0850213579087784, "adoption": 0, "mean_wall_s": 28.4527, "mean_input_tokens": 14096, "mean_output_tokens": 1363, "mean_cache_tokens": 144014, "costs": [ 0.1429476, 0.1399095, 0.1485522, 0.1608399, 0.14931719999999998, 0.13546319999999998, 0.16789679999999998, 0.15809820000000002, 0.1367889, 0.1425483 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 6, 5, 6, 8, 6, 5, 6, 7, 5, 7 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-overview-cobra": { "n": 10, "successes": 10, "mean_cost": 0.26854932, "ratio": 0.8350978960797254, "ci_lo": 0.6312215368331918, "ci_hi": 1.10348686439534, "adoption": 0, "mean_wall_s": 73.94, "mean_input_tokens": 19325, "mean_output_tokens": 3009, "mean_cache_tokens": 251741, "costs": [ 0.31857240000000003, 0.2087766, 0.25195320000000004, 0.2611848, 0.2369409, 0.24549149999999997, 0.2687979, 0.3081891, 0.3380832, 0.24750359999999996 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 21, 11, 10, 13, 14, 16, 18, 20, 18, 16 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-refactor-split-py": { "n": 10, "successes": 10, "mean_cost": 0.1296105, "ratio": 1.0928094411086664, "ci_lo": 1.0375941569895744, "ci_hi": 1.1494868876429747, "adoption": 0, "mean_wall_s": 21.7381, "mean_input_tokens": 11436, "mean_output_tokens": 1419, "mean_cache_tokens": 132319, "costs": [ 0.133821, 0.13112490000000002, 0.1221186, 0.1280235, 0.13566060000000002, 0.1210206, 0.1336206, 0.13828769999999999, 0.13127909999999998, 0.1211484 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 11, 11, 10, 11, 11, 10, 11, 12, 11, 10 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-settings-inventory-django": { "n": 10, "successes": 10, "mean_cost": 0.40681098, "ratio": 1.1642744306220636, "ci_lo": 0.7569758817829382, "ci_hi": 1.7734343808694149, "adoption": 0, "mean_wall_s": 131.4989, "mean_input_tokens": 23440, "mean_output_tokens": 7221, "mean_cache_tokens": 525999, "costs": [ 0.20887529999999999, 0.35644289999999995, 0.44702790000000003, 0.6915903, 0.33265859999999997, 0.3567251999999999, 0.37273470000000003, 0.1861833, 0.8933985, 0.22247310000000003 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 5, 15, 17, 29, 12, 10, 17, 8, 42, 7 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "log-needle-zh": { "n": 10, "successes": 10, "mean_cost": 0.14538342, "ratio": 1.1904901579290774, "ci_lo": 1.009387052215463, "ci_hi": 1.4990682345961586, "adoption": 0, "mean_wall_s": 27.0105, "mean_input_tokens": 15335, "mean_output_tokens": 1301, "mean_cache_tokens": 112811, "costs": [ 0.1256832, 0.1368378, 0.12751200000000001, 0.1249971, 0.1262556, 0.1331082, 0.1262934, 0.2969607, 0.14003700000000002, 0.11614920000000001 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "report-pdf": { "n": 10, "successes": 10, "mean_cost": 0.21940352999999999, "ratio": 0.7908883985027144, "ci_lo": 0.6627484920949936, "ci_hi": 0.9402989339322119, "adoption": 0, "mean_wall_s": 75.9594, "mean_input_tokens": 14732, "mean_output_tokens": 5345, "mean_cache_tokens": 169376, "costs": [ 0.2131533, 0.1813281, 0.2039622, 0.22000619999999999, 0.2791341, 0.20673959999999997, 0.2009154, 0.21044279999999999, 0.22485480000000002, 0.25349879999999997 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 5, 5, 5, 5, 12, 5, 5, 6, 5, 9 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "seo-audit": { "n": 10, "successes": 10, "mean_cost": 0.26061855, "ratio": 0.9843940084813521, "ci_lo": 0.9373430038884705, "ci_hi": 1.031758179979396, "adoption": 0, "mean_wall_s": 108.1204, "mean_input_tokens": 20400, "mean_output_tokens": 6192, "mean_cache_tokens": 151080, "costs": [ 0.2581581, 0.26811569999999996, 0.2581485, 0.2548743, 0.2831553, 0.2676765, 0.237939, 0.2527161, 0.2721147, 0.2532873 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 15, 14, 14, 14, 14, 14, 4, 14, 14, 14 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "web-research-oss-inventory": { "n": 10, "successes": 10, "mean_cost": 4.61551802, "ratio": 0.921933670172551, "ci_lo": 0.8264054831225789, "ci_hi": 1.030884340077378, "adoption": 0, "mean_wall_s": 931.6774, "mean_input_tokens": 100204, "mean_output_tokens": 28136, "mean_cache_tokens": 895657, "costs": [ 4.317198300000001, 4.0146659, 4.0159006, 5.4170557, 4.476827800000001, 4.2818717, 4.7310706, 4.8656047, 4.1991583, 5.835826599999998 ], "scores": [ 0.6491, 0.5614, 0.6667, 0.614, 0.6491, 0.5965, 0.4561, 0.5614, 0.614, 0.5439 ], "tool_calls": [ 42, 49, 42, 67, 54, 50, 52, 60, 49, 65 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] } } }, "rtk": { "version": "v0.42.3", "aggregate_cost_ratio": 1.0521527335785215, "aggregate_ci_lo": 0.9134305001515856, "aggregate_ci_hi": 1.1895371095370226, "n_success": 170, "n_runs": 170, "tasks_compared": 17, "per_task": { "code-bugfix-py": { "n": 10, "successes": 10, "mean_cost": 0.09638142, "ratio": 1.048877539094, "ci_lo": 1.0379049817633756, "ci_hi": 1.0602732359456744, "adoption": 0, "mean_wall_s": 17.5741, "mean_input_tokens": 8772, "mean_output_tokens": 720, "mean_cache_tokens": 109771, "costs": [ 0.09632009999999998, 0.09649319999999999, 0.09686099999999999, 0.09648390000000001, 0.09638909999999999, 0.09660959999999999, 0.0970278, 0.09681269999999999, 0.0962562, 0.0945606 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 5, 5, 5, 5, 5, 5, 5, 5, 5, 4 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-comprehension-django": { "n": 10, "successes": 10, "mean_cost": 0.17378202, "ratio": 0.4519499653668562, "ci_lo": 0.3587440395949794, "ci_hi": 0.6090585036532755, "adoption": 0, "mean_wall_s": 50.4375, "mean_input_tokens": 15364, "mean_output_tokens": 3136, "mean_cache_tokens": 115142, "costs": [ 0.15646439999999998, 0.18383579999999997, 0.19813830000000002, 0.19958370000000003, 0.1734192, 0.1692555, 0.2059434, 0.1566525, 0.1538871, 0.1406403 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 13, 14, 17, 16, 14, 15, 15, 14, 13, 11 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-cascade-py": { "n": 10, "successes": 10, "mean_cost": 0.77661525, "ratio": 1.1706834747343775, "ci_lo": 0.7642837798512072, "ci_hi": 1.831044495901304, "adoption": 0, "mean_wall_s": 271.9519, "mean_input_tokens": 38144, "mean_output_tokens": 18415, "mean_cache_tokens": 904880, "costs": [ 0.9982736999999997, 1.1526816, 0.9904425, 0.3459777, 0.3460797, 0.4525734, 0.8783337000000001, 1.0270521, 1.1905506000000001, 0.3841875 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 46, 37, 43, 26, 27, 31, 38, 41, 39, 25 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-ledger-py": { "n": 10, "successes": 10, "mean_cost": 0.52984173, "ratio": 1.3437013103555682, "ci_lo": 0.9003165273397947, "ci_hi": 1.9626450623642957, "adoption": 0, "mean_wall_s": 217.0154, "mean_input_tokens": 26787, "mean_output_tokens": 13482, "mean_cache_tokens": 556104, "costs": [ 0.23635350000000005, 0.6261722999999999, 0.4772349, 0.6602525999999999, 0.5066112, 0.5842281, 0.2246526, 1.1170122000000002, 0.639831, 0.22606889999999996 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 17, 22, 17, 33, 24, 26, 17, 35, 28, 15 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-pipeline-py": { "n": 10, "successes": 10, "mean_cost": 0.37884381, "ratio": 0.9689835535341914, "ci_lo": 0.8806593868919369, "ci_hi": 1.0817496478752033, "adoption": 0, "mean_wall_s": 88.8908, "mean_input_tokens": 27065, "mean_output_tokens": 5689, "mean_cache_tokens": 436923, "costs": [ 0.49170719999999996, 0.3532242, 0.3522648, 0.3664032, 0.3531177, 0.3932169, 0.34542810000000007, 0.3680349, 0.3662745, 0.3987665999999999 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 24, 24, 23, 24, 23, 24, 19, 23, 24, 20 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-js": { "n": 10, "successes": 10, "mean_cost": 0.16719843, "ratio": 1.188087618589596, "ci_lo": 1.0169784224356369, "ci_hi": 1.3791361329034981, "adoption": 0, "mean_wall_s": 50.376799999999996, "mean_input_tokens": 10486, "mean_output_tokens": 2475, "mean_cache_tokens": 223745, "costs": [ 0.18971399999999997, 0.167061, 0.1851864, 0.1659108, 0.125544, 0.1530393, 0.1383054, 0.19264410000000004, 0.22599029999999998, 0.128589 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 11, 11, 12, 10, 8, 10, 9, 13, 14, 8 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-validate-py": { "n": 10, "successes": 10, "mean_cost": 0.25093403999999997, "ratio": 1.048927918706866, "ci_lo": 0.8880515188348309, "ci_hi": 1.2344340105122948, "adoption": 0, "mean_wall_s": 61.228, "mean_input_tokens": 17841, "mean_output_tokens": 5022, "mean_cache_tokens": 228441, "costs": [ 0.34676940000000006, 0.2003427, 0.20876099999999997, 0.22651439999999998, 0.203118, 0.27591959999999993, 0.3050316, 0.28647750000000005, 0.24334799999999998, 0.21305819999999998 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 40, 23, 23, 25, 23, 34, 40, 32, 24, 23 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-iterate-tests": { "n": 10, "successes": 10, "mean_cost": 0.13313799, "ratio": 1.0384506307313959, "ci_lo": 0.9930820435600316, "ci_hi": 1.1052150905545763, "adoption": 0, "mean_wall_s": 27.659399999999998, "mean_input_tokens": 11015, "mean_output_tokens": 1665, "mean_cache_tokens": 140167, "costs": [ 0.1199145, 0.1245072, 0.13180920000000002, 0.1304448, 0.1311918, 0.1667361, 0.1312236, 0.13212390000000002, 0.1321155, 0.1313133 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 10, 10, 10, 10, 10, 13, 10, 10, 10, 10 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py": { "n": 10, "successes": 10, "mean_cost": 0.22542215999999998, "ratio": 1.2198945127676957, "ci_lo": 0.7822327651867216, "ci_hi": 1.841584013834522, "adoption": 0, "mean_wall_s": 46.4475, "mean_input_tokens": 20798, "mean_output_tokens": 3355, "mean_cache_tokens": 167474, "costs": [ 0.1357956, 0.17804189999999998, 0.4263153, 0.38206680000000004, 0.150414, 0.1453275, 0.29685629999999996, 0.2599014, 0.15929820000000003, 0.12020459999999997 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 7, 10, 52, 41, 7, 8, 29, 27, 9, 6 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py-xl": { "n": 10, "successes": 10, "mean_cost": 0.17578815, "ratio": 1.198558973178901, "ci_lo": 1.1002600817624901, "ci_hi": 1.3006415811250522, "adoption": 0, "mean_wall_s": 38.1351, "mean_input_tokens": 14608, "mean_output_tokens": 2063, "mean_cache_tokens": 190582, "costs": [ 0.1666509, 0.2010597, 0.1869384, 0.1890894, 0.17156580000000002, 0.1894767, 0.14284319999999998, 0.14502690000000001, 0.1907004, 0.17453010000000002 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 8, 11, 9, 10, 9, 11, 7, 8, 10, 8 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-overview-cobra": { "n": 10, "successes": 10, "mean_cost": 0.287419005, "ratio": 0.8937762581891032, "ci_lo": 0.6618306586528667, "ci_hi": 1.1890653787800631, "adoption": 0, "mean_wall_s": 90.015, "mean_input_tokens": 17652, "mean_output_tokens": 3436, "mean_cache_tokens": 264586, "costs": [ 0.23942220000000003, 0.22741440000000004, 0.36979455, 0.27735, 0.26606369999999996, 0.37116404999999997, 0.283401, 0.24436350000000004, 0.24052530000000003, 0.35469135000000007 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 17, 11, 30, 20, 17, 32, 21, 15, 16, 35 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-refactor-split-py": { "n": 10, "successes": 10, "mean_cost": 0.12744867, "ratio": 1.0745819963100431, "ci_lo": 1.0196257967622402, "ci_hi": 1.1317842630275545, "adoption": 0, "mean_wall_s": 29.1233, "mean_input_tokens": 9972, "mean_output_tokens": 1618, "mean_cache_tokens": 144390, "costs": [ 0.1251522, 0.12603989999999998, 0.11692229999999999, 0.1239168, 0.1262331, 0.1314534, 0.1316718, 0.1316337, 0.14122200000000001, 0.12024149999999999 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 11, 12, 11, 11, 12, 13, 11, 13, 14, 11 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-settings-inventory-django": { "n": 10, "successes": 10, "mean_cost": 0.43736562, "ratio": 1.2517204137389946, "ci_lo": 0.8747544199591717, "ci_hi": 1.791361492943593, "adoption": 0, "mean_wall_s": 156.3537, "mean_input_tokens": 25823, "mean_output_tokens": 9164, "mean_cache_tokens": 483078, "costs": [ 0.40117710000000006, 0.4376109, 0.3457332, 0.2713896, 0.6984507, 0.238287, 0.5130474, 0.3718068, 0.7691252999999998, 0.3270282 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 0.9972, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 13, 15, 10, 8, 23, 8, 16, 17, 24, 11 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "log-needle-zh": { "n": 10, "successes": 10, "mean_cost": 0.17791106999999998, "ratio": 1.4568468524239635, "ci_lo": 1.111472839160921, "ci_hi": 1.9179956018463555, "adoption": 0, "mean_wall_s": 44.8214, "mean_input_tokens": 15478, "mean_output_tokens": 2299, "mean_cache_tokens": 168442, "costs": [ 0.14593319999999999, 0.1266441, 0.2637693, 0.39026489999999997, 0.12471869999999999, 0.14598989999999998, 0.1537395, 0.18156659999999997, 0.12516149999999998, 0.12132299999999999 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 11, 10, 20, 16, 9, 11, 11, 15, 10, 8 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "report-pdf": { "n": 10, "successes": 10, "mean_cost": 0.31124913, "ratio": 1.1219661140413884, "ci_lo": 0.9016879624974711, "ci_hi": 1.39804069882808, "adoption": 0, "mean_wall_s": 116.0979, "mean_input_tokens": 20630, "mean_output_tokens": 8270, "mean_cache_tokens": 211297, "costs": [ 0.3015845999999999, 0.5087724, 0.27660029999999997, 0.34997130000000004, 0.22705199999999998, 0.26036580000000004, 0.37027740000000003, 0.28941929999999993, 0.2322561, 0.2961921 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 9, 10, 6, 6, 7, 6, 5, 8, 5, 6 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "seo-audit": { "n": 10, "successes": 10, "mean_cost": 0.25932849, "ratio": 0.9795212650232158, "ci_lo": 0.9330551171076635, "ci_hi": 1.0270833775943147, "adoption": 0, "mean_wall_s": 119.2658, "mean_input_tokens": 18981, "mean_output_tokens": 6848, "mean_cache_tokens": 142364, "costs": [ 0.2592474, 0.24518220000000002, 0.251247, 0.2438055, 0.25788720000000004, 0.28347120000000003, 0.25269630000000004, 0.2632449, 0.2725425, 0.2639607 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "web-research-oss-inventory": { "n": 10, "successes": 10, "mean_cost": 4.5148312, "ratio": 0.9018218281001409, "ci_lo": 0.7896527050037936, "ci_hi": 1.0199808411422642, "adoption": 0, "mean_wall_s": 979.6329, "mean_input_tokens": 95708, "mean_output_tokens": 30906, "mean_cache_tokens": 925159, "costs": [ 4.4635336, 3.2249438, 5.549284299999998, 3.6244636000000003, 4.028764799999999, 5.027611299999999, 4.617336499999998, 4.9196861, 5.510018999999998, 4.182669 ], "scores": [ 0.5439, 0.6842, 0.7368, 0.6316, 0.7018, 0.6842, 0.6667, 0.5263, 0.7193, 0.7193 ], "tool_calls": [ 53, 45, 65, 41, 47, 59, 54, 55, 55, 53 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] } } }, "squeez": { "version": "1.22.1", "aggregate_cost_ratio": 1.0103271833491907, "aggregate_ci_lo": 0.837159103466605, "aggregate_ci_hi": 1.2373268845280945, "n_success": 170, "n_runs": 170, "tasks_compared": 17, "per_task": { "code-bugfix-py": { "n": 10, "successes": 10, "mean_cost": 0.10909743, "ratio": 1.1872604066206944, "ci_lo": 1.1176723065016332, "ci_hi": 1.25411577703119, "adoption": 0, "mean_wall_s": 23.004, "mean_input_tokens": 8778, "mean_output_tokens": 828, "mean_cache_tokens": 146639, "costs": [ 0.1125717, 0.10277909999999998, 0.0907767, 0.11304539999999999, 0.10354530000000002, 0.10327829999999999, 0.10118669999999999, 0.1239471, 0.1209027, 0.11894130000000001 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 6, 5, 4, 6, 5, 5, 5, 7, 7, 7 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-comprehension-django": { "n": 10, "successes": 10, "mean_cost": 0.16453635, "ratio": 0.4279050139024102, "ci_lo": 0.3427700708364611, "ci_hi": 0.572924223375134, "adoption": 0, "mean_wall_s": 49.153, "mean_input_tokens": 14488, "mean_output_tokens": 2968, "mean_cache_tokens": 110254, "costs": [ 0.1678278, 0.1688115, 0.1746006, 0.1494267, 0.1623438, 0.14848139999999999, 0.16589700000000002, 0.17434289999999997, 0.17875949999999996, 0.15487230000000002 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 14, 13, 14, 13, 12, 11, 13, 15, 14, 13 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-cascade-py": { "n": 10, "successes": 10, "mean_cost": 0.41669415, "ratio": 0.6281320839675604, "ci_lo": 0.45613508262858515, "ci_hi": 0.9125724784522785, "adoption": 0, "mean_wall_s": 124.5278, "mean_input_tokens": 23199, "mean_output_tokens": 8076, "mean_cache_tokens": 521020, "costs": [ 0.49124579999999995, 0.32012550000000006, 0.4344795, 0.3995559, 0.43941630000000004, 0.3460542, 0.40123109999999995, 0.4514619, 0.4699305, 0.41344079999999994 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 37, 35, 36, 36, 35, 35, 35, 36, 38, 36 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-ledger-py": { "n": 10, "successes": 10, "mean_cost": 0.24806354999999997, "ratio": 0.6290997826585949, "ci_lo": 0.5072943635335482, "ci_hi": 0.8176929571110603, "adoption": 0, "mean_wall_s": 69.2269, "mean_input_tokens": 15314, "mean_output_tokens": 4098, "mean_cache_tokens": 315546, "costs": [ 0.25909019999999994, 0.2462768999999999, 0.2192817, 0.2313573, 0.20294489999999998, 0.27367620000000004, 0.25986119999999996, 0.2518341, 0.2486952, 0.2876178 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 20, 20, 20, 22, 19, 20, 20, 20, 20, 22 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-pipeline-py": { "n": 10, "successes": 10, "mean_cost": 0.3545799, "ratio": 0.906922806825848, "ci_lo": 0.8165283869773565, "ci_hi": 1.0137506015427966, "adoption": 0, "mean_wall_s": 96.1969, "mean_input_tokens": 20367, "mean_output_tokens": 5499, "mean_cache_tokens": 499454, "costs": [ 0.3716459999999999, 0.3715794000000001, 0.4182128999999999, 0.3347049, 0.3439599, 0.2955795, 0.3594203999999999, 0.28918140000000003, 0.3481098, 0.4134047999999999 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 31, 30, 29, 29, 29, 25, 29, 25, 27, 28 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-js": { "n": 10, "successes": 10, "mean_cost": 0.14422443000000001, "ratio": 1.024837730719971, "ci_lo": 0.8812879138225274, "ci_hi": 1.1901003350613513, "adoption": 0, "mean_wall_s": 39.2084, "mean_input_tokens": 9748, "mean_output_tokens": 1891, "mean_cache_tokens": 191124, "costs": [ 0.12671280000000001, 0.11929320000000002, 0.1514355, 0.15718230000000002, 0.1538958, 0.15167580000000003, 0.14851409999999998, 0.20393219999999998, 0.10832250000000002, 0.1212801 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 8, 7, 9, 9, 9, 9, 10, 13, 6, 7 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-validate-py": { "n": 10, "successes": 10, "mean_cost": 0.29399217, "ratio": 1.2289149570708509, "ci_lo": 1.0540606403366581, "ci_hi": 1.4257680093454927, "adoption": 0, "mean_wall_s": 72.9844, "mean_input_tokens": 18512, "mean_output_tokens": 5791, "mean_cache_tokens": 320029, "costs": [ 0.34459710000000005, 0.2490813, 0.2932079999999999, 0.2298654, 0.3330833999999999, 0.3289536, 0.332067, 0.233652, 0.3324456, 0.2629683 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 43, 29, 33, 26, 43, 44, 44, 27, 44, 32 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-iterate-tests": { "n": 10, "successes": 10, "mean_cost": 0.15269562, "ratio": 1.1909963707497877, "ci_lo": 1.1484379134319653, "ci_hi": 1.2396729911821704, "adoption": 0, "mean_wall_s": 39.1518, "mean_input_tokens": 11246, "mean_output_tokens": 1851, "mean_cache_tokens": 191407, "costs": [ 0.1571931, 0.1478967, 0.1594011, 0.15972930000000002, 0.14662619999999998, 0.1463487, 0.1732167, 0.13905420000000002, 0.1491438, 0.1483464 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 13, 13, 14, 14, 12, 12, 15, 12, 13, 13 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py": { "n": 10, "successes": 10, "mean_cost": 0.16230966, "ratio": 0.8783549212871988, "ci_lo": 0.5847266645892668, "ci_hi": 1.3641955016178282, "adoption": 0, "mean_wall_s": 33.1901, "mean_input_tokens": 14259, "mean_output_tokens": 1846, "mean_cache_tokens": 163485, "costs": [ 0.137865, 0.12227129999999999, 0.1406355, 0.1656939, 0.1238133, 0.1120425, 0.14888790000000002, 0.12378179999999997, 0.16659030000000002, 0.3815151 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 8, 8, 8, 10, 7, 7, 9, 7, 11, 42 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py-xl": { "n": 10, "successes": 10, "mean_cost": 0.15253668, "ratio": 1.040025772800491, "ci_lo": 0.9455871409844276, "ci_hi": 1.1445172032367896, "adoption": 0, "mean_wall_s": 33.2443, "mean_input_tokens": 12128, "mean_output_tokens": 1719, "mean_cache_tokens": 179849, "costs": [ 0.18184619999999999, 0.1518705, 0.1446123, 0.14223600000000003, 0.1659756, 0.1428645, 0.13580340000000002, 0.1209162, 0.15121679999999998, 0.1880253 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 11, 8, 8, 8, 9, 7, 7, 6, 8, 11 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-overview-cobra": { "n": 10, "successes": 10, "mean_cost": 0.47099508, "ratio": 1.4646359945052254, "ci_lo": 1.0625526848174445, "ci_hi": 2.010269424280098, "adoption": 0, "mean_wall_s": 145.5854, "mean_input_tokens": 23017, "mean_output_tokens": 4746, "mean_cache_tokens": 446433, "costs": [ 0.4277301, 0.703764, 0.6784164000000001, 0.4143576, 0.44586960000000003, 0.4257387, 0.2983803, 0.3902685, 0.5689671000000001, 0.3564585 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 26, 54, 27, 30, 26, 43, 20, 22, 35, 23 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-refactor-split-py": { "n": 10, "successes": 10, "mean_cost": 0.11567912999999999, "ratio": 0.9753472550698961, "ci_lo": 0.9228956376965016, "ci_hi": 1.0314177704771972, "adoption": 0, "mean_wall_s": 24.6803, "mean_input_tokens": 9497, "mean_output_tokens": 1390, "mean_cache_tokens": 126108, "costs": [ 0.10755029999999999, 0.108735, 0.1193613, 0.1075734, 0.1088865, 0.1187325, 0.1184049, 0.1188303, 0.1300554, 0.11866170000000001 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 10, 10, 11, 10, 10, 11, 11, 11, 12, 11 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-settings-inventory-django": { "n": 10, "successes": 10, "mean_cost": 0.35111547, "ratio": 1.0048764266806374, "ci_lo": 0.7344210858925297, "ci_hi": 1.3977177515575847, "adoption": 0, "mean_wall_s": 109.2402, "mean_input_tokens": 19442, "mean_output_tokens": 5512, "mean_cache_tokens": 505740, "costs": [ 0.5737407, 0.3443109, 0.437538, 0.4039707, 0.25482899999999997, 0.3044484, 0.31723260000000003, 0.34004760000000006, 0.343029, 0.19200779999999998 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 0.8182, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 28, 16, 21, 21, 13, 14, 16, 16, 16, 9 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "log-needle-zh": { "n": 10, "successes": 10, "mean_cost": 0.35543877, "ratio": 2.910554432076347, "ci_lo": 2.426541212532041, "ci_hi": 3.3469483237235704, "adoption": 0, "mean_wall_s": 47.7986, "mean_input_tokens": 40718, "mean_output_tokens": 1836, "mean_cache_tokens": 278565, "costs": [ 0.3853197, 0.45348959999999994, 0.37586430000000004, 0.19725779999999998, 0.30702930000000006, 0.44473650000000003, 0.2014809, 0.4125243, 0.40013879999999996, 0.3765465 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 7, 10, 6, 6, 7, 7, 6, 6, 6, 6 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "report-pdf": { "n": 10, "successes": 10, "mean_cost": 0.26732841, "ratio": 0.9636441950554627, "ci_lo": 0.7751939084736298, "ci_hi": 1.191854287649735, "adoption": 0, "mean_wall_s": 100.3614, "mean_input_tokens": 17192, "mean_output_tokens": 7075, "mean_cache_tokens": 193408, "costs": [ 0.2318802, 0.42560070000000005, 0.21112680000000003, 0.28633980000000003, 0.24102330000000002, 0.3358179, 0.27216989999999996, 0.254028, 0.17779199999999998, 0.2375055 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 7, 7, 5, 7, 4, 14, 6, 6, 5, 5 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "seo-audit": { "n": 10, "successes": 10, "mean_cost": 0.33651426, "ratio": 1.271063097053283, "ci_lo": 1.1825507712747836, "ci_hi": 1.3599344217065064, "adoption": 0, "mean_wall_s": 141.4266, "mean_input_tokens": 22777, "mean_output_tokens": 7998, "mean_cache_tokens": 266148, "costs": [ 0.3116259, 0.36618300000000004, 0.2784321, 0.3042576, 0.37233449999999996, 0.3513807, 0.3291, 0.3162294, 0.3828192, 0.3527802 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 17, 21, 17, 17, 19, 21, 19, 20, 23, 20 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "web-research-oss-inventory": { "n": 10, "successes": 10, "mean_cost": 4.66958772, "ratio": 0.9327339046749675, "ci_lo": 0.8140298684929375, "ci_hi": 1.058624858708032, "adoption": 0, "mean_wall_s": 933.4312, "mean_input_tokens": 99466, "mean_output_tokens": 26718, "mean_cache_tokens": 979464, "costs": [ 4.2589223, 3.5659293999999995, 4.045137700000001, 4.103119900000001, 5.9574681, 5.704548499999999, 5.246578799999998, 3.9707801000000007, 4.331544499999998, 5.511847900000002 ], "scores": [ 0.6491, 0.6316, 0.5789, 0.5088, 0.5614, 0.6667, 0.6842, 0.6316, 0.6316, 0.5965 ], "tool_calls": [ 52, 39, 44, 51, 68, 68, 59, 46, 55, 64 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] } } }, "tokenade": { "version": "0.8.13", "aggregate_cost_ratio": 0.7678373535557741, "aggregate_ci_lo": 0.6471280299665931, "aggregate_ci_hi": 0.899539373214061, "n_success": 170, "n_runs": 170, "tasks_compared": 17, "per_task": { "code-bugfix-py": { "n": 10, "successes": 10, "mean_cost": 0.09954117, "ratio": 1.083263739298897, "ci_lo": 1.0573168859054654, "ci_hi": 1.1119500179875474, "adoption": 0, "mean_wall_s": 18.673000000000002, "mean_input_tokens": 8950, "mean_output_tokens": 749, "mean_cache_tokens": 115285, "costs": [ 0.0984444, 0.0987549, 0.10741859999999999, 0.0948597, 0.100125, 0.09504390000000001, 0.0981828, 0.0988359, 0.09847890000000001, 0.1052676 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 5, 5, 6, 4, 5, 4, 5, 5, 5, 5 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-comprehension-django": { "n": 10, "successes": 10, "mean_cost": 0.19123446, "ratio": 0.49733803056236453, "ci_lo": 0.397970917428212, "ci_hi": 0.6711830446724659, "adoption": 0, "mean_wall_s": 51.150999999999996, "mean_input_tokens": 16867, "mean_output_tokens": 3138, "mean_cache_tokens": 143131, "costs": [ 0.2070465, 0.17448569999999997, 0.18304049999999997, 0.18697859999999997, 0.173748, 0.18210929999999997, 0.17765279999999997, 0.19629990000000003, 0.2206773, 0.21030600000000002 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 15, 13, 13, 8, 13, 13, 13, 15, 14, 15 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-cascade-py": { "n": 10, "successes": 10, "mean_cost": 0.43386363, "ratio": 0.6540136598261113, "ci_lo": 0.4149506452330786, "ci_hi": 1.0559716539905337, "adoption": 0, "mean_wall_s": 153.4233, "mean_input_tokens": 31887, "mean_output_tokens": 10907, "mean_cache_tokens": 262957, "costs": [ 0.2500791, 0.25464689999999995, 0.26361090000000004, 0.3497598, 0.35084099999999996, 0.5097192, 0.28473420000000005, 0.46491779999999994, 1.0103313, 0.5999961 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 26, 26, 26, 29, 27, 27, 26, 26, 34, 30 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-ledger-py": { "n": 10, "successes": 10, "mean_cost": 0.18914607, "ratio": 0.4796825310599941, "ci_lo": 0.38437170846494434, "ci_hi": 0.6253542754605554, "adoption": 0, "mean_wall_s": 48.1131, "mean_input_tokens": 14289, "mean_output_tokens": 2854, "mean_cache_tokens": 201476, "costs": [ 0.1937868, 0.16533389999999998, 0.18648389999999998, 0.18832590000000002, 0.2048532, 0.2190039, 0.1572183, 0.20088989999999995, 0.1665309, 0.20903399999999997 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 13, 13, 13, 14, 17, 14, 13, 13, 13, 13 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-debug-pipeline-py": { "n": 10, "successes": 10, "mean_cost": 0.28472937, "ratio": 0.7282633883820131, "ci_lo": 0.6708302808754976, "ci_hi": 0.7992547121238914, "adoption": 0, "mean_wall_s": 74.821, "mean_input_tokens": 20532, "mean_output_tokens": 4619, "mean_cache_tokens": 307339, "costs": [ 0.2861694, 0.2620524, 0.2720583, 0.2985936, 0.31730280000000005, 0.2829099, 0.27299490000000004, 0.29666639999999994, 0.2999328, 0.25861320000000004 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 23, 23, 24, 24, 22, 24, 23, 26, 17, 24 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-js": { "n": 10, "successes": 10, "mean_cost": 0.1375887, "ratio": 0.9776852027129582, "ci_lo": 0.8681134579367683, "ci_hi": 1.1043185187217324, "adoption": 0, "mean_wall_s": 36.1697, "mean_input_tokens": 10038, "mean_output_tokens": 1568, "mean_cache_tokens": 179372, "costs": [ 0.1396068, 0.16017689999999998, 0.1106775, 0.137658, 0.1325991, 0.13995599999999997, 0.1579713, 0.1363344, 0.1371021, 0.1238049 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 8, 9, 6, 8, 8, 8, 9, 8, 8, 7 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-feature-validate-py": { "n": 10, "successes": 10, "mean_cost": 0.20706411, "ratio": 0.865547479892284, "ci_lo": 0.7699174112795323, "ci_hi": 0.9769455059635033, "adoption": 0, "mean_wall_s": 42.9413, "mean_input_tokens": 17072, "mean_output_tokens": 3612, "mean_cache_tokens": 168081, "costs": [ 0.1970691, 0.22635960000000002, 0.1978863, 0.2054202, 0.205974, 0.20535750000000003, 0.21967920000000002, 0.2053578, 0.2105013, 0.19703610000000002 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 23, 24, 23, 23, 23, 23, 23, 23, 23, 23 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-iterate-tests": { "n": 10, "successes": 10, "mean_cost": 0.13797984, "ratio": 1.0762161264130325, "ci_lo": 1.0172014196462618, "ci_hi": 1.1329496584338543, "adoption": 0, "mean_wall_s": 28.8712, "mean_input_tokens": 10955, "mean_output_tokens": 1538, "mean_cache_tokens": 163806, "costs": [ 0.1317285, 0.11934479999999999, 0.147837, 0.14383200000000002, 0.1306587, 0.12032789999999997, 0.15780749999999996, 0.14649749999999997, 0.14445119999999997, 0.1373133 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 11, 10, 10, 10, 11, 10, 10, 10, 10, 9 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-migration-py": { "n": 10, "successes": 10, "mean_cost": 0.16592709, "ratio": 0.8979310047002992, "ci_lo": 0.5744261860752344, "ci_hi": 1.3842438518138462, "adoption": 3, "mean_wall_s": 30.882, "mean_input_tokens": 16046, "mean_output_tokens": 2253, "mean_cache_tokens": 119417, "costs": [ 0.2664009, 0.2891955, 0.11932259999999999, 0.3083466, 0.1163805, 0.1119276, 0.1118409, 0.1136346, 0.1064565, 0.11576520000000001 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 27, 28, 6, 29, 5, 5, 6, 6, 5, 5 ], "competitor_tool_calls": [ 0, 0, 1, 0, 1, 1, 0, 0, 0, 0 ] }, "code-migration-py-xl": { "n": 10, "successes": 10, "mean_cost": 0.13362381, "ratio": 0.9110740200966481, "ci_lo": 0.8581486516030377, "ci_hi": 0.9723626239131911, "adoption": 7, "mean_wall_s": 25.740099999999998, "mean_input_tokens": 12499, "mean_output_tokens": 1386, "mean_cache_tokens": 126083, "costs": [ 0.14289390000000002, 0.1229328, 0.12066780000000002, 0.1317921, 0.13314990000000002, 0.13929000000000002, 0.1386411, 0.14184180000000002, 0.13763550000000002, 0.1273932 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 6, 5, 5, 5, 5, 6, 7, 6, 5, 5 ], "competitor_tool_calls": [ 0, 0, 1, 1, 0, 1, 1, 1, 1, 1 ] }, "code-overview-cobra": { "n": 10, "successes": 10, "mean_cost": 0.2014545, "ratio": 0.6264556138358238, "ci_lo": 0.4755469896895197, "ci_hi": 0.819908129497842, "adoption": 0, "mean_wall_s": 61.1248, "mean_input_tokens": 17595, "mean_output_tokens": 2774, "mean_cache_tokens": 179526, "costs": [ 0.20014110000000002, 0.20458440000000003, 0.1735359, 0.20621579999999998, 0.20193689999999997, 0.20237460000000002, 0.18981359999999997, 0.25105350000000004, 0.1852293, 0.1996599 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 8, 12, 7, 10, 8, 8, 11, 13, 8, 9 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-refactor-split-py": { "n": 10, "successes": 10, "mean_cost": 0.12765267, "ratio": 1.0763020199654274, "ci_lo": 0.9622237417179454, "ci_hi": 1.212483879703578, "adoption": 0, "mean_wall_s": 24.854499999999998, "mean_input_tokens": 10211, "mean_output_tokens": 1520, "mean_cache_tokens": 145210, "costs": [ 0.1331526, 0.1131294, 0.1334577, 0.11212439999999999, 0.1329078, 0.10357859999999999, 0.1118292, 0.17125049999999997, 0.16186860000000003, 0.1032279 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 12, 10, 12, 10, 12, 9, 10, 15, 14, 9 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "code-settings-inventory-django": { "n": 10, "successes": 10, "mean_cost": 0.27493818, "ratio": 0.7868605045413632, "ci_lo": 0.5121056188045342, "ci_hi": 1.2163993910196567, "adoption": 5, "mean_wall_s": 95.2818, "mean_input_tokens": 18910, "mean_output_tokens": 5644, "mean_cache_tokens": 255951, "costs": [ 0.2417379, 0.3809616, 0.2902437, 0.7124438999999999, 0.20911799999999997, 0.1906581, 0.1930227, 0.16713479999999997, 0.16203120000000001, 0.20202989999999998 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 5, 15, 10, 27, 6, 4, 5, 3, 6, 4 ], "competitor_tool_calls": [ 0, 1, 1, 2, 1, 0, 0, 0, 2, 0 ] }, "log-needle-zh": { "n": 10, "successes": 10, "mean_cost": 0.12418401, "ratio": 1.0168961610420646, "ci_lo": 0.9259453010321245, "ci_hi": 1.1122925117097695, "adoption": 0, "mean_wall_s": 28.308600000000002, "mean_input_tokens": 12451, "mean_output_tokens": 1254, "mean_cache_tokens": 102133, "costs": [ 0.120081, 0.1208148, 0.0955737, 0.15721530000000003, 0.10609559999999998, 0.1184499, 0.123981, 0.1305489, 0.1396332, 0.1294467 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 7, 7, 6, 9, 5, 7, 7, 6, 6, 7 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "report-pdf": { "n": 10, "successes": 10, "mean_cost": 0.1985685, "ratio": 0.7157839391102152, "ci_lo": 0.6040349817647251, "ci_hi": 0.845230991267483, "adoption": 2, "mean_wall_s": 74.2617, "mean_input_tokens": 13219, "mean_output_tokens": 5279, "mean_cache_tokens": 133499, "costs": [ 0.2027001, 0.188328, 0.1777599, 0.2281224, 0.19617300000000001, 0.2107287, 0.2305188, 0.17511089999999999, 0.1853865, 0.1908567 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 5, 5, 5, 5, 6, 6, 6, 5, 5, 5 ], "competitor_tool_calls": [ 0, 2, 0, 0, 1, 0, 0, 0, 0, 0 ] }, "seo-audit": { "n": 10, "successes": 10, "mean_cost": 0.25757322, "ratio": 0.9728913560191672, "ci_lo": 0.9230131438610205, "ci_hi": 1.0222959821469162, "adoption": 0, "mean_wall_s": 112.7206, "mean_input_tokens": 19173, "mean_output_tokens": 6661, "mean_cache_tokens": 141999, "costs": [ 0.2374725, 0.24586410000000003, 0.27707760000000003, 0.250818, 0.2591682, 0.2517942, 0.2459508, 0.2574477, 0.2648598, 0.2852793 ], "scores": [ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ], "tool_calls": [ 14, 14, 15, 5, 14, 14, 14, 14, 14, 14 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] }, "web-research-oss-inventory": { "n": 10, "successes": 10, "mean_cost": 1.62873914, "ratio": 0.3253349779130284, "ci_lo": 0.29529801532754113, "ci_hi": 0.35574543841665296, "adoption": 0, "mean_wall_s": 557.8261, "mean_input_tokens": 109981, "mean_output_tokens": 26308, "mean_cache_tokens": 1206644, "costs": [ 1.8361329000000002, 1.5209502, 1.7583640000000003, 1.5692279, 1.4693115, 1.5839243, 1.7479741000000002, 1.7994540999999997, 1.5050774, 1.496975 ], "scores": [ 0.5263, 0.5965, 0.5614, 0.6316, 0.5789, 0.5614, 0.614, 0.5965, 0.5439, 0.5614 ], "tool_calls": [ 72, 56, 63, 60, 62, 67, 75, 80, 62, 56 ], "competitor_tool_calls": [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] } } } }, "ranking": [ { "rank": 1, "competitor": "tokenade", "aggregate_cost_ratio": 0.7678373535557741, "aggregate_ci_lo": 0.6471280299665931, "aggregate_ci_hi": 0.899539373214061, "n_success": 170, "n_runs": 170, "tasks_compared": 17 }, { "rank": 2, "competitor": "claude-token-efficient", "aggregate_cost_ratio": 0.9522367932207415, "aggregate_ci_lo": 0.8247980689646457, "aggregate_ci_hi": 1.0759563303036992, "n_success": 170, "n_runs": 170, "tasks_compared": 17 }, { "rank": 3, "competitor": "ponytail", "aggregate_cost_ratio": 0.9576172154231043, "aggregate_ci_lo": 0.8493117124279771, "aggregate_ci_hi": 1.0654785686813864, "n_success": 170, "n_runs": 170, "tasks_compared": 17 }, { "rank": 4, "competitor": "caveman", "aggregate_cost_ratio": 0.9807129853494566, "aggregate_ci_lo": 0.8613692671665824, "aggregate_ci_hi": 1.1113763247130377, "n_success": 170, "n_runs": 170, "tasks_compared": 17 }, { "rank": 5, "competitor": "graphify", "aggregate_cost_ratio": 0.9911855904549622, "aggregate_ci_lo": 0.8707359930059344, "aggregate_ci_hi": 1.1120196999324352, "n_success": 170, "n_runs": 170, "tasks_compared": 17 }, { "rank": 6, "competitor": "edgee", "aggregate_cost_ratio": 1.0092565725594083, "aggregate_ci_lo": 0.8513202313728729, "aggregate_ci_hi": 1.1695418779070588, "n_success": 169, "n_runs": 170, "tasks_compared": 17 }, { "rank": 7, "competitor": "squeez", "aggregate_cost_ratio": 1.0103271833491907, "aggregate_ci_lo": 0.837159103466605, "aggregate_ci_hi": 1.2373268845280945, "n_success": 170, "n_runs": 170, "tasks_compared": 17 }, { "rank": 8, "competitor": "code-review-graph", "aggregate_cost_ratio": 1.0113475169386419, "aggregate_ci_lo": 0.9021437384639193, "aggregate_ci_hi": 1.1283570512884105, "n_success": 170, "n_runs": 170, "tasks_compared": 17 }, { "rank": 9, "competitor": "codegraph", "aggregate_cost_ratio": 1.0494160451367467, "aggregate_ci_lo": 0.9459574187229038, "aggregate_ci_hi": 1.1661736916132155, "n_success": 170, "n_runs": 170, "tasks_compared": 17 }, { "rank": 10, "competitor": "rtk", "aggregate_cost_ratio": 1.0521527335785215, "aggregate_ci_lo": 0.9134305001515856, "aggregate_ci_hi": 1.1895371095370226, "n_success": 170, "n_runs": 170, "tasks_compared": 17 }, { "rank": 11, "competitor": "lean-ctx", "aggregate_cost_ratio": 1.2342724885887937, "aggregate_ci_lo": 1.0417273386923387, "aggregate_ci_hi": 1.4559856011249679, "n_success": 170, "n_runs": 170, "tasks_compared": 17 }, { "rank": 12, "competitor": "headroom", "aggregate_cost_ratio": 1.5568980915065826, "aggregate_ci_lo": 1.3014624134696557, "aggregate_ci_hi": 1.897666681105211, "n_success": 170, "n_runs": 170, "tasks_compared": 17 } ], "non_ok": [], "campaign": "2.1.206 (Claude Code)" }