{"snapshot":{"schemaVersion":1,"generatedAt":"2026-11-21T22:00:15.273Z","snapshotHash":"055cc51f9a1f8b55","sources":[{"sourceId":"artificial-analysis","label":"Artificial Analysis Frontier Suite","status":"ok","fetchedAt":"2026-11-21T22:00:15.273Z","lastHealthyAt":"2026-11-21T22:00:15.273Z","staleDays":0,"rowCount":990,"license":"Public Benchmarks / Attribution Required","sourceUrl":"https://artificialanalysis.ai/leaderboards/models","message":null,"subFeeds":[{"benchmarkId":"aa_agent_frontiercode","status":"ok","rowCount":32,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"aa_agent_swe_atlas","status":"ok","rowCount":32,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"aa_analyst_agent","status":"ok","rowCount":23,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"aa_briefcase","status":"ok","rowCount":28,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"aa_code2video","status":"ok","rowCount":25,"rejectedCount":2,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"aa_mka","status":"ok","rowCount":18,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"aa_omniscience","status":"ok","rowCount":88,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"aa_vials","status":"ok","rowCount":15,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"critpt_v2_hard","status":"ok","rowCount":85,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"frontiercode_2_0","status":"ok","rowCount":86,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"gdp_pdf","status":"ok","rowCount":101,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"gdpval_aa","status":"ok","rowCount":24,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"hle_diamond","status":"ok","rowCount":98,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"mlcr_aa","status":"ok","rowCount":74,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"scicode_aa","status":"ok","rowCount":74,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"terminalbench_4_0","status":"ok","rowCount":100,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"terminalbench_science_aa","status":"ok","rowCount":87,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null}],"pricingHealth":null},{"sourceId":"aikido","label":"Aikido Arena Real-World CVE Detection","status":"ok","fetchedAt":"2026-11-21T22:00:15.273Z","lastHealthyAt":"2026-11-21T22:00:15.273Z","staleDays":0,"rowCount":13,"license":"Public Research Leaderboard / Attribution Required","sourceUrl":"https://www.aikido.dev/aikido-arena","message":null,"subFeeds":[{"benchmarkId":"aikido_arena","status":"ok","rowCount":13,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null}],"pricingHealth":null},{"sourceId":"cwe-bench","label":"Collinear CWE-bench v2 (250 Public + Held-Out)","status":"ok","fetchedAt":"2026-11-21T22:00:15.273Z","lastHealthyAt":"2026-11-21T22:00:15.273Z","staleDays":0,"rowCount":20,"license":"Apache-2.0","sourceUrl":"https://cwe-bench.com/","message":null,"subFeeds":[{"benchmarkId":"cwe_bench_v2","status":"ok","rowCount":20,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null}],"pricingHealth":null},{"sourceId":"cyber-arena","label":"CyberArena Attack-Defense CTF (mc-v1)","status":"ok","fetchedAt":"2026-11-21T22:00:15.273Z","lastHealthyAt":"2026-11-21T22:00:15.273Z","staleDays":0,"rowCount":14,"license":"Public Research Leaderboard / Attribution Required","sourceUrl":"https://cyber-arena.ai/","message":null,"subFeeds":[{"benchmarkId":"cyberarena_mc_v1","status":"ok","rowCount":14,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null}],"pricingHealth":null},{"sourceId":"epoch","label":"Epoch AI Benchmark Hub (6 Frontier Benchmarks)","status":"ok","fetchedAt":"2026-11-21T22:00:15.273Z","lastHealthyAt":"2026-11-21T22:00:15.273Z","staleDays":0,"rowCount":444,"license":"CC-BY-4.0","sourceUrl":"https://epoch.ai/data/ai-benchmarking-dashboard","message":null,"subFeeds":[{"benchmarkId":"epoch_ale_bench","status":"ok","rowCount":142,"rejectedCount":1,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"epoch_apex_agents","status":"ok","rowCount":70,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"epoch_cursorbench","status":"ok","rowCount":53,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"epoch_frontierswe","status":"ok","rowCount":25,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"epoch_mirrorcode","status":"ok","rowCount":48,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"epoch_simpleqa_verified","status":"ok","rowCount":106,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null}],"pricingHealth":null},{"sourceId":"frontier-harness","label":"FrontierHarness Eval v1.1 (TB 2.1 + TB 4.0)","status":"ok","fetchedAt":"2026-11-21T22:00:15.273Z","lastHealthyAt":"2026-11-21T22:00:15.273Z","staleDays":0,"rowCount":130,"license":"Public Research Leaderboard / Attribution Required","sourceUrl":"https://frontierharness.org/leaderboard/v1.1/","message":null,"subFeeds":[{"benchmarkId":"__harness_leaderboard__","status":"ok","rowCount":130,"rejectedCount":53,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null}],"pricingHealth":null},{"sourceId":"lmarena","label":"LMArena Style-Controlled & Agent Parquet Leaderboards","status":"ok","fetchedAt":"2026-11-21T22:00:15.273Z","lastHealthyAt":"2026-11-21T22:00:15.273Z","staleDays":0,"rowCount":803,"license":"Apache-2.0 / CC-BY-4.0","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","message":null,"subFeeds":[{"benchmarkId":"lmarena_agent","status":"ok","rowCount":68,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"lmarena_creative_writing","status":"ok","rowCount":386,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"lmarena_factuality","status":"ok","rowCount":195,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"lmarena_webdev","status":"ok","rowCount":154,"rejectedCount":2,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null}],"pricingHealth":null},{"sourceId":"tbench","label":"Terminal-Bench 4.1 Live Harness Board","status":"ok","fetchedAt":"2026-11-21T22:00:15.273Z","lastHealthyAt":"2026-11-21T22:00:15.273Z","staleDays":0,"rowCount":54,"license":"Apache-2.0","sourceUrl":"https://www.tbench.ai/","message":null,"subFeeds":[{"benchmarkId":"tbench_4_1","status":"ok","rowCount":54,"rejectedCount":1,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null}],"pricingHealth":null},{"sourceId":"harness-leaderboard","label":"Harness Leaderboard (TB 2.1 + DeepSWE)","status":"ok","fetchedAt":"2026-11-21T22:00:15.273Z","lastHealthyAt":"2026-11-21T22:00:15.273Z","staleDays":0,"rowCount":65,"license":"Apache-2.0","sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard","message":null,"subFeeds":[{"benchmarkId":"__harness_leaderboard__","status":"ok","rowCount":65,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null}],"pricingHealth":null},{"sourceId":"livebench","label":"LiveBench 2.0 Long-Agentic","status":"ok","fetchedAt":"2026-11-21T22:00:15.273Z","lastHealthyAt":"2026-11-21T22:00:15.273Z","staleDays":0,"rowCount":38,"license":"Apache-2.0","sourceUrl":"https://livebench.ai/","message":null,"subFeeds":[{"benchmarkId":"livebench_long_agentic","status":"ok","rowCount":38,"rejectedCount":1,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null}],"pricingHealth":null},{"sourceId":"vals","label":"Vals AI Frontier Suite (10 Benchmarks)","status":"ok","fetchedAt":"2026-11-21T22:00:15.273Z","lastHealthyAt":"2026-11-21T22:00:15.273Z","staleDays":0,"rowCount":602,"license":"Public Benchmarks / Attribution Required","sourceUrl":"https://www.vals.ai/benchmarks","message":null,"subFeeds":[{"benchmarkId":"vals_code_migration","status":"ok","rowCount":28,"rejectedCount":1,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"vals_emb","status":"ok","rowCount":46,"rejectedCount":2,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"vals_finance_agent","status":"ok","rowCount":42,"rejectedCount":2,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"vals_legal_research","status":"ok","rowCount":89,"rejectedCount":6,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"vals_medcode","status":"ok","rowCount":103,"rejectedCount":6,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"vals_mystery_mechanism","status":"ok","rowCount":32,"rejectedCount":2,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"vals_sre_bench","status":"ok","rowCount":31,"rejectedCount":1,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"vals_swebench_multimodal","status":"ok","rowCount":21,"rejectedCount":1,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"vals_tax_agent","status":"ok","rowCount":84,"rejectedCount":4,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"vals_vibe_code","status":"ok","rowCount":126,"rejectedCount":6,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null}],"pricingHealth":null},{"sourceId":"vulcanbench","label":"VulcanBench SWE v4 Agent Stacks","status":"ok","fetchedAt":"2026-11-21T22:00:15.273Z","lastHealthyAt":"2026-11-21T22:00:15.273Z","staleDays":0,"rowCount":91,"license":"Public Research Leaderboard / Attribution Required","sourceUrl":"https://vulcanbench.com/","message":null,"subFeeds":[{"benchmarkId":"vulcanbench_swe_v4","status":"ok","rowCount":91,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null}],"pricingHealth":null},{"sourceId":"designarena","label":"DesignArena Visual Arenas (Website, SVG, 3D, GameDev)","status":"ok","fetchedAt":"2026-11-21T22:00:15.273Z","lastHealthyAt":"2026-11-21T22:00:15.273Z","staleDays":0,"rowCount":686,"license":"Public Leaderboard / Attribution Required","sourceUrl":"https://designarena.ai/leaderboard","message":"designarena_3d: Upstream btStdErr null on 167 rows; conservatively down-weighted by reliability model; designarena_gamedev: Upstream btStdErr null on 173 rows; conservatively down-weighted by reliability model; designarena_svg: Upstream btStdErr null on 143 rows; conservatively down-weighted by reliability model; designarena_website: Upstream btStdErr null on 203 rows; conservatively down-weighted by reliability model","subFeeds":[{"benchmarkId":"designarena_3d","status":"ok","rowCount":167,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":"Upstream btStdErr null on 167 rows; conservatively down-weighted by reliability model"},{"benchmarkId":"designarena_gamedev","status":"ok","rowCount":173,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":"Upstream btStdErr null on 173 rows; conservatively down-weighted by reliability model"},{"benchmarkId":"designarena_svg","status":"ok","rowCount":143,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":"Upstream btStdErr null on 143 rows; conservatively down-weighted by reliability model"},{"benchmarkId":"designarena_website","status":"ok","rowCount":203,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":"Upstream btStdErr null on 203 rows; conservatively down-weighted by reliability model"}],"pricingHealth":null},{"sourceId":"writing-feeds","label":"Lech Mazur & EQ-Bench v3 Creative and Longform Writing Feeds","status":"ok","fetchedAt":"2026-11-21T22:00:15.273Z","lastHealthyAt":"2026-11-21T22:00:15.273Z","staleDays":0,"rowCount":356,"license":"MIT / Public Research","sourceUrl":"https://github.com/lechmazur/writing","message":null,"subFeeds":[{"benchmarkId":"eqbench_creative_writing","status":"ok","rowCount":146,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"eqbench_longform_writing","status":"ok","rowCount":143,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"lechmazur_writing","status":"ok","rowCount":67,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null}],"pricingHealth":null},{"sourceId":"scale-seal","label":"Scale SEAL (PRBench Finance & Legal)","status":"ok","fetchedAt":"2026-11-21T22:00:15.273Z","lastHealthyAt":"2026-11-21T22:00:15.273Z","staleDays":0,"rowCount":97,"license":"Public Research Leaderboard","sourceUrl":"https://scale.com/leaderboard/prbench","message":null,"subFeeds":[{"benchmarkId":"scale_prbench_finance","status":"ok","rowCount":45,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"scale_prbench_legal","status":"ok","rowCount":52,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null}],"pricingHealth":null},{"sourceId":"surge","label":"Surge AI Frontier Benchmarks (7 Benchmarks)","status":"ok","fetchedAt":"2026-11-21T22:00:15.273Z","lastHealthyAt":"2026-11-21T22:00:15.273Z","staleDays":0,"rowCount":240,"license":"Public Research Leaderboards / Attribution Required","sourceUrl":"https://www.surgehq.ai/benchmarks","message":null,"subFeeds":[{"benchmarkId":"surge_complex_constraints","status":"ok","rowCount":26,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"surge_dayjob_finance","status":"ok","rowCount":50,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"surge_dayjob_healthcare","status":"ok","rowCount":50,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"surge_dayjob_law","status":"ok","rowCount":25,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"surge_gdp_xlsx","status":"ok","rowCount":36,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"surge_riemann","status":"ok","rowCount":28,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null},{"benchmarkId":"surge_sudo_l7","status":"ok","rowCount":25,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null}],"pricingHealth":null},{"sourceId":"pricing","label":"First-Party API Pricing (LiteLLM Verified SKUs)","status":"ok","fetchedAt":"2026-11-21T22:00:15.273Z","lastHealthyAt":"2026-11-21T22:00:15.273Z","staleDays":0,"rowCount":71,"license":"MIT / Public API","sourceUrl":"https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json","message":null,"subFeeds":[],"pricingHealth":{"benchmarkId":"__pricing__","status":"ok","rowCount":71,"rejectedCount":0,"consecutiveDropDays":0,"lastLowCountDate":null,"lastHealthyAt":"2026-11-21T22:00:15.273Z","message":null}}],"domains":{"coding":{"domainId":"coding","epochVersion":"coding-v2.0.0","costUnit":"usd_per_task","activeBenchmarkIds":["frontiercode_2_0","terminalbench_4_0","scicode_aa","epoch_mirrorcode","epoch_ale_bench","epoch_frontierswe","epoch_cursorbench","lmarena_agent","surge_sudo_l7","vals_code_migration","vals_sre_bench","livebench_long_agentic"],"candidateBenchmarkIds":[],"benchmarkVersions":{"frontiercode_2_0":"2.0","terminalbench_4_0":"4.0","scicode_aa":"1.0","epoch_mirrorcode":"2026.11","epoch_ale_bench":"2026.11","epoch_frontierswe":"2026.11","epoch_cursorbench":"2026.11","lmarena_agent":"agent_capability","surge_sudo_l7":"1.0","vals_code_migration":"1.1","vals_sre_bench":"1","livebench_long_agentic":"20261106"},"coverageFloor":1,"tauResid":0.488,"regimeMeta":{"standardized":{"activeBenchmarkIds":["frontiercode_2_0","terminalbench_4_0","scicode_aa","epoch_mirrorcode","epoch_ale_bench","epoch_frontierswe","epoch_cursorbench","lmarena_agent","surge_sudo_l7","vals_code_migration","vals_sre_bench","livebench_long_agentic"],"costSource":"suite","measuredCostBenchmarkIds":[],"coverageFloor":1,"tauResid":0.488,"calibrationTauResidSq":0.23812166,"priorMean":-0.29718,"tauModelSq":0.992496,"anchorLabs":9,"withheldBenchmarkIds":[],"linkJointCov":{"benchmarkIds":["epoch_cursorbench"],"covMatrix":[[0.7779303973457067,-0.014166086849922143],[-0.014166086849922143,0.00026179564728802526]]},"epochLink":{"previousEpoch":"coding-v1.0.0","linked":true,"overlapCount":189,"overlapLabs":9,"slope":0.850603,"intercept":0.031054,"pearsonR":0.9392,"reason":"Verified Mean-Sigma link (n=189, labs=9, r=0.939)"}}},"anchorStats":{"frontiercode_2_0::standardized":{"benchmarkId":"frontiercode_2_0","mean":22.771472,"sd":11.649818,"anchorCount":74,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":0.153084,"reliability":0.9048,"discrimination":0.6042,"redundancyFactor":0.2743,"effectiveWeight":0.15,"taskFamily":"repo_swe","datasetLineageId":"aa-frontiercode-2.0","introducedAt":"2026-08-01","invertScore":false,"popoviciuSe":4.351941,"boundedMax":100,"sampleSize":132},"terminalbench_4_0::standardized":{"benchmarkId":"terminalbench_4_0","mean":28.446907,"sd":22.505511,"anchorCount":78,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":0.141804,"reliability":0.9535,"discrimination":0.4615,"redundancyFactor":0.5286,"effectiveWeight":0.2326,"taskFamily":"terminal_ops","datasetLineageId":"terminal-bench-4.0","introducedAt":"2026-07-15","invertScore":false,"popoviciuSe":6.154575,"boundedMax":100,"sampleSize":66},"scicode_aa::standardized":{"benchmarkId":"scicode_aa","mean":51.902681,"sd":7.721951,"anchorCount":63,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":0.197784,"reliability":0.3804,"discrimination":0.1308,"redundancyFactor":0.7018,"effectiveWeight":0.1,"taskFamily":"scientific_coding","datasetLineageId":"scicode-1.0","introducedAt":"2026-05-01","invertScore":false,"popoviciuSe":7.418702,"boundedMax":100,"sampleSize":45.423877},"epoch_mirrorcode::standardized":{"benchmarkId":"epoch_mirrorcode","mean":49.615864,"sd":17.79024,"anchorCount":32,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":-0.020835,"reliability":0.6027,"discrimination":0.4375,"redundancyFactor":0.2193,"effectiveWeight":0.1,"taskFamily":"repo_swe","datasetLineageId":"epoch-mirrorcode","introducedAt":"2026-06-26","invertScore":false,"popoviciuSe":9.128709,"boundedMax":100,"sampleSize":30},"epoch_ale_bench::standardized":{"benchmarkId":"epoch_ale_bench","mean":1542.619321,"sd":505.843109,"anchorCount":33,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":0.382739,"reliability":0.5,"discrimination":0.7494,"redundancyFactor":0.4982,"effectiveWeight":0.1867,"taskFamily":"algorithmic_optimization","datasetLineageId":"sakana-ale-bench","introducedAt":"2026-08-01","invertScore":false},"epoch_frontierswe::standardized":{"benchmarkId":"epoch_frontierswe","mean":32.688375,"sd":17.043487,"anchorCount":22,"anchorLabs":8,"linkedViaMeanSigma":false,"raschDelta":-0.085499,"reliability":0.7969,"discrimination":0.6518,"redundancyFactor":0.1637,"effectiveWeight":0.1,"taskFamily":"repo_swe","datasetLineageId":"frontierswe","introducedAt":"2026-09-02","invertScore":false,"popoviciuSe":8.574929,"boundedMax":100,"sampleSize":34},"lmarena_agent::standardized":{"benchmarkId":"lmarena_agent","mean":1144.496399,"sd":95.845256,"anchorCount":44,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":-0.047179,"reliability":0.9857,"discrimination":0.8542,"redundancyFactor":0.4153,"effectiveWeight":0.3497,"taskFamily":"live_agentic","datasetLineageId":"lmarena-agent-code","introducedAt":"2026-10-01","invertScore":false},"surge_sudo_l7::standardized":{"benchmarkId":"surge_sudo_l7","mean":21.529375,"sd":12.771979,"anchorCount":21,"anchorLabs":8,"linkedViaMeanSigma":false,"raschDelta":-0.517325,"reliability":0.8408,"discrimination":0.6518,"redundancyFactor":0.5189,"effectiveWeight":0.2844,"taskFamily":"systems_engineering","datasetLineageId":"surge-sudo-l7","introducedAt":"2026-10-08","invertScore":false,"popoviciuSe":6.454972,"boundedMax":100,"sampleSize":60},"vals_code_migration::standardized":{"benchmarkId":"vals_code_migration","mean":55.376852,"sd":16.096045,"anchorCount":18,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":-0.022165,"reliability":0.9648,"discrimination":0.875,"redundancyFactor":0.4283,"effectiveWeight":0.3616,"taskFamily":"code_migration","datasetLineageId":"vals-code-migration-1.1","introducedAt":"2026-09-01","invertScore":false,"popoviciuSe":4.38529,"boundedMax":100,"sampleSize":130},"vals_sre_bench::standardized":{"benchmarkId":"vals_sre_bench","mean":10.403698,"sd":13.648239,"anchorCount":24,"anchorLabs":8,"linkedViaMeanSigma":false,"raschDelta":0.273492,"reliability":0.9856,"discrimination":0.8929,"redundancyFactor":0.3109,"effectiveWeight":0.2736,"taskFamily":"terminal_ops","datasetLineageId":"vals-sre-bench-1","introducedAt":"2026-09-15","invertScore":false,"popoviciuSe":3.08901,"boundedMax":100,"sampleSize":262},"livebench_long_agentic::standardized":{"benchmarkId":"livebench_long_agentic","mean":44.78254,"sd":17.799477,"anchorCount":26,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":-0.325066,"reliability":0.7083,"discrimination":0.561,"redundancyFactor":0.3192,"effectiveWeight":0.1268,"taskFamily":"live_agentic","datasetLineageId":"livebench-2-long-agentic","introducedAt":"2026-10-14","invertScore":false,"popoviciuSe":11.785113,"boundedMax":100,"sampleSize":18},"epoch_cursorbench::standardized":{"benchmarkId":"epoch_cursorbench","mean":42.05443,"sd":11.013814,"anchorCount":43,"anchorLabs":5,"linkedViaMeanSigma":true,"raschDelta":0.014832,"reliability":0.5,"discrimination":0.173,"redundancyFactor":0.2241,"effectiveWeight":0.1,"taskFamily":"repo_swe","datasetLineageId":"cursor-cursorbench","introducedAt":"2026-09-10","invertScore":false,"linkVarIntercept":0.7779303973457067,"linkVarSlope":0.00026179564728802526,"linkCovIS":-0.014166086849922143}},"configs":[{"id":"anthropic:claude-fable-5.2:max:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-fable-5.2","displayName":"Claude Fable 5.2","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":2.401742,"shrinkageFactor":0.9742,"score":86.03,"se":3.16,"samplingSe":2.4003,"residualSe":2.0514,"coverageCount":7,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":26.538558,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1000000,"releaseDate":"2026-11-09","paretoProb":0.9152,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:claude-fable-5-2_max:max:","observedHarness":"Cursor","rawScore":61.3,"zScore":1.747403,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":48.11,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:claude-fable-5-2_max:max:","observedHarness":"Proximus","rawScore":69.94,"zScore":2.185681,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":168.78,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:593f2d23-247b-41c9-a679-cebd225f0a9a","observedHarness":"Stirrup","rawScore":48.76,"zScore":2.23081,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Claude Fable 5.2 (Max)","observedHarness":"LMArena Agent Harness","rawScore":1488.82,"zScore":3.5,"seModel":"upstream_ci","ciHalfWidth":55.2022,"ciLower":1433.6202,"ciUpper":1544.0245,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:593f2d23-247b-41c9-a679-cebd225f0a9a","observedHarness":"Direct API","rawScore":67.71,"zScore":2.047063,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:Claude Fable 5.2 (Adaptive/Max)","observedHarness":"Direct API","rawScore":52.2,"zScore":2.4014,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:593f2d23-247b-41c9-a679-cebd225f0a9a","observedHarness":"mini-SWE-agent","rawScore":59.6,"zScore":1.384243,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"anthropic:claude-opus-5.5:max:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.966179,"shrinkageFactor":0.977352,"score":79.49,"se":2.85,"samplingSe":2.2489,"residualSe":1.7523,"coverageCount":9,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":10.03641,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.6439,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:claude-opus-5-5_max:max:","observedHarness":"Cursor","rawScore":58.1,"zScore":1.456859,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":11.88,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:claude-opus-5-5_max:max:","observedHarness":"Proximus","rawScore":62.33,"zScore":1.739176,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":98.87,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:M2dkPVSr5EVDErXzKnSecn","observedHarness":"Epoch MirrorCode Scaffold","rawScore":95.56,"zScore":2.582547,"seModel":"upstream_se","seRaw":2.55452,"ciHalfWidth":5.0069,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-30","publishedDate":"2026-10-30","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:2f3c4dc9-a450-4303-8697-0237257cf08f","observedHarness":"Stirrup","rawScore":46.44,"zScore":2.031665,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:claude-opus-5.5","observedHarness":"Direct API","rawScore":72.8,"zScore":1.574061,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":83.917756,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Claude Opus 5.5 (Max)","observedHarness":"LMArena Agent Harness","rawScore":1372.52,"zScore":2.379081,"seModel":"upstream_ci","ciHalfWidth":39.9353,"ciLower":1332.588,"ciUpper":1412.4587,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:2f3c4dc9-a450-4303-8697-0237257cf08f","observedHarness":"Direct API","rawScore":66.9,"zScore":1.942167,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:Claude Opus 5.5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":46.1,"zScore":1.923792,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:2f3c4dc9-a450-4303-8697-0237257cf08f","observedHarness":"mini-SWE-agent","rawScore":59.6,"zScore":1.384243,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"anthropic:claude-sonnet-5.5:max:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.8432,"shrinkageFactor":0.983204,"score":77.65,"se":2.24,"samplingSe":1.9367,"residualSe":1.1259,"coverageCount":10,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":10.020621,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.7871,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:claude-sonnet-5-5_max:max:","observedHarness":"Cursor","rawScore":54.4,"zScore":1.120917,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":8.63,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:claude-sonnet-5-5_max:max:","observedHarness":"Proximus","rawScore":61.89,"zScore":1.71336,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":110.45,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:UTD5SC5v2BgiwNauF9YrFo","observedHarness":"Epoch MirrorCode Scaffold","rawScore":97.78,"zScore":2.707335,"seModel":"upstream_se","seRaw":1.514189,"ciHalfWidth":2.9678,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-05","publishedDate":"2026-10-05","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:b171d979-5ec1-45de-b8b9-db1ee65e77ec","observedHarness":"Stirrup","rawScore":41.7,"zScore":1.624792,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:claude-sonnet-5.5","observedHarness":"Direct API","rawScore":62.5,"zScore":0.995392,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":69.888089,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Claude Sonnet 5.5 (Max)","observedHarness":"LMArena Agent Harness","rawScore":1329.27,"zScore":1.927833,"seModel":"upstream_ci","ciHalfWidth":53.3504,"ciLower":1275.9244,"ciUpper":1382.6251,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:b171d979-5ec1-45de-b8b9-db1ee65e77ec","observedHarness":"Direct API","rawScore":61,"zScore":1.178111,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:Claude Sonnet 5.5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":50.6,"zScore":2.276125,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:b171d979-5ec1-45de-b8b9-db1ee65e77ec","observedHarness":"mini-SWE-agent","rawScore":63.64,"zScore":1.563754,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"vals_code_migration","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:code-migration-v1-1:anthropic/claude-sonnet-5-5","observedHarness":"Pi","rawScore":86.88,"zScore":1.957198,"seModel":"upstream_se","seRaw":1.521,"ciHalfWidth":2.9812,"ciLower":null,"ciUpper":null,"measuredCostUsd":43.879213,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/code-migration-v1-1"}]},{"id":"openai:gpt-6-astra:max:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6-astra","displayName":"GPT-6 Astra","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.811072,"shrinkageFactor":0.982328,"score":77.17,"se":2.3,"samplingSe":1.9865,"residualSe":1.1688,"coverageCount":8,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":27.818269,"costLogSd":1.0653,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.3696,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recie6OeSzfrEhXOW","observedHarness":"ALE-Agent","rawScore":2951.32,"zScore":2.784857,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.86415,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:gpt-6-astra_max:max:","observedHarness":"Proximus","rawScore":65.51,"zScore":1.925758,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1029.65,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:FGZUtmNQZpZtk8ft5uCJkA","observedHarness":"Epoch MirrorCode Scaffold","rawScore":60,"zScore":0.583698,"seModel":"upstream_se","seRaw":11.818737,"ciHalfWidth":23.1647,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-29","publishedDate":"2026-10-29","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:gpt-6-astra","observedHarness":"Direct API","rawScore":74.7,"zScore":1.680806,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":50.620217,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:GPT-6 Astra (Max)","observedHarness":"LMArena Agent Harness","rawScore":1288.22,"zScore":1.499538,"seModel":"upstream_ci","ciHalfWidth":36.3365,"ciLower":1251.8816,"ciUpper":1324.5546,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:GPT 6 Astra (Max reasoning)","observedHarness":"Direct API","rawScore":31.1,"zScore":0.749346,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"},{"benchmarkId":"vals_code_migration","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:code-migration-v1-1:openai/gpt-6-astra","observedHarness":"Pi","rawScore":80.58,"zScore":1.565798,"seModel":"upstream_se","seRaw":2.336,"ciHalfWidth":4.5786,"ciLower":null,"ciUpper":null,"measuredCostUsd":62.066613,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/code-migration-v1-1"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:openai/gpt-6-astra","observedHarness":"Vals SRE Runner","rawScore":56.87,"zScore":3.404564,"seModel":"upstream_se","seRaw":3.066,"ciHalfWidth":6.0094,"ciLower":null,"ciUpper":null,"measuredCostUsd":10.238062,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"google:gemini-4-neon:xhigh:standardized-suite:standardized","provider":"google","modelSlug":"gemini-4-neon","displayName":"Gemini 4 Neon","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.725908,"shrinkageFactor":0.95675,"score":75.89,"se":4.61,"samplingSe":3.1078,"residualSe":3.4095,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":6.444838,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1048576,"releaseDate":"2026-11-18","paretoProb":0.545,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:gemini-4-neon_xhigh:xhigh:","observedHarness":"Cursor","rawScore":65.4,"zScore":2.119663,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":10.67,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:7c667f2d-0149-4eb1-864c-7e66c166ad88","observedHarness":"Stirrup","rawScore":44.41,"zScore":1.857413,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:gemini-4-neon","observedHarness":"Direct API","rawScore":82.6,"zScore":2.124639,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":13.120544,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:7c667f2d-0149-4eb1-864c-7e66c166ad88","observedHarness":"mini-SWE-agent","rawScore":63.64,"zScore":1.563754,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-6.2-astra:max:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6.2-astra","displayName":"GPT-6.2 Astra","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.69096,"shrinkageFactor":0.970019,"score":75.36,"se":3.37,"samplingSe":2.5875,"residualSe":2.1634,"coverageCount":6,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":3.997906,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.2804,"benchmarks":[{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:R7MHmgZqxmYVVTvzpjvc5k","observedHarness":"Epoch MirrorCode Scaffold","rawScore":67.78,"zScore":1.021017,"seModel":"upstream_se","seRaw":11.668556,"ciHalfWidth":22.8704,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-27","publishedDate":"2026-10-27","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:75c1b0b8-9db3-4c7e-95be-d74e59defee4","observedHarness":"Stirrup","rawScore":45.38,"zScore":1.940676,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:gpt-6.2-astra","observedHarness":"Direct API","rawScore":70.8,"zScore":1.461698,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":17.023439,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:GPT-6.2 Astra (Max)","observedHarness":"LMArena Agent Harness","rawScore":1338.34,"zScore":2.022464,"seModel":"upstream_ci","ciHalfWidth":55.7431,"ciLower":1282.5941,"ciUpper":1394.0802,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:GPT 6.2 Astra (Max reasoning)","observedHarness":"Direct API","rawScore":38.3,"zScore":1.31308,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:75c1b0b8-9db3-4c7e-95be-d74e59defee4","observedHarness":"mini-SWE-agent","rawScore":64.14,"zScore":1.585971,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"google:gemini-4-neon:high:standardized-suite:standardized","provider":"google","modelSlug":"gemini-4-neon","displayName":"Gemini 4 Neon","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.677795,"shrinkageFactor":0.969779,"score":75.17,"se":3.55,"samplingSe":2.5978,"residualSe":2.4174,"coverageCount":5,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.74275,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1048576,"releaseDate":"2026-11-18","paretoProb":0.908,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:gemini-4-neon_high:high:","observedHarness":"Cursor","rawScore":63.7,"zScore":1.965311,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.54,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:9b696f5b-b5d2-4b1f-9ed5-7c1446ec8421","observedHarness":"Stirrup","rawScore":43.28,"zScore":1.760416,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Gemini 4 Neon (High)","observedHarness":"LMArena Agent Harness","rawScore":1325.18,"zScore":1.88516,"seModel":"upstream_ci","ciHalfWidth":44.073,"ciLower":1281.1109,"ciUpper":1369.257,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:Gemini 4 Neon (High reasoning)","observedHarness":"Direct API","rawScore":38.3,"zScore":1.31308,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:9b696f5b-b5d2-4b1f-9ed5-7c1446ec8421","observedHarness":"mini-SWE-agent","rawScore":61.62,"zScore":1.473999,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"anthropic:claude-fable-5.2:xhigh:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-fable-5.2","displayName":"Claude Fable 5.2","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.634568,"shrinkageFactor":0.952671,"score":74.52,"se":4.46,"samplingSe":3.251,"residualSe":3.0604,"coverageCount":5,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":14.828588,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1000000,"releaseDate":"2026-11-09","paretoProb":0.1153,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:claude-fable-5-2_xhigh:xhigh:","observedHarness":"Cursor","rawScore":58.8,"zScore":1.520415,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":17.84,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:76942526-cc96-46b0-bc27-29561c53ed70","observedHarness":"Stirrup","rawScore":47.58,"zScore":2.129521,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:claude-fable-5.2","observedHarness":"Direct API","rawScore":82.9,"zScore":2.141493,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":160.927911,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:76942526-cc96-46b0-bc27-29561c53ed70","observedHarness":"Direct API","rawScore":65.63,"zScore":1.777701,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:76942526-cc96-46b0-bc27-29561c53ed70","observedHarness":"mini-SWE-agent","rawScore":61.62,"zScore":1.473999,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-6.2-astra:xhigh:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6.2-astra","displayName":"GPT-6.2 Astra","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.608815,"shrinkageFactor":0.954845,"score":74.13,"se":5.69,"samplingSe":3.1755,"residualSe":4.7249,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":2.521143,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.3187,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:592572d5-f789-4106-abbc-fadf2206d10b","observedHarness":"Stirrup","rawScore":48.95,"zScore":2.247119,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:592572d5-f789-4106-abbc-fadf2206d10b","observedHarness":"mini-SWE-agent","rawScore":64.14,"zScore":1.585971,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-6.1-sol:max:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6.1-sol","displayName":"GPT-6.1 Sol","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.547223,"shrinkageFactor":0.980338,"score":73.21,"se":2.47,"samplingSe":2.0954,"residualSe":1.3009,"coverageCount":7,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.487251,"costLogSd":1.048,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.2106,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recbQjMjXINIX4xjc","observedHarness":"ALE-Agent","rawScore":2743.75,"zScore":2.374512,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.596828,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:jGPEpcBgJ4gakLdxgAqbR8","observedHarness":"Epoch MirrorCode Scaffold","rawScore":61.11,"zScore":0.646092,"seModel":"upstream_se","seRaw":11.346709,"ciHalfWidth":22.2395,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-05","publishedDate":"2026-10-05","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:gpt-6.1-sol","observedHarness":"Direct API","rawScore":64.7,"zScore":1.118991,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.421489,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:GPT-6.1 Sol (Max)","observedHarness":"LMArena Agent Harness","rawScore":1245.91,"zScore":1.058097,"seModel":"upstream_ci","ciHalfWidth":20.9517,"ciLower":1224.9559,"ciUpper":1266.8593,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:GPT 6.1 Sol (Max reasoning)","observedHarness":"Direct API","rawScore":33.9,"zScore":0.968575,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"},{"benchmarkId":"vals_code_migration","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:code-migration-v1-1:openai/gpt-6.1-sol","observedHarness":"Pi","rawScore":77.32,"zScore":1.363263,"seModel":"upstream_se","seRaw":2.976,"ciHalfWidth":5.833,"ciLower":null,"ciUpper":null,"measuredCostUsd":9.505623,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/code-migration-v1-1"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:openai/gpt-6.1-sol","observedHarness":"Vals SRE Runner","rawScore":50.76,"zScore":2.956887,"seModel":"upstream_se","seRaw":3.095,"ciHalfWidth":6.0662,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.685175,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"anthropic:claude-fable-5.1:max:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-fable-5.1","displayName":"Claude Fable 5.1","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.51297,"shrinkageFactor":0.965717,"score":72.69,"se":3.9,"samplingSe":2.7669,"residualSe":2.7523,"coverageCount":5,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":27.818269,"costLogSd":1.0653,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-09-12","paretoProb":0.2284,"benchmarks":[{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:claude-fable-5-1_max:max:","observedHarness":"Proximus","rawScore":56.29,"zScore":1.384789,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":138.55,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:j3QzF2QVnNghvyFneqEuiu","observedHarness":"Epoch MirrorCode Scaffold","rawScore":93.33,"zScore":2.457198,"seModel":"upstream_se","seRaw":3.917131,"ciHalfWidth":7.6776,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-09","publishedDate":"2026-09-09","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:claude-fable-5.1","observedHarness":"Direct API","rawScore":74.2,"zScore":1.652715,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":80.293672,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Claude Fable 5.1 (Max)","observedHarness":"LMArena Agent Harness","rawScore":1272.96,"zScore":1.340323,"seModel":"upstream_ci","ciHalfWidth":41.123,"ciLower":1231.8324,"ciUpper":1314.0785,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:Claude Fable 5.1 (Adaptive/Max)","observedHarness":"Direct API","rawScore":32.2,"zScore":0.835472,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"}]},{"id":"openai:gpt-6.2-sol:max:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6.2-sol","displayName":"GPT-6.2 Sol","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.461382,"shrinkageFactor":0.969315,"score":71.92,"se":3.52,"samplingSe":2.6177,"residualSe":2.3585,"coverageCount":5,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":1.01438,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.3959,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:a1a84b18-e8c9-48fb-8e04-6003bd9f6ff4","observedHarness":"Stirrup","rawScore":44.38,"zScore":1.854838,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:gpt-6.2-sol","observedHarness":"Direct API","rawScore":63.6,"zScore":1.057192,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.074128,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:GPT-6.2 Sol (Max)","observedHarness":"LMArena Agent Harness","rawScore":1287.56,"zScore":1.492652,"seModel":"upstream_ci","ciHalfWidth":49.2381,"ciLower":1238.322,"ciUpper":1336.7981,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:GPT 6.2 Sol (Max reasoning)","observedHarness":"Direct API","rawScore":36.1,"zScore":1.140828,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:a1a84b18-e8c9-48fb-8e04-6003bd9f6ff4","observedHarness":"mini-SWE-agent","rawScore":60.1,"zScore":1.40646,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-6.2-astra:high:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6.2-astra","displayName":"GPT-6.2 Astra","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.444467,"shrinkageFactor":0.953936,"score":71.67,"se":5.71,"samplingSe":3.2073,"residualSe":4.7204,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.073875,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.2417,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:fe959a7a-944c-4ba8-8e41-349712599b39","observedHarness":"Stirrup","rawScore":47.13,"zScore":2.090893,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:fe959a7a-944c-4ba8-8e41-349712599b39","observedHarness":"mini-SWE-agent","rawScore":60.1,"zScore":1.40646,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"google:gemini-4-carbon:high:standardized-suite:standardized","provider":"google","modelSlug":"gemini-4-carbon","displayName":"Gemini 4 Carbon","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.443355,"shrinkageFactor":0.976543,"score":71.65,"se":2.87,"samplingSe":2.2887,"residualSe":1.7362,"coverageCount":8,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":7.755546,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-10-12","paretoProb":0.0637,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recouaA1ujfzoOmWb","observedHarness":"ALE-Agent","rawScore":2374.7,"zScore":1.644938,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.068274,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:gemini-4-carbon_high:high:","observedHarness":"Cursor","rawScore":61.5,"zScore":1.765562,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":8.52,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:gemini-4-carbon_high:high:","observedHarness":"Proximus","rawScore":61.2,"zScore":1.672875,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":61.93,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:909ed0ba-bcc5-4f38-8162-08a4e187a772","observedHarness":"Stirrup","rawScore":40.56,"zScore":1.526936,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:gemini-4-carbon","observedHarness":"Direct API","rawScore":71.8,"zScore":1.517879,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":9.054861,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Gemini 4 Carbon (High)","observedHarness":"LMArena Agent Harness","rawScore":1290.94,"zScore":1.527917,"seModel":"upstream_ci","ciHalfWidth":12.172,"ciLower":1278.7704,"ciUpper":1303.1144,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:Gemini 4 Carbon (High reasoning)","observedHarness":"Direct API","rawScore":30.6,"zScore":0.710197,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:909ed0ba-bcc5-4f38-8162-08a4e187a772","observedHarness":"mini-SWE-agent","rawScore":65.15,"zScore":1.630849,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-6.2-astra:medium:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6.2-astra","displayName":"GPT-6.2 Astra","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.386816,"shrinkageFactor":0.953686,"score":70.8,"se":5.71,"samplingSe":3.216,"residualSe":4.7192,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.617688,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.2627,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:8cec011f-c959-4726-8668-2ee2409a6bca","observedHarness":"Stirrup","rawScore":46.56,"zScore":2.041966,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:8cec011f-c959-4726-8668-2ee2409a6bca","observedHarness":"mini-SWE-agent","rawScore":58.59,"zScore":1.339365,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"anthropic:claude-fable-5.2:high:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-fable-5.2","displayName":"Claude Fable 5.2","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.322493,"shrinkageFactor":0.938473,"score":69.84,"se":4.72,"samplingSe":3.7067,"residualSe":2.9196,"coverageCount":5,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":9.078415,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1000000,"releaseDate":"2026-11-09","paretoProb":0.2583,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recQTbO5YuBHhJdH6","observedHarness":"ALE-Agent","rawScore":2509.8,"zScore":1.912017,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.503272,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:claude-fable-5-2_high:high:","observedHarness":"Cursor","rawScore":56.5,"zScore":1.311587,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":10.51,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:fdc1913b-9497-45ef-83f7-17dae756f37e","observedHarness":"Stirrup","rawScore":49.13,"zScore":2.26257,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:fdc1913b-9497-45ef-83f7-17dae756f37e","observedHarness":"Direct API","rawScore":59.49,"zScore":0.982565,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:fdc1913b-9497-45ef-83f7-17dae756f37e","observedHarness":"mini-SWE-agent","rawScore":59.6,"zScore":1.384243,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"anthropic:claude-opus-5.5:xhigh:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.306408,"shrinkageFactor":0.942496,"score":69.6,"se":4.96,"samplingSe":3.5835,"residualSe":3.4315,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":6.047194,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.3581,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:claude-opus-5-5_xhigh:xhigh:","observedHarness":"Cursor","rawScore":56.7,"zScore":1.329746,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.54,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:df5bf99d-d228-4715-b41b-19ca86118777","observedHarness":"Stirrup","rawScore":43.75,"zScore":1.80076,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:df5bf99d-d228-4715-b41b-19ca86118777","observedHarness":"Direct API","rawScore":65.05,"zScore":1.70259,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:df5bf99d-d228-4715-b41b-19ca86118777","observedHarness":"mini-SWE-agent","rawScore":59.6,"zScore":1.384243,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"google:gemini-4-carbon:medium:standardized-suite:standardized","provider":"google","modelSlug":"gemini-4-carbon","displayName":"Gemini 4 Carbon","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.277572,"shrinkageFactor":0.950301,"score":69.16,"se":5.17,"samplingSe":3.3314,"residualSe":3.954,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.585777,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-10-12","paretoProb":0.0791,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:gemini-4-carbon_medium:medium:","observedHarness":"Cursor","rawScore":60.4,"zScore":1.665687,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.22,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:0d3eaafd-54ed-4f78-bdd7-05a2db883986","observedHarness":"Stirrup","rawScore":39.45,"zScore":1.431656,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:0d3eaafd-54ed-4f78-bdd7-05a2db883986","observedHarness":"mini-SWE-agent","rawScore":60.61,"zScore":1.429121,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"anthropic:claude-opus-5.5:high:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.23938,"shrinkageFactor":0.972587,"score":68.59,"se":3.33,"samplingSe":2.4742,"residualSe":2.2236,"coverageCount":7,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.464662,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.6045,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:rec2XltGq6LqiHAqJ","observedHarness":"ALE-Agent","rawScore":2147.38,"zScore":1.19555,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.271178,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:claude-opus-5-5_high:high:","observedHarness":"Cursor","rawScore":54.5,"zScore":1.129996,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.87,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:mirrorcode-ml-2l-claude-opus-5-5","observedHarness":"Epoch MirrorCode Scaffold","rawScore":78.89,"zScore":1.645517,"seModel":"upstream_se","seRaw":7.88,"ciHalfWidth":15.4448,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-22","publishedDate":"2026-09-22","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:f314eade-2232-44bf-a3ab-9b326bc67de6","observedHarness":"Stirrup","rawScore":35.93,"zScore":1.129505,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Claude Opus 5.5 (High)","observedHarness":"LMArena Agent Harness","rawScore":1315.12,"zScore":1.780199,"seModel":"upstream_ci","ciHalfWidth":19.6966,"ciLower":1295.4262,"ciUpper":1334.8194,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:f314eade-2232-44bf-a3ab-9b326bc67de6","observedHarness":"Direct API","rawScore":60.42,"zScore":1.103001,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:f314eade-2232-44bf-a3ab-9b326bc67de6","observedHarness":"mini-SWE-agent","rawScore":56.57,"zScore":1.249609,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-6.1-terra:max:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.227976,"shrinkageFactor":0.971644,"score":68.42,"se":3.33,"samplingSe":2.5164,"residualSe":2.1774,"coverageCount":7,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.961108,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.2154,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recHSwa2wuj8rGCoD","observedHarness":"ALE-Agent","rawScore":2608.82,"zScore":2.10777,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.413917,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:QTeGw8iACPtPrzw9JCLEfR","observedHarness":"Epoch MirrorCode Scaffold","rawScore":76.67,"zScore":1.520729,"seModel":"upstream_se","seRaw":9.315175,"ciHalfWidth":18.2577,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-30","publishedDate":"2026-10-30","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:b0c462cf-f2d0-4126-9434-28db9502219f","observedHarness":"Stirrup","rawScore":42.32,"zScore":1.678011,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:gpt-6.1-terra","observedHarness":"Direct API","rawScore":55.3,"zScore":0.590886,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.145944,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:GPT-6.1 Terra (Max)","observedHarness":"LMArena Agent Harness","rawScore":1275.76,"zScore":1.369537,"seModel":"upstream_ci","ciHalfWidth":21.5079,"ciLower":1254.2504,"ciUpper":1297.2662,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:b0c462cf-f2d0-4126-9434-28db9502219f","observedHarness":"Direct API","rawScore":55.09,"zScore":0.412761,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:b0c462cf-f2d0-4126-9434-28db9502219f","observedHarness":"mini-SWE-agent","rawScore":55.56,"zScore":1.204731,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-6.2-sol:xhigh:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6.2-sol","displayName":"GPT-6.2 Sol","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.199281,"shrinkageFactor":0.953637,"score":67.99,"se":5.71,"samplingSe":3.2177,"residualSe":4.7189,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.647968,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.3145,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:27c326a6-ec9b-415f-a29e-b1f4927484dc","observedHarness":"Stirrup","rawScore":42.75,"zScore":1.714922,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:27c326a6-ec9b-415f-a29e-b1f4927484dc","observedHarness":"mini-SWE-agent","rawScore":56.06,"zScore":1.226948,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-6.2-sol:high:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6.2-sol","displayName":"GPT-6.2 Sol","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.184345,"shrinkageFactor":0.954336,"score":67.77,"se":5.7,"samplingSe":3.1933,"residualSe":4.7224,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.563645,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.4369,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:cd85e412-c43f-4535-bf8e-5ba113d97ca3","observedHarness":"Stirrup","rawScore":40.22,"zScore":1.497751,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:cd85e412-c43f-4535-bf8e-5ba113d97ca3","observedHarness":"mini-SWE-agent","rawScore":58.59,"zScore":1.339365,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-6.2-sol:medium:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6.2-sol","displayName":"GPT-6.2 Sol","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.156208,"shrinkageFactor":0.953803,"score":67.34,"se":5.71,"samplingSe":3.2119,"residualSe":4.7197,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.406056,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.7508,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:82b37079-56d9-4451-8a1a-22b0a921f4f8","observedHarness":"Stirrup","rawScore":41.4,"zScore":1.59904,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:82b37079-56d9-4451-8a1a-22b0a921f4f8","observedHarness":"mini-SWE-agent","rawScore":56.06,"zScore":1.226948,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"google:gemini-4-neon:medium:standardized-suite:standardized","provider":"google","modelSlug":"gemini-4-neon","displayName":"Gemini 4 Neon","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.042073,"shrinkageFactor":0.950298,"score":65.63,"se":5.17,"samplingSe":3.3315,"residualSe":3.9539,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":1.904556,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1048576,"releaseDate":"2026-11-18","paretoProb":0.5186,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:gemini-4-neon_medium:medium:","observedHarness":"Cursor","rawScore":57.7,"zScore":1.420541,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.07,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:2ba1a8f6-e052-4a33-b178-4db1e86157e4","observedHarness":"Stirrup","rawScore":36.93,"zScore":1.215343,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:2ba1a8f6-e052-4a33-b178-4db1e86157e4","observedHarness":"mini-SWE-agent","rawScore":54.55,"zScore":1.159853,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"anthropic:claude-fable-5.2:medium:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-fable-5.2","displayName":"Claude Fable 5.2","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.029031,"shrinkageFactor":0.941157,"score":65.44,"se":4.99,"samplingSe":3.625,"residualSe":3.4266,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.802605,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1000000,"releaseDate":"2026-11-09","paretoProb":0.2033,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:claude-fable-5-2_medium:medium:","observedHarness":"Cursor","rawScore":54.2,"zScore":1.102758,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.01,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:0c9b4e05-9259-42cc-b40e-d6acbbeaa7e8","observedHarness":"Stirrup","rawScore":41.3,"zScore":1.590456,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:0c9b4e05-9259-42cc-b40e-d6acbbeaa7e8","observedHarness":"Direct API","rawScore":59.72,"zScore":1.01235,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:0c9b4e05-9259-42cc-b40e-d6acbbeaa7e8","observedHarness":"mini-SWE-agent","rawScore":55.05,"zScore":1.18207,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"google:gemini-4-hydrogen:high:standardized-suite:standardized","provider":"google","modelSlug":"gemini-4-hydrogen","displayName":"Gemini 4 Hydrogen","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.016391,"shrinkageFactor":0.98,"score":65.25,"se":3.54,"samplingSe":2.1134,"residualSe":2.8361,"coverageCount":5,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.873314,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":1.5,"outputPer1mUsd":7.5,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.9548,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:gemini-4-hydrogen_high:high:","observedHarness":"Cursor","rawScore":57.3,"zScore":1.384223,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.57,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:gemini-4-hydrogen_high:high:","observedHarness":"Proximus","rawScore":53.11,"zScore":1.198207,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":61.79,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:dae0262d-26a4-44d1-925a-6f2731cbda6a","observedHarness":"Stirrup","rawScore":36.33,"zScore":1.16384,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Gemini 4 Hydrogen (High)","observedHarness":"LMArena Agent Harness","rawScore":1228.81,"zScore":0.879685,"seModel":"upstream_ci","ciHalfWidth":29.4921,"ciLower":1199.3154,"ciUpper":1258.2996,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:dae0262d-26a4-44d1-925a-6f2731cbda6a","observedHarness":"mini-SWE-agent","rawScore":54.04,"zScore":1.137192,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"google:gemini-4-hydrogen:xhigh:standardized-suite:standardized","provider":"google","modelSlug":"gemini-4-hydrogen","displayName":"Gemini 4 Hydrogen","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":1.00777,"shrinkageFactor":0.955176,"score":65.12,"se":3.98,"samplingSe":3.1638,"residualSe":2.4136,"coverageCount":6,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.160159,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":1.5,"outputPer1mUsd":7.5,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.3058,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:reciIQ1ZPayHYVunu","observedHarness":"ALE-Agent","rawScore":1825.67,"zScore":0.559562,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.285945,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:gemini-4-hydrogen_xhigh:xhigh:","observedHarness":"Cursor","rawScore":61.4,"zScore":1.756482,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":8.09,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:dad0e3b1-e3e7-477a-99f9-c14457a91114","observedHarness":"Stirrup","rawScore":38.96,"zScore":1.389595,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:gemini-4-hydrogen","observedHarness":"Direct API","rawScore":61.8,"zScore":0.956065,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.15235,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:Gemini 4 Hydrogen (xHigh reasoning)","observedHarness":"Direct API","rawScore":31.7,"zScore":0.796323,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:dad0e3b1-e3e7-477a-99f9-c14457a91114","observedHarness":"mini-SWE-agent","rawScore":53.54,"zScore":1.114975,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"google:gemini-4-carbon:low:standardized-suite:standardized","provider":"google","modelSlug":"gemini-4-carbon","displayName":"Gemini 4 Carbon","effort":"low","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.975792,"shrinkageFactor":0.95246,"score":64.64,"se":5.13,"samplingSe":3.2582,"residualSe":3.9629,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":7.186577,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-10-12","paretoProb":0.0101,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:gemini-4-carbon_low:low:","observedHarness":"Cursor","rawScore":55,"zScore":1.175394,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":8.64,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:b3866703-a9f7-4ea2-8f3f-b361c90aaf0e","observedHarness":"Stirrup","rawScore":29.62,"zScore":0.587866,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:b3866703-a9f7-4ea2-8f3f-b361c90aaf0e","observedHarness":"mini-SWE-agent","rawScore":62.63,"zScore":1.518877,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-6.1-sol:medium:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6.1-sol","displayName":"GPT-6.1 Sol","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.932313,"shrinkageFactor":0.991521,"score":63.98,"se":6.66,"samplingSe":1.376,"residualSe":6.521,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.968438,"costLogSd":1.0541,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.2149,"benchmarks":[{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:GPT-6.1 Sol (Medium)","observedHarness":"LMArena Agent Harness","rawScore":1230.34,"zScore":0.895648,"seModel":"upstream_ci","ciHalfWidth":17.3068,"ciLower":1213.0378,"ciUpper":1247.6513,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"}]},{"id":"openai:gpt-6-sol:max:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6-sol","displayName":"GPT-6 Sol","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.931634,"shrinkageFactor":0.967765,"score":63.97,"se":3.97,"samplingSe":2.683,"residualSe":2.924,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.487251,"costLogSd":1.048,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0304,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:receiDDkmha5O2EgZ","observedHarness":"ALE-Agent","rawScore":2462,"zScore":1.817521,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.843984,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:TQ4LXFJtzphWHf9brvHq5i","observedHarness":"Epoch MirrorCode Scaffold","rawScore":48.89,"zScore":-0.040801,"seModel":"upstream_se","seRaw":11.205945,"ciHalfWidth":21.9637,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-29","publishedDate":"2026-10-29","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:GPT-6 Sol (Max)","observedHarness":"LMArena Agent Harness","rawScore":1219.6,"zScore":0.783592,"seModel":"upstream_ci","ciHalfWidth":27.9816,"ciLower":1191.6135,"ciUpper":1247.5767,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:openai/gpt-6-sol","observedHarness":"Vals SRE Runner","rawScore":30.53,"zScore":1.474645,"seModel":"upstream_se","seRaw":2.851,"ciHalfWidth":5.588,"ciLower":null,"ciUpper":null,"measuredCostUsd":20.651048,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"google:gemini-4-argon:high:standardized-suite:standardized","provider":"google","modelSlug":"gemini-4-argon","displayName":"Gemini 4 Argon","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.927366,"shrinkageFactor":0.98675,"score":63.91,"se":1.72,"samplingSe":1.7201,"residualSe":0,"coverageCount":12,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":9.946609,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:rec1Zj9NnhZWnpeWx","observedHarness":"ALE-Agent","rawScore":2100.1,"zScore":1.102082,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.949743,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:gemini-4-argon_high:high:","observedHarness":"Cursor","rawScore":56.5,"zScore":1.311587,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":9.14,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:gemini-4-argon_high:high:","observedHarness":"Proximus","rawScore":54.96,"zScore":1.306753,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":129.36,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:h7xMzFUTMGSGqUSKUpxkUW","observedHarness":"Epoch MirrorCode Scaffold","rawScore":58.89,"zScore":0.521305,"seModel":"upstream_se","seRaw":11.713813,"ciHalfWidth":22.9591,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-14","publishedDate":"2026-10-14","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:d366dfa1-d940-4788-aa73-d298c45699b3","observedHarness":"Stirrup","rawScore":25.71,"zScore":0.252238,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:gemini-4-argon","observedHarness":"Direct API","rawScore":58.5,"zScore":0.770666,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":8.875,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Gemini 4 Argon (High)","observedHarness":"LMArena Agent Harness","rawScore":1212.63,"zScore":0.710871,"seModel":"upstream_ci","ciHalfWidth":16.2878,"ciLower":1196.3455,"ciUpper":1228.921,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:d366dfa1-d940-4788-aa73-d298c45699b3","observedHarness":"Direct API","rawScore":61.81,"zScore":1.283007,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:Gemini 4 Argon (High reasoning)","observedHarness":"Direct API","rawScore":16.1,"zScore":-0.425101,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:d366dfa1-d940-4788-aa73-d298c45699b3","observedHarness":"mini-SWE-agent","rawScore":57.07,"zScore":1.271826,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"vals_code_migration","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:code-migration-v1-1:google/gemini-4-argon","observedHarness":"Pi","rawScore":70.8,"zScore":0.958195,"seModel":"upstream_se","seRaw":3.723,"ciHalfWidth":7.2971,"ciLower":null,"ciUpper":null,"measuredCostUsd":44.790194,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/code-migration-v1-1"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:google/gemini-4-argon","observedHarness":"Vals SRE Runner","rawScore":44.27,"zScore":2.481368,"seModel":"upstream_se","seRaw":3.075,"ciHalfWidth":6.027,"ciLower":null,"ciUpper":null,"measuredCostUsd":30.451343,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"anthropic:claude-fable-5.2:low:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-fable-5.2","displayName":"Claude Fable 5.2","effort":"low","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.90805,"shrinkageFactor":0.941002,"score":63.62,"se":4.99,"samplingSe":3.6297,"residualSe":3.4261,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":3.097242,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1000000,"releaseDate":"2026-11-09","paretoProb":0.4309,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:claude-fable-5-2_low:low:","observedHarness":"Cursor","rawScore":53.4,"zScore":1.030122,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.26,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:cfc7acb1-d903-4858-9b6c-6c7776caf8f9","observedHarness":"Stirrup","rawScore":38.44,"zScore":1.344959,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:cfc7acb1-d903-4858-9b6c-6c7776caf8f9","observedHarness":"Direct API","rawScore":57.75,"zScore":0.757233,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:cfc7acb1-d903-4858-9b6c-6c7776caf8f9","observedHarness":"mini-SWE-agent","rawScore":54.55,"zScore":1.159853,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"anthropic:claude-fable-5:max:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-fable-5","displayName":"Claude Fable 5","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.869433,"shrinkageFactor":0.869353,"score":63.04,"se":7.08,"samplingSe":5.4014,"residualSe":4.5819,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":27.818269,"costLogSd":1.0653,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-06-09","paretoProb":0.0753,"benchmarks":[{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:claude-fable-5_max:max:","observedHarness":"Proximus","rawScore":46.96,"zScore":0.837365,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":301.28,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:5z3kmMLG4o5sSC6R7Ny4BS","observedHarness":"Epoch MirrorCode Scaffold","rawScore":70,"zScore":1.145804,"seModel":"upstream_se","seRaw":10.440053,"ciHalfWidth":20.4625,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-30","publishedDate":"2026-10-30","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"}]},{"id":"anthropic:claude-opus-5:max:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-opus-5","displayName":"Claude Opus 5","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.853219,"shrinkageFactor":0.980258,"score":62.8,"se":4.76,"samplingSe":2.0997,"residualSe":4.2753,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":14.827231,"costLogSd":1.0558,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-07-28","paretoProb":0.1378,"benchmarks":[{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:claude-opus-5_max:max:","observedHarness":"Proximus","rawScore":52.01,"zScore":1.133666,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":196.71,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:mNbPPJ3qiGSKnpqRsbxeKv","observedHarness":"Epoch MirrorCode Scaffold","rawScore":87.78,"zScore":2.145229,"seModel":"upstream_se","seRaw":7.176289,"ciHalfWidth":14.0655,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-20","publishedDate":"2026-10-20","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Claude Opus 5 (Max)","observedHarness":"LMArena Agent Harness","rawScore":1179.23,"zScore":0.362392,"seModel":"upstream_ci","ciHalfWidth":23.3319,"ciLower":1155.8997,"ciUpper":1202.5636,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"}]},{"id":"anthropic:claude-sonnet-5.5:xhigh:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.831353,"shrinkageFactor":0.941175,"score":62.47,"se":4.99,"samplingSe":3.6244,"residualSe":3.4267,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.091985,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.0695,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:claude-sonnet-5-5_xhigh:xhigh:","observedHarness":"Cursor","rawScore":49.9,"zScore":0.712339,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.13,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:975aeaef-50bd-4a65-9c73-f1015adcd4d4","observedHarness":"Stirrup","rawScore":35.65,"zScore":1.10547,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:975aeaef-50bd-4a65-9c73-f1015adcd4d4","observedHarness":"Direct API","rawScore":57.29,"zScore":0.697663,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:975aeaef-50bd-4a65-9c73-f1015adcd4d4","observedHarness":"mini-SWE-agent","rawScore":57.07,"zScore":1.271826,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-6.2-astra:low:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6.2-astra","displayName":"GPT-6.2 Astra","effort":"low","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.74709,"shrinkageFactor":0.95521,"score":61.21,"se":5.69,"samplingSe":3.1626,"residualSe":4.7267,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.661705,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.0535,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:cc93ecfd-845e-455d-9642-6799d7133b7e","observedHarness":"Stirrup","rawScore":32.66,"zScore":0.848814,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:cc93ecfd-845e-455d-9642-6799d7133b7e","observedHarness":"mini-SWE-agent","rawScore":51.01,"zScore":1.002559,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"anthropic:claude-opus-5.5:medium:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.745207,"shrinkageFactor":0.941566,"score":61.18,"se":4.98,"samplingSe":3.6123,"residualSe":3.4281,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":2.451089,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.5179,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:claude-opus-5-5_medium:medium:","observedHarness":"Cursor","rawScore":50,"zScore":0.721419,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.85,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:ea9e0da6-169f-43c6-92f5-730a10d6ff8d","observedHarness":"Stirrup","rawScore":33.08,"zScore":0.884866,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:ea9e0da6-169f-43c6-92f5-730a10d6ff8d","observedHarness":"Direct API","rawScore":59.26,"zScore":0.95278,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:ea9e0da6-169f-43c6-92f5-730a10d6ff8d","observedHarness":"mini-SWE-agent","rawScore":52.53,"zScore":1.070098,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"anthropic:claude-fable-5.1:high:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-fable-5.1","displayName":"Claude Fable 5.1","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.74015,"shrinkageFactor":0.802437,"score":61.1,"se":7.87,"samplingSe":6.6421,"residualSe":4.221,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":15.786777,"costLogSd":1.0567,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-09-12","paretoProb":0.1211,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recoromAv2aFG1mQC","observedHarness":"ALE-Agent","rawScore":2143.2,"zScore":1.187286,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.294856,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:mirrorcode-ml-2l-claude-fable-5-1","observedHarness":"Epoch MirrorCode Scaffold","rawScore":73.3,"zScore":1.331299,"seModel":"upstream_se","seRaw":9.17,"ciHalfWidth":17.9732,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-10","publishedDate":"2026-09-10","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"}]},{"id":"google:gemini-4-argon:medium:standardized-suite:standardized","provider":"google","modelSlug":"gemini-4-argon","displayName":"Gemini 4 Argon","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.698647,"shrinkageFactor":0.942783,"score":60.48,"se":4.96,"samplingSe":3.5745,"residualSe":3.4325,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.629243,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.0008,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:gemini-4-argon_medium:medium:","observedHarness":"Cursor","rawScore":55.8,"zScore":1.24803,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.64,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:48e6f735-f2bc-418f-b99f-3dc5a0ddda77","observedHarness":"Stirrup","rawScore":25.02,"zScore":0.19301,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:48e6f735-f2bc-418f-b99f-3dc5a0ddda77","observedHarness":"Direct API","rawScore":58.91,"zScore":0.907454,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:48e6f735-f2bc-418f-b99f-3dc5a0ddda77","observedHarness":"mini-SWE-agent","rawScore":55.05,"zScore":1.18207,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-5.6-sol:max:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.6-sol","displayName":"GPT-5.6 Sol","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.670769,"shrinkageFactor":0.93822,"score":60.06,"se":4.99,"samplingSe":3.7143,"residualSe":3.325,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":12.117077,"costLogSd":1.0534,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0038,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recB9LSLRE0ygr7gb","observedHarness":"ALE-Agent","rawScore":2176.88,"zScore":1.253868,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.534078,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:gpt-5.6-sol_max:max:","observedHarness":"Cursor","rawScore":40.9,"zScore":-0.104817,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.6,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:gpt-5.6-sol_max:max:","observedHarness":"Proximus","rawScore":32.2,"zScore":-0.028655,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":179.64,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:openai/gpt-5.6-sol","observedHarness":"Vals SRE Runner","rawScore":30.53,"zScore":1.474645,"seModel":"upstream_se","seRaw":2.851,"ciHalfWidth":5.588,"ciLower":null,"ciUpper":null,"measuredCostUsd":42.521152,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"anthropic:claude-opus-4.8:max:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-opus-4.8","displayName":"Claude Opus 4.8","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.586233,"shrinkageFactor":0.727542,"score":58.79,"se":9.47,"samplingSe":7.8002,"residualSe":5.3743,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":14.827231,"costLogSd":1.0558,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-06-12","paretoProb":0.1164,"benchmarks":[{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:2Nf4PQvFjvbixGhtQXGQoS","observedHarness":"Epoch MirrorCode Scaffold","rawScore":65.56,"zScore":0.896229,"seModel":"upstream_se","seRaw":10.846045,"ciHalfWidth":21.2582,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-21","publishedDate":"2026-10-21","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"}]},{"id":"openai:gpt-6.1-terra:xhigh:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.582414,"shrinkageFactor":0.934723,"score":58.74,"se":5.45,"samplingSe":3.818,"residualSe":3.8891,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.400689,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.3588,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:e8d443ba-8aca-4830-a186-ae869bb9c6fe","observedHarness":"Stirrup","rawScore":36.86,"zScore":1.209335,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:e8d443ba-8aca-4830-a186-ae869bb9c6fe","observedHarness":"Direct API","rawScore":53.36,"zScore":0.188724,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:e8d443ba-8aca-4830-a186-ae869bb9c6fe","observedHarness":"mini-SWE-agent","rawScore":46.46,"zScore":0.800386,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"anthropic:claude-fable-5:high:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-fable-5","displayName":"Claude Fable 5","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.532204,"shrinkageFactor":0.946608,"score":57.98,"se":5.1,"samplingSe":3.453,"residualSe":3.7546,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":15.786777,"costLogSd":1.0567,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-06-09","paretoProb":0.0645,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recOJYPyImXeBu5y6","observedHarness":"ALE-Agent","rawScore":2041.3,"zScore":0.985841,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.108822,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:mirrorcode-ml-2l-claude-fable-5","observedHarness":"Epoch MirrorCode Scaffold","rawScore":63.89,"zScore":0.802358,"seModel":"upstream_se","seRaw":10.404093,"ciHalfWidth":20.392,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-10","publishedDate":"2026-08-10","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Claude Fable 5 (High)","observedHarness":"LMArena Agent Harness","rawScore":1187.54,"zScore":0.449095,"seModel":"upstream_ci","ciHalfWidth":22.9404,"ciLower":1164.6007,"ciUpper":1210.4815,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"}]},{"id":"moonshot:kimi-k3.1-pro:max:standardized-suite:standardized","provider":"moonshot","modelSlug":"kimi-k3.1-pro","displayName":"Kimi K3.1 Pro","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.493874,"shrinkageFactor":0.982187,"score":57.41,"se":2.4,"samplingSe":1.9945,"residualSe":1.3318,"coverageCount":9,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.6812,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-10-10","paretoProb":0.9846,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recQfC5gdLA6UCLLC","observedHarness":"ALE-Agent","rawScore":1932.75,"zScore":0.771248,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.664453,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:S2mVtBattZSe5WZPkXNicA","observedHarness":"Epoch MirrorCode Scaffold","rawScore":43.33,"zScore":-0.353332,"seModel":"upstream_se","seRaw":10.38925,"ciHalfWidth":20.3629,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-17","publishedDate":"2026-11-17","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:665c67ca-09eb-48f0-a7a3-0096e1d5de51","observedHarness":"Stirrup","rawScore":25.45,"zScore":0.22992,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:kimi-k3.1-pro","observedHarness":"Direct API","rawScore":38.7,"zScore":-0.341726,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.093917,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Kimi K3.1 Pro (Max)","observedHarness":"LMArena Agent Harness","rawScore":1169.14,"zScore":0.257119,"seModel":"upstream_ci","ciHalfWidth":19.8782,"ciLower":1149.261,"ciUpper":1189.0175,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:665c67ca-09eb-48f0-a7a3-0096e1d5de51","observedHarness":"Direct API","rawScore":60.76,"zScore":1.147031,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:Kimi K3.1 Pro (Max reasoning)","observedHarness":"Direct API","rawScore":16.7,"zScore":-0.378123,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:665c67ca-09eb-48f0-a7a3-0096e1d5de51","observedHarness":"mini-SWE-agent","rawScore":67.17,"zScore":1.720605,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:kimi/kimi-k3.1-pro","observedHarness":"Vals SRE Runner","rawScore":27.1,"zScore":1.22333,"seModel":"upstream_se","seRaw":2.751,"ciHalfWidth":5.392,"ciLower":null,"ciUpper":null,"measuredCostUsd":27.390864,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"anthropic:claude-sonnet-5.5:high:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.465014,"shrinkageFactor":0.967074,"score":56.98,"se":3.61,"samplingSe":2.7116,"residualSe":2.3892,"coverageCount":6,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.636129,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.4991,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recYjibuLhTTZDmIz","observedHarness":"ALE-Agent","rawScore":1819.12,"zScore":0.546614,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.110921,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:claude-sonnet-5-5_high:high:","observedHarness":"Cursor","rawScore":43.8,"zScore":0.158489,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.41,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:268525e2-f873-4178-bc07-e6a0ee1f002d","observedHarness":"Stirrup","rawScore":25.74,"zScore":0.254813,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Claude Sonnet 5.5 (High)","observedHarness":"LMArena Agent Harness","rawScore":1238.24,"zScore":0.978072,"seModel":"upstream_ci","ciHalfWidth":33.5909,"ciLower":1204.6509,"ciUpper":1271.8327,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:268525e2-f873-4178-bc07-e6a0ee1f002d","observedHarness":"Direct API","rawScore":53.7,"zScore":0.232755,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:268525e2-f873-4178-bc07-e6a0ee1f002d","observedHarness":"mini-SWE-agent","rawScore":43.94,"zScore":0.688413,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"anthropic:claude-opus-5:high:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-opus-5","displayName":"Claude Opus 5","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.463552,"shrinkageFactor":0.664991,"score":56.95,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.432844,"costLogSd":1.0492,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-07-28","paretoProb":0.1809,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:reczXRrSMMBAUjHxZ","observedHarness":"ALE-Agent","rawScore":2164.57,"zScore":1.229533,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.890512,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"openai:gpt-6.1-terra:high:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.462955,"shrinkageFactor":0.934849,"score":56.94,"se":5.45,"samplingSe":3.8143,"residualSe":3.8897,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.25928,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.3879,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:ea3e1f50-f62d-41d2-849a-9af0413e4d37","observedHarness":"Stirrup","rawScore":37.46,"zScore":1.260838,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:ea3e1f50-f62d-41d2-849a-9af0413e4d37","observedHarness":"Direct API","rawScore":51.62,"zScore":-0.036607,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:ea3e1f50-f62d-41d2-849a-9af0413e4d37","observedHarness":"mini-SWE-agent","rawScore":41.92,"zScore":0.598658,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"meta:muse-wonder:max:standardized-suite:standardized","provider":"meta","modelSlug":"muse-wonder","displayName":"Muse Wonder","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.420206,"shrinkageFactor":0.984857,"score":56.3,"se":2.28,"samplingSe":1.8389,"residualSe":1.3516,"coverageCount":8,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.952036,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.4666,"benchmarks":[{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:4SF6nvuJows76zkypDWF3b","observedHarness":"Epoch MirrorCode Scaffold","rawScore":45.56,"zScore":-0.227983,"seModel":"upstream_se","seRaw":10.599324,"ciHalfWidth":20.7747,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-14","publishedDate":"2026-10-14","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:ff2f3f3d-783f-4e27-8a39-47237501a313","observedHarness":"Stirrup","rawScore":27.88,"zScore":0.438507,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:muse-wonder-1.0","observedHarness":"Direct API","rawScore":46.2,"zScore":0.079635,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":16.436667,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Muse Wonder (Max)","observedHarness":"LMArena Agent Harness","rawScore":1125.31,"zScore":-0.200181,"seModel":"upstream_ci","ciHalfWidth":22.6497,"ciLower":1102.6619,"ciUpper":1147.9613,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:ff2f3f3d-783f-4e27-8a39-47237501a313","observedHarness":"Direct API","rawScore":60.3,"zScore":1.087461,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:Muse Wonder 1.0 (Max reasoning)","observedHarness":"Direct API","rawScore":16.1,"zScore":-0.425101,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:ff2f3f3d-783f-4e27-8a39-47237501a313","observedHarness":"mini-SWE-agent","rawScore":52.02,"zScore":1.047436,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"vals_code_migration","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:code-migration-v1-1:meta/muse-wonder","observedHarness":"Pi","rawScore":72.36,"zScore":1.055113,"seModel":"upstream_se","seRaw":3.187,"ciHalfWidth":6.2465,"ciLower":null,"ciUpper":null,"measuredCostUsd":47.931802,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/code-migration-v1-1"}]},{"id":"alibaba:qwen4-max:xhigh:standardized-suite:standardized","provider":"alibaba","modelSlug":"qwen4-max","displayName":"Qwen4 Max","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.386542,"shrinkageFactor":0.979757,"score":55.8,"se":3.19,"samplingSe":2.1261,"residualSe":2.384,"coverageCount":5,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.268133,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2.25,"inputPer1mUsd":1.5,"outputPer1mUsd":4.5,"contextWindow":1000000,"releaseDate":"2026-10-22","paretoProb":0.9207,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:fef6625d-9097-45f6-95c0-bcaa06e3d8e1","observedHarness":"Stirrup","rawScore":26.45,"zScore":0.315758,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:qwen4-max","observedHarness":"Direct API","rawScore":46.6,"zScore":0.102107,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.628494,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Qwen4-Max (xHigh)","observedHarness":"LMArena Agent Harness","rawScore":1192.17,"zScore":0.497402,"seModel":"upstream_ci","ciHalfWidth":24.6142,"ciLower":1167.5542,"ciUpper":1216.7825,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:Qwen 4 Max (xHigh reasoning)","observedHarness":"Direct API","rawScore":15.6,"zScore":-0.464249,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:fef6625d-9097-45f6-95c0-bcaa06e3d8e1","observedHarness":"mini-SWE-agent","rawScore":48.48,"zScore":0.890142,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-6.2-sol:low:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6.2-sol","displayName":"GPT-6.2 Sol","effort":"low","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.368078,"shrinkageFactor":0.957013,"score":55.52,"se":5.66,"samplingSe":3.0983,"residualSe":4.7356,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.183585,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.4873,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:a77965fb-2117-4e2c-bd8b-80b5579a7e7e","observedHarness":"Stirrup","rawScore":31.37,"zScore":0.738083,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:a77965fb-2117-4e2c-bd8b-80b5579a7e7e","observedHarness":"mini-SWE-agent","rawScore":37.88,"zScore":0.419146,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-6.1-terra:medium:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.347291,"shrinkageFactor":0.977284,"score":55.21,"se":3.86,"samplingSe":2.2523,"residualSe":3.1373,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.168805,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.9038,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:c441e158-a87d-4116-911b-93325e4b4f1c","observedHarness":"Stirrup","rawScore":34.54,"zScore":1.01019,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:GPT-6.1 Terra (Medium)","observedHarness":"LMArena Agent Harness","rawScore":1191.87,"zScore":0.494272,"seModel":"upstream_ci","ciHalfWidth":10.2032,"ciLower":1181.6641,"ciUpper":1202.0704,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:c441e158-a87d-4116-911b-93325e4b4f1c","observedHarness":"Direct API","rawScore":51.62,"zScore":-0.036607,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:c441e158-a87d-4116-911b-93325e4b4f1c","observedHarness":"mini-SWE-agent","rawScore":32.32,"zScore":0.172095,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"meta:muse-wonder:high:standardized-suite:standardized","provider":"meta","modelSlug":"muse-wonder","displayName":"Muse Wonder","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.229134,"shrinkageFactor":0.95845,"score":53.44,"se":5.64,"samplingSe":3.0461,"residualSe":4.7427,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.560546,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.4202,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:38a98e3d-779b-49ec-b005-230a3a72088a","observedHarness":"Stirrup","rawScore":26.63,"zScore":0.331209,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:38a98e3d-779b-49ec-b005-230a3a72088a","observedHarness":"mini-SWE-agent","rawScore":38.38,"zScore":0.441363,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"google:gemini-4-hydrogen:medium:standardized-suite:standardized","provider":"google","modelSlug":"gemini-4-hydrogen","displayName":"Gemini 4 Hydrogen","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.190268,"shrinkageFactor":0.95316,"score":52.85,"se":5.12,"samplingSe":3.2342,"residualSe":3.9658,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.797987,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":1.5,"outputPer1mUsd":7.5,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.5157,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:gemini-4-hydrogen_medium:medium:","observedHarness":"Cursor","rawScore":46.4,"zScore":0.394556,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.28,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:f0537484-af30-4a8a-bf34-368c552e5cfd","observedHarness":"Stirrup","rawScore":31.52,"zScore":0.750958,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:f0537484-af30-4a8a-bf34-368c552e5cfd","observedHarness":"mini-SWE-agent","rawScore":29.29,"zScore":0.037462,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"deepseek:deepseek-v4.1-flash-1017:max:standardized-suite:standardized","provider":"deepseek","modelSlug":"deepseek-v4.1-flash-1017","displayName":"DeepSeek V4.1 Flash (1017)","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.188135,"shrinkageFactor":0.981043,"score":52.82,"se":2.67,"samplingSe":2.0575,"residualSe":1.6965,"coverageCount":7,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.602105,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.525,"inputPer1mUsd":0.3,"outputPer1mUsd":1.2,"contextWindow":1048576,"releaseDate":"2026-10-17","paretoProb":0.9094,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recmrTBOqivIMzmGy","observedHarness":"ALE-Agent","rawScore":1622,"zScore":0.156927,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.100044,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:f151fd0b-2139-40cf-a3ec-995e8cedc592","observedHarness":"Stirrup","rawScore":28.47,"zScore":0.489152,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:deepseek-v4.1-flash-1017","observedHarness":"Direct API","rawScore":38.8,"zScore":-0.336108,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.417983,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:DeepSeek-V4.1-Flash-20261017 (Max)","observedHarness":"LMArena Agent Harness","rawScore":1127.73,"zScore":-0.174932,"seModel":"upstream_ci","ciHalfWidth":8.5959,"ciLower":1119.1331,"ciUpper":1136.3248,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:f151fd0b-2139-40cf-a3ec-995e8cedc592","observedHarness":"Direct API","rawScore":55.9,"zScore":0.517657,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:f151fd0b-2139-40cf-a3ec-995e8cedc592","observedHarness":"mini-SWE-agent","rawScore":33.84,"zScore":0.239634,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"vals_code_migration","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:code-migration-v1-1:deepseek/deepseek-v4.1-flash-1017","observedHarness":"Pi","rawScore":67.52,"zScore":0.754418,"seModel":"upstream_se","seRaw":2.763,"ciHalfWidth":5.4155,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.764297,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/code-migration-v1-1"}]},{"id":"anthropic:claude-haiku-5.5:max:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-haiku-5.5","displayName":"Claude Haiku 5.5","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.184451,"shrinkageFactor":0.98366,"score":52.77,"se":2.6,"samplingSe":1.9102,"residualSe":1.758,"coverageCount":9,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.411007,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1000000,"releaseDate":"2026-10-07","paretoProb":0.5764,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:claude-haiku-5-5_max:max:","observedHarness":"Cursor","rawScore":48.4,"zScore":0.576146,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.14,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:claude-haiku-5-5_max:max:","observedHarness":"Proximus","rawScore":43.77,"zScore":0.650197,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":20.55,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:ifSAAMJA9S73m6pAyRWxUD","observedHarness":"Epoch MirrorCode Scaffold","rawScore":78.89,"zScore":1.645517,"seModel":"upstream_se","seRaw":8.045286,"ciHalfWidth":15.7688,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-21","publishedDate":"2026-10-21","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:7d064a11-6e78-4030-b535-cdf940615041","observedHarness":"Stirrup","rawScore":34.93,"zScore":1.043667,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:claude-haiku-5.5","observedHarness":"Direct API","rawScore":47.9,"zScore":0.175143,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":16.248561,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:7d064a11-6e78-4030-b535-cdf940615041","observedHarness":"Direct API","rawScore":54.98,"zScore":0.398516,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:7d064a11-6e78-4030-b535-cdf940615041","observedHarness":"mini-SWE-agent","rawScore":32.83,"zScore":0.194756,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"vals_code_migration","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:code-migration-v1-1:anthropic/claude-haiku-5-5","observedHarness":"Pi","rawScore":52.36,"zScore":-0.187428,"seModel":"upstream_se","seRaw":3.489,"ciHalfWidth":6.8384,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.199353,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/code-migration-v1-1"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:anthropic/claude-haiku-5-5","observedHarness":"Vals SRE Runner","rawScore":5.72,"zScore":-0.343172,"seModel":"upstream_se","seRaw":1.438,"ciHalfWidth":2.8185,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.140034,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"google:gemini-4-neon:low:standardized-suite:standardized","provider":"google","modelSlug":"gemini-4-neon","displayName":"Gemini 4 Neon","effort":"low","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.130148,"shrinkageFactor":0.953104,"score":51.95,"se":5.12,"samplingSe":3.2361,"residualSe":3.9656,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.745228,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1048576,"releaseDate":"2026-11-18","paretoProb":0.8727,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:gemini-4-neon_low:low:","observedHarness":"Cursor","rawScore":45.5,"zScore":0.312841,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.26,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:09fea5f5-7217-452d-b693-fcca01e07bdf","observedHarness":"Stirrup","rawScore":26.95,"zScore":0.358678,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:09fea5f5-7217-452d-b693-fcca01e07bdf","observedHarness":"mini-SWE-agent","rawScore":32.83,"zScore":0.194756,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"spacexai:grok-4.7:high:standardized-suite:standardized","provider":"spacexai","modelSlug":"grok-4.7","displayName":"Grok 4.7","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.081694,"shrinkageFactor":0.944282,"score":51.23,"se":4.93,"samplingSe":3.5274,"residualSe":3.438,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":6.183596,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-10-18","paretoProb":0.2733,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:grok-4.7_high:high:","observedHarness":"Cursor","rawScore":46,"zScore":0.358238,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.88,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:272f5f03-aea9-4675-a244-2b753b618cdf","observedHarness":"Stirrup","rawScore":27.78,"zScore":0.429923,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:272f5f03-aea9-4675-a244-2b753b618cdf","observedHarness":"Direct API","rawScore":57.75,"zScore":0.757233,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:272f5f03-aea9-4675-a244-2b753b618cdf","observedHarness":"mini-SWE-agent","rawScore":24.75,"zScore":-0.164267,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"anthropic:claude-haiku-5.5:xhigh:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-haiku-5.5","displayName":"Claude Haiku 5.5","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.064008,"shrinkageFactor":0.942492,"score":50.96,"se":4.96,"samplingSe":3.5836,"residualSe":3.4315,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.248361,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1000000,"releaseDate":"2026-10-07","paretoProb":0.7508,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:claude-haiku-5-5_xhigh:xhigh:","observedHarness":"Cursor","rawScore":45.2,"zScore":0.285602,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.56,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:41b5bf5e-daf6-42d9-b8d1-60e340a3028d","observedHarness":"Stirrup","rawScore":29.93,"zScore":0.614476,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:41b5bf5e-daf6-42d9-b8d1-60e340a3028d","observedHarness":"Direct API","rawScore":51.74,"zScore":-0.021067,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:41b5bf5e-daf6-42d9-b8d1-60e340a3028d","observedHarness":"mini-SWE-agent","rawScore":29.29,"zScore":0.037462,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"google:gemini-4-argon:low:standardized-suite:standardized","provider":"google","modelSlug":"gemini-4-argon","displayName":"Gemini 4 Argon","effort":"low","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.045228,"shrinkageFactor":0.782743,"score":50.68,"se":8.09,"samplingSe":6.9653,"residualSe":4.1174,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.360951,"costLogSd":0.5619,"costMaxUsd":8.629243,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.0276,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:rechU0EZO7d9E7amZ","observedHarness":"ALE-Agent","rawScore":1705.4,"zScore":0.321801,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.170538,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:gemini-4-argon_low:low:","observedHarness":"Cursor","rawScore":47.9,"zScore":0.530749,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.96,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"}]},{"id":"meta:muse-ember:max:standardized-suite:standardized","provider":"meta","modelSlug":"muse-ember","displayName":"Muse Ember","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.025943,"shrinkageFactor":0.985699,"score":50.39,"se":3.99,"samplingSe":1.787,"residualSe":3.5657,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.135936,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.5031,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:dbf63365-127f-41ee-9e51-81173adb3502","observedHarness":"Stirrup","rawScore":27.17,"zScore":0.377562,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Muse Ember (Max)","observedHarness":"LMArena Agent Harness","rawScore":1110.63,"zScore":-0.353345,"seModel":"upstream_ci","ciHalfWidth":19.4463,"ciLower":1091.1844,"ciUpper":1130.0771,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:dbf63365-127f-41ee-9e51-81173adb3502","observedHarness":"mini-SWE-agent","rawScore":40.91,"zScore":0.55378,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-5.6-sol:xhigh:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.6-sol","displayName":"GPT-5.6 Sol","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.011969,"shrinkageFactor":0.965262,"score":50.18,"se":5.76,"samplingSe":2.7852,"residualSe":5.0379,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.461528,"costLogSd":1.056,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0015,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:gpt-5.6-sol_xhigh:xhigh:","observedHarness":"Cursor","rawScore":37.4,"zScore":-0.422599,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.32,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:GPT-5.6 Sol (xHigh)","observedHarness":"LMArena Agent Harness","rawScore":1154.81,"zScore":0.107607,"seModel":"upstream_ci","ciHalfWidth":19.9434,"ciLower":1134.8687,"ciUpper":1174.7555,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"}]},{"id":"meta:muse-ember:xhigh:standardized-suite:standardized","provider":"meta","modelSlug":"muse-ember","displayName":"Muse Ember","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.001206,"shrinkageFactor":0.961122,"score":50.02,"se":5.59,"samplingSe":2.9465,"residualSe":4.756,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.116228,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.962,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:7a64ffc4-36c3-4369-981e-73abdd6ff5e2","observedHarness":"Stirrup","rawScore":21.97,"zScore":-0.068797,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:7a64ffc4-36c3-4369-981e-73abdd6ff5e2","observedHarness":"mini-SWE-agent","rawScore":35.35,"zScore":0.306729,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-6.1-terra:none:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"none","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.037748,"shrinkageFactor":0.938268,"score":49.43,"se":5.39,"samplingSe":3.7129,"residualSe":3.9039,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.225896,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.0719,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:0d3c777f-6817-42d7-9bb9-223314bc5a57","observedHarness":"Stirrup","rawScore":33.35,"zScore":0.908042,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:0d3c777f-6817-42d7-9bb9-223314bc5a57","observedHarness":"Direct API","rawScore":47.8,"zScore":-0.531301,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:0d3c777f-6817-42d7-9bb9-223314bc5a57","observedHarness":"mini-SWE-agent","rawScore":26.77,"zScore":-0.074511,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"alibaba:qwen3.8-max-0902:xhigh:standardized-suite:standardized","provider":"alibaba","modelSlug":"qwen3.8-max-0902","displayName":"Qwen3.8 Max (0902)","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.089095,"shrinkageFactor":0.954878,"score":48.66,"se":4.59,"samplingSe":3.1743,"residualSe":3.3135,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.977532,"costLogSd":1.0508,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":1000000,"releaseDate":"2026-09-02","paretoProb":0.7533,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recI2rYXFPKPKdBSs","observedHarness":"ALE-Agent","rawScore":1679.12,"zScore":0.269848,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.52514,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:qwen3.8-max-0902_xhigh:xhigh:","observedHarness":"Proximus","rawScore":17.81,"zScore":-0.872965,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":51.27,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:WPxarSzywRZJNbqmd4gMDK","observedHarness":"Epoch MirrorCode Scaffold","rawScore":47.78,"zScore":-0.103195,"seModel":"upstream_se","seRaw":10.269731,"ciHalfWidth":20.1287,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-21","publishedDate":"2026-10-21","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"vals_code_migration","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:code-migration-v1-1:alibaba/qwen3.8-max-internal","observedHarness":"Pi","rawScore":57.19,"zScore":0.112646,"seModel":"upstream_se","seRaw":2.801,"ciHalfWidth":5.49,"ciLower":null,"ciUpper":null,"measuredCostUsd":27.080702,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/code-migration-v1-1"}]},{"id":"anthropic:claude-opus-5.5:low:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"low","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.089004,"shrinkageFactor":0.944966,"score":48.66,"se":4.91,"samplingSe":3.5057,"residualSe":3.4405,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.975529,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.132,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:claude-opus-5-5_low:low:","observedHarness":"Cursor","rawScore":42.7,"zScore":0.058615,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.43,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:4416ca93-fde6-4b7b-abbc-3a6be109cc3d","observedHarness":"Stirrup","rawScore":15.84,"zScore":-0.594985,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:4416ca93-fde6-4b7b-abbc-3a6be109cc3d","observedHarness":"Direct API","rawScore":58.56,"zScore":0.862129,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:4416ca93-fde6-4b7b-abbc-3a6be109cc3d","observedHarness":"mini-SWE-agent","rawScore":31.31,"zScore":0.127217,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"spacexai:grok-4.7:medium:standardized-suite:standardized","provider":"spacexai","modelSlug":"grok-4.7","displayName":"Grok 4.7","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.089029,"shrinkageFactor":0.644822,"score":48.66,"se":10.1,"samplingSe":8.9059,"residualSe":4.7632,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":4.238698,"costLogSd":0.5297,"costMinUsd":2.922201,"costMaxUsd":6.183596,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-10-18","paretoProb":0.3349,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:grok-4.7_medium:medium:","observedHarness":"Cursor","rawScore":42.5,"zScore":0.040456,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.52,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"}]},{"id":"spacexai:grok-4.7:xhigh:standardized-suite:standardized","provider":"spacexai","modelSlug":"grok-4.7","displayName":"Grok 4.7","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.097739,"shrinkageFactor":0.988196,"score":48.53,"se":1.62,"samplingSe":1.6236,"residualSe":0,"coverageCount":12,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":7.771539,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-10-18","paretoProb":0.0514,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:rec9SCyOICbUV06tg","observedHarness":"ALE-Agent","rawScore":1545.85,"zScore":0.006387,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.673255,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:grok-4.7_xhigh:xhigh:","observedHarness":"Cursor","rawScore":48.6,"zScore":0.594305,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":10.19,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:grok-4.7_xhigh:xhigh:","observedHarness":"Proximus","rawScore":29.5,"zScore":-0.187073,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":318.77,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:2CP56MmAkTeFXXT8pSGre4","observedHarness":"Epoch MirrorCode Scaffold","rawScore":54.44,"zScore":0.271168,"seModel":"upstream_se","seRaw":11.439589,"ciHalfWidth":22.4216,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-29","publishedDate":"2026-10-29","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:59627802-1546-422c-9a65-65b61c022d36","observedHarness":"Stirrup","rawScore":28.66,"zScore":0.505461,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:grok-4.7","observedHarness":"Direct API","rawScore":45.3,"zScore":0.029072,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":34.207056,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Grok 4.7 (xHigh)","observedHarness":"LMArena Agent Harness","rawScore":1136.42,"zScore":-0.084265,"seModel":"upstream_ci","ciHalfWidth":27.4088,"ciLower":1109.0088,"ciUpper":1163.8263,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:59627802-1546-422c-9a65-65b61c022d36","observedHarness":"Direct API","rawScore":57.41,"zScore":0.713203,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:Grok 4.7 (xHigh reasoning)","observedHarness":"Direct API","rawScore":12.2,"zScore":-0.730456,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:59627802-1546-422c-9a65-65b61c022d36","observedHarness":"mini-SWE-agent","rawScore":25.76,"zScore":-0.119389,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"vals_code_migration","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:code-migration-v1-1:grok/grok-4.7","observedHarness":"Pi","rawScore":53.22,"zScore":-0.133999,"seModel":"upstream_se","seRaw":2.563,"ciHalfWidth":5.0235,"ciLower":null,"ciUpper":null,"measuredCostUsd":38.051442,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/code-migration-v1-1"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:grok/grok-4.7","observedHarness":"Vals SRE Runner","rawScore":3.82,"zScore":-0.482384,"seModel":"upstream_se","seRaw":1.186,"ciHalfWidth":2.3246,"ciLower":null,"ciUpper":null,"measuredCostUsd":64.767258,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"zhipu:glm-5.3:max:standardized-suite:standardized","provider":"zhipu","modelSlug":"glm-5.3","displayName":"GLM-5.3","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.125786,"shrinkageFactor":0.991543,"score":48.11,"se":1.71,"samplingSe":1.3743,"residualSe":1.0167,"coverageCount":8,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":5.009265,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":1048576,"releaseDate":"2026-09-18","paretoProb":0.5712,"benchmarks":[{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:glm-5.3_max:max:","observedHarness":"Proximus","rawScore":30.18,"zScore":-0.147175,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":97.22,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:cd684ea4-b475-4269-b001-d469d06d8a7a","observedHarness":"Stirrup","rawScore":21.59,"zScore":-0.101415,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:GLM-5.3 (Max)","observedHarness":"LMArena Agent Harness","rawScore":1100.76,"zScore":-0.456323,"seModel":"upstream_ci","ciHalfWidth":15.4698,"ciLower":1085.2877,"ciUpper":1116.2274,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:cd684ea4-b475-4269-b001-d469d06d8a7a","observedHarness":"Direct API","rawScore":59.03,"zScore":0.922995,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:GLM 5.3 (Max reasoning)","observedHarness":"Direct API","rawScore":9.4,"zScore":-0.949686,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:cd684ea4-b475-4269-b001-d469d06d8a7a","observedHarness":"mini-SWE-agent","rawScore":41.92,"zScore":0.598658,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"vals_code_migration","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:code-migration-v1-1:zai/glm-5.3","observedHarness":"Pi","rawScore":55.07,"zScore":-0.019064,"seModel":"upstream_se","seRaw":2.115,"ciHalfWidth":4.1454,"ciLower":null,"ciUpper":null,"measuredCostUsd":34.907391,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/code-migration-v1-1"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:zai/glm-5.3","observedHarness":"Vals SRE Runner","rawScore":9.16,"zScore":-0.091125,"seModel":"upstream_se","seRaw":1.786,"ciHalfWidth":3.5006,"ciLower":null,"ciUpper":null,"measuredCostUsd":27.465302,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"spacexai:grok-4.6:high:standardized-suite:standardized","provider":"spacexai","modelSlug":"grok-4.6","displayName":"Grok 4.6","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.159969,"shrinkageFactor":0.641624,"score":47.6,"se":10.12,"samplingSe":8.9459,"residualSe":4.7396,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":2.430646,"costLogSd":1.0437,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-08-25","paretoProb":0.4806,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:grok-4.6_high:high:","observedHarness":"Cursor","rawScore":41.3,"zScore":-0.068499,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.28,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"}]},{"id":"openai:gpt-6-astra:high:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6-astra","displayName":"GPT-6 Astra","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.190733,"shrinkageFactor":0.690711,"score":47.14,"se":9.75,"samplingSe":8.3107,"residualSe":5.1022,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":15.786777,"costLogSd":1.0567,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0017,"benchmarks":[{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:mirrorcode-ml-2l-gpt-6-astra","observedHarness":"Epoch MirrorCode Scaffold","rawScore":46.7,"zScore":-0.163902,"seModel":"upstream_se","seRaw":11.86,"ciHalfWidth":23.2456,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-30","publishedDate":"2026-08-30","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"}]},{"id":"meta:muse-spark-1.3:max:standardized-suite:standardized","provider":"meta","modelSlug":"muse-spark-1.3","displayName":"Muse Spark 1.3","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.202454,"shrinkageFactor":0.983701,"score":46.96,"se":2.79,"samplingSe":1.9078,"residualSe":2.0321,"coverageCount":7,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.092883,"costLogSd":1.0457,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.0689,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:muse-spark-1.3_max:max:","observedHarness":"Cursor","rawScore":41.9,"zScore":-0.014021,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.89,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:muse-spark-1.3_max:max:","observedHarness":"Proximus","rawScore":25.8,"zScore":-0.404165,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":243.34,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:HQrqaQZvkv2xZWm3KQ98ta","observedHarness":"Epoch MirrorCode Scaffold","rawScore":54.44,"zScore":0.271168,"seModel":"upstream_se","seRaw":10.846045,"ciHalfWidth":21.2582,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-20","publishedDate":"2026-10-20","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:muse-spark-1.3","observedHarness":"Direct API","rawScore":47.9,"zScore":0.175143,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":11.260556,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Muse Spark 1.3 (Max)","observedHarness":"LMArena Agent Harness","rawScore":1120.27,"zScore":-0.252766,"seModel":"upstream_ci","ciHalfWidth":16.0657,"ciLower":1104.2011,"ciUpper":1136.3324,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:Muse Spark 1.3 (Max reasoning)","observedHarness":"Direct API","rawScore":12.8,"zScore":-0.683479,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:meta/muse_spark_1_3_max","observedHarness":"Vals SRE Runner","rawScore":3.82,"zScore":-0.482384,"seModel":"upstream_se","seRaw":1.186,"ciHalfWidth":2.3246,"ciLower":null,"ciUpper":null,"measuredCostUsd":25.861222,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"openai:gpt-5.3-codex:xhigh:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.3-codex","displayName":"GPT-5.3 Codex","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.206049,"shrinkageFactor":0.664991,"score":46.91,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.006963,"costLogSd":1.0533,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4.8125,"inputPer1mUsd":1.75,"outputPer1mUsd":14,"contextWindow":400000,"releaseDate":"2026-02-15","paretoProb":0.0059,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recjPl07t1MvmNgDm","observedHarness":"ALE-Agent","rawScore":1655.22,"zScore":0.2226,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.559265,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"openai:gpt-5.6-terra:max:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.6-terra","displayName":"GPT-5.6 Terra","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.225318,"shrinkageFactor":0.94139,"score":46.62,"se":4.87,"samplingSe":3.6178,"residualSe":3.2579,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":7.571751,"costLogSd":1.049,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4.5,"inputPer1mUsd":2,"outputPer1mUsd":12,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0009,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recy6tGLt0oXl5Z3t","observedHarness":"ALE-Agent","rawScore":1951.42,"zScore":0.808157,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.254047,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:Vubr2UGkJt5WdCCxePP6bL","observedHarness":"Epoch MirrorCode Scaffold","rawScore":41.11,"zScore":-0.47812,"seModel":"upstream_se","seRaw":12.37281,"ciHalfWidth":24.2507,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-20","publishedDate":"2026-10-20","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:gpt-5.6-terra","observedHarness":"Direct API","rawScore":25.9,"zScore":-1.060848,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":12.262994,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:openai/gpt-5.6-terra","observedHarness":"Vals SRE Runner","rawScore":9.54,"zScore":-0.063283,"seModel":"upstream_se","seRaw":1.819,"ciHalfWidth":3.5652,"ciLower":null,"ciUpper":null,"measuredCostUsd":30.855656,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"anthropic:claude-haiku-5.5:high:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-haiku-5.5","displayName":"Claude Haiku 5.5","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.232529,"shrinkageFactor":0.969828,"score":46.51,"se":3.53,"samplingSe":2.5957,"residualSe":2.396,"coverageCount":6,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.15856,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1000000,"releaseDate":"2026-10-07","paretoProb":0.8586,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recxv7Egql7NoCF6X","observedHarness":"ALE-Agent","rawScore":1587,"zScore":0.087736,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.02089,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:claude-haiku-5-5_high:high:","observedHarness":"Cursor","rawScore":37.5,"zScore":-0.41352,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.31,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:20829974-af06-497e-9f42-77fb958d43b9","observedHarness":"Stirrup","rawScore":23.2,"zScore":0.036784,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Claude Haiku 5.5 (High)","observedHarness":"LMArena Agent Harness","rawScore":1145.67,"zScore":0.012245,"seModel":"upstream_ci","ciHalfWidth":18.0808,"ciLower":1127.5849,"ciUpper":1163.7466,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:20829974-af06-497e-9f42-77fb958d43b9","observedHarness":"Direct API","rawScore":48.73,"zScore":-0.410865,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:20829974-af06-497e-9f42-77fb958d43b9","observedHarness":"mini-SWE-agent","rawScore":21.72,"zScore":-0.2989,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-5.5:medium:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.5","displayName":"GPT-5.5","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.292603,"shrinkageFactor":0.664991,"score":45.61,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":5.234615,"costLogSd":1.0599,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":11.25,"inputPer1mUsd":5,"outputPer1mUsd":30,"contextWindow":1050000,"releaseDate":"2026-04-23","paretoProb":0.006,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:rec8dhBUXYFTQ1qEi","observedHarness":"ALE-Agent","rawScore":1589.38,"zScore":0.092441,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.328965,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"anthropic:claude-sonnet-5:high:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-sonnet-5","displayName":"Claude Sonnet 5","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.296297,"shrinkageFactor":0.936759,"score":45.56,"se":5.74,"samplingSe":3.758,"residualSe":4.3445,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.700253,"costLogSd":1.0442,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-07-15","paretoProb":0.0422,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:rec4QpFWnpOanOnNm","observedHarness":"ALE-Agent","rawScore":1463.12,"zScore":-0.157162,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.96046,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Claude Sonnet 5 (High)","observedHarness":"LMArena Agent Harness","rawScore":1124.05,"zScore":-0.213327,"seModel":"upstream_ci","ciHalfWidth":23.1065,"ciLower":1100.9424,"ciUpper":1147.1555,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"}]},{"id":"spacexai:grok-4.6:medium:standardized-suite:standardized","provider":"spacexai","modelSlug":"grok-4.6","displayName":"Grok 4.6","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.299755,"shrinkageFactor":0.634403,"score":45.5,"se":10.18,"samplingSe":9.0356,"residualSe":4.6863,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.293502,"costLogSd":1.054,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-08-25","paretoProb":0.5368,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:grok-4.6_medium:medium:","observedHarness":"Cursor","rawScore":38.9,"zScore":-0.286407,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.59,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"}]},{"id":"meta:muse-spark-1.3:xhigh:standardized-suite:standardized","provider":"meta","modelSlug":"muse-spark-1.3","displayName":"Muse Spark 1.3","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.301604,"shrinkageFactor":0.776074,"score":45.48,"se":8.17,"samplingSe":7.0714,"residualSe":4.0823,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.165655,"costLogSd":1.051,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.181,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recoUMQ4ZOtgdCB0E","observedHarness":"ALE-Agent","rawScore":1688.35,"zScore":0.288095,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.220853,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:muse-spark-1.3_xhigh:xhigh:","observedHarness":"Cursor","rawScore":34.6,"zScore":-0.676825,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.33,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"}]},{"id":"anthropic:claude-opus-4.7:max:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-opus-4.7","displayName":"Claude Opus 4.7","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.324492,"shrinkageFactor":0.773344,"score":45.13,"se":9.12,"samplingSe":7.1144,"residualSe":5.7126,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":14.827231,"costLogSd":1.0558,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-04-16","paretoProb":0.0172,"benchmarks":[{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:exn3JadCGHYhNqvo6oRXRD","observedHarness":"Epoch MirrorCode Scaffold","rawScore":43.33,"zScore":-0.353332,"seModel":"upstream_se","seRaw":9.594972,"ciHalfWidth":18.8061,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-30","publishedDate":"2026-10-30","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"}]},{"id":"anthropic:claude-opus-4.8:high:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-opus-4.8","displayName":"Claude Opus 4.8","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.326192,"shrinkageFactor":0.664991,"score":45.11,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.432844,"costLogSd":1.0492,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-06-12","paretoProb":0.0351,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:rec9PMFOTHSfxdEBy","observedHarness":"ALE-Agent","rawScore":1563.83,"zScore":0.041931,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.963945,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"deepseek:deepseek-v4.1-flash-1017:high:standardized-suite:standardized","provider":"deepseek","modelSlug":"deepseek-v4.1-flash-1017","displayName":"DeepSeek V4.1 Flash (1017)","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.335231,"shrinkageFactor":0.965964,"score":44.97,"se":5.52,"samplingSe":2.7569,"residualSe":4.7799,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.456194,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.525,"inputPer1mUsd":0.3,"outputPer1mUsd":1.2,"contextWindow":1048576,"releaseDate":"2026-10-17","paretoProb":0.9926,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:c3b66080-fdcb-48e4-8ade-6baf77921c7c","observedHarness":"Stirrup","rawScore":16.84,"zScore":-0.509147,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:c3b66080-fdcb-48e4-8ade-6baf77921c7c","observedHarness":"mini-SWE-agent","rawScore":28.79,"zScore":0.015245,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-6.1-terra:low:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"low","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.376951,"shrinkageFactor":0.944116,"score":44.35,"se":5.28,"samplingSe":3.5327,"residualSe":3.9282,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.071338,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.8324,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:c955b262-a17f-4f1a-b097-6860b866295b","observedHarness":"Stirrup","rawScore":24.49,"zScore":0.147515,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:c955b262-a17f-4f1a-b097-6860b866295b","observedHarness":"Direct API","rawScore":52.43,"zScore":0.068288,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:c955b262-a17f-4f1a-b097-6860b866295b","observedHarness":"mini-SWE-agent","rawScore":15.15,"zScore":-0.590829,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-5.4:medium:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.4","displayName":"GPT-5.4","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.382865,"shrinkageFactor":0.664991,"score":44.26,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.804746,"costLogSd":1.0553,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":5.625,"inputPer1mUsd":2.5,"outputPer1mUsd":15,"contextWindow":1050000,"releaseDate":"2026-03-05","paretoProb":0.0139,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:rec6y7fVydiA2X9U9","observedHarness":"ALE-Agent","rawScore":1520.72,"zScore":-0.043293,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.26258,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"spacexai:grok-4.6:xhigh:standardized-suite:standardized","provider":"spacexai","modelSlug":"grok-4.6","displayName":"Grok 4.6","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.44923,"shrinkageFactor":0.963415,"score":43.26,"se":3.91,"samplingSe":2.8583,"residualSe":2.6728,"coverageCount":6,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.977532,"costLogSd":1.0508,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-08-25","paretoProb":0.2258,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recnZJup9M0H9yYwP","observedHarness":"ALE-Agent","rawScore":1508.08,"zScore":-0.068281,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.24005,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:grok-4.6_xhigh:xhigh:","observedHarness":"Cursor","rawScore":44.3,"zScore":0.203887,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.26,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:grok-4.6_xhigh:xhigh:","observedHarness":"Proximus","rawScore":25.29,"zScore":-0.434088,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":243.43,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:Hoz4Uioui4GNZaLD6fiAvD","observedHarness":"Epoch MirrorCode Scaffold","rawScore":41.11,"zScore":-0.47812,"seModel":"upstream_se","seRaw":11.015461,"ciHalfWidth":21.5903,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-21","publishedDate":"2026-10-21","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:grok-4.6","observedHarness":"Direct API","rawScore":18.7,"zScore":-1.465354,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":14.573172,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Grok 4.6 (xHigh)","observedHarness":"LMArena Agent Harness","rawScore":1099.47,"zScore":-0.469782,"seModel":"upstream_ci","ciHalfWidth":22.9269,"ciLower":1076.5425,"ciUpper":1122.3963,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"}]},{"id":"anthropic:claude-sonnet-5.5:medium:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.458652,"shrinkageFactor":0.943952,"score":43.12,"se":4.93,"samplingSe":3.5378,"residualSe":3.4368,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.780656,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.0189,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:claude-sonnet-5-5_medium:medium:","observedHarness":"Cursor","rawScore":34.3,"zScore":-0.704064,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.63,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:042e8a37-fcd0-40f4-8f41-99215fb68eda","observedHarness":"Stirrup","rawScore":10.97,"zScore":-1.013018,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:042e8a37-fcd0-40f4-8f41-99215fb68eda","observedHarness":"Direct API","rawScore":52.89,"zScore":0.127859,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:042e8a37-fcd0-40f4-8f41-99215fb68eda","observedHarness":"mini-SWE-agent","rawScore":29.8,"zScore":0.060123,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"spacexai:grok-4.7:low:standardized-suite:standardized","provider":"spacexai","modelSlug":"grok-4.7","displayName":"Grok 4.7","effort":"low","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.460566,"shrinkageFactor":0.943948,"score":43.09,"se":4.93,"samplingSe":3.538,"residualSe":3.4368,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.922201,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-10-18","paretoProb":0.1209,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:grok-4.7_low:low:","observedHarness":"Cursor","rawScore":31.5,"zScore":-0.95829,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.73,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:208c16f0-8c5b-4524-ac8a-ac74c02e80a9","observedHarness":"Stirrup","rawScore":21.82,"zScore":-0.081673,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:208c16f0-8c5b-4524-ac8a-ac74c02e80a9","observedHarness":"Direct API","rawScore":54.86,"zScore":0.382976,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:208c16f0-8c5b-4524-ac8a-ac74c02e80a9","observedHarness":"mini-SWE-agent","rawScore":16.16,"zScore":-0.545951,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-5.5:xhigh:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.5","displayName":"GPT-5.5","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.480424,"shrinkageFactor":0.936253,"score":42.79,"se":5.34,"samplingSe":3.773,"residualSe":3.7765,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":12.089838,"costLogSd":1.0598,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":11.25,"inputPer1mUsd":5,"outputPer1mUsd":30,"contextWindow":1050000,"releaseDate":"2026-04-23","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recvu236lk6eJfew6","observedHarness":"ALE-Agent","rawScore":1942.97,"zScore":0.791452,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.517132,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:Tmy5qo2ScNoA2bapbEG7HP","observedHarness":"Epoch MirrorCode Scaffold","rawScore":23.33,"zScore":-1.477544,"seModel":"upstream_se","seRaw":10.131412,"ciHalfWidth":19.8576,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-29","publishedDate":"2026-10-29","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:openai/gpt-5.5","observedHarness":"Vals SRE Runner","rawScore":3.82,"zScore":-0.482384,"seModel":"upstream_se","seRaw":1.186,"ciHalfWidth":2.3246,"ciLower":null,"ciUpper":null,"measuredCostUsd":17.773692,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"openai:gpt-6-luna:max:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.513182,"shrinkageFactor":0.990046,"score":42.3,"se":2.04,"samplingSe":1.4909,"residualSe":1.3867,"coverageCount":8,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.151956,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.1678,"benchmarks":[{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:WQ4gVVhGBfWoRkRwUvNYgs","observedHarness":"Epoch MirrorCode Scaffold","rawScore":26.67,"zScore":-1.289801,"seModel":"upstream_se","seRaw":9.026709,"ciHalfWidth":17.6924,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-21","publishedDate":"2026-10-21","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","observedHarness":"Stirrup","rawScore":25.17,"zScore":0.205885,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:gpt-6-luna","observedHarness":"Direct API","rawScore":24.7,"zScore":-1.128266,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.666939,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:GPT-6 Luna (Max)","observedHarness":"LMArena Agent Harness","rawScore":1125.18,"zScore":-0.201537,"seModel":"upstream_ci","ciHalfWidth":8.8464,"ciLower":1116.336,"ciUpper":1134.0287,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","observedHarness":"Direct API","rawScore":54.63,"zScore":0.35319,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","observedHarness":"mini-SWE-agent","rawScore":12.63,"zScore":-0.702802,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"vals_code_migration","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:code-migration-v1-1:openai/gpt-6-luna","observedHarness":"Pi","rawScore":46.67,"zScore":-0.540931,"seModel":"upstream_se","seRaw":3.613,"ciHalfWidth":7.0815,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.827763,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/code-migration-v1-1"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:openai/gpt-6-luna","observedHarness":"Vals SRE Runner","rawScore":2.67,"zScore":-0.566644,"seModel":"upstream_se","seRaw":0.998,"ciHalfWidth":1.9561,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.550056,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"anthropic:claude-sonnet-5:max:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-sonnet-5","displayName":"Claude Sonnet 5","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.520311,"shrinkageFactor":0.974194,"score":42.2,"se":5.67,"samplingSe":2.4006,"residualSe":5.1367,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.487251,"costLogSd":1.048,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-07-15","paretoProb":0.0071,"benchmarks":[{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:DVxFHRdAdTpXLCiFZMJnYY","observedHarness":"Epoch MirrorCode Scaffold","rawScore":63.33,"zScore":0.77088,"seModel":"upstream_se","seRaw":10.69045,"ciHalfWidth":20.9533,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-21","publishedDate":"2026-10-21","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:anthropic/claude-sonnet-5","observedHarness":"Vals SRE Runner","rawScore":0.38,"zScore":-0.734432,"seModel":"upstream_se","seRaw":0.382,"ciHalfWidth":0.7487,"ciLower":null,"ciUpper":null,"measuredCostUsd":18.493954,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"anthropic:claude-opus-4.8:none:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-opus-4.8","displayName":"Claude Opus 4.8","effort":"none","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.526014,"shrinkageFactor":0.664991,"score":42.11,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.977031,"costLogSd":1.049,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-06-12","paretoProb":0.0697,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recw5YfULeyAfAYgR","observedHarness":"ALE-Agent","rawScore":1411.83,"zScore":-0.258557,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.421746,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"zhipu:glm-5.3:low:standardized-suite:standardized","provider":"zhipu","modelSlug":"glm-5.3","displayName":"GLM-5.3","effort":"low","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.538169,"shrinkageFactor":0.942461,"score":41.93,"se":5.31,"samplingSe":3.5846,"residualSe":3.9213,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":1.57493,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":1048576,"releaseDate":"2026-09-18","paretoProb":0.3988,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:cc1e488b-8ba4-464d-875d-690166b15ad3","observedHarness":"Stirrup","rawScore":12.74,"zScore":-0.861084,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:cc1e488b-8ba4-464d-875d-690166b15ad3","observedHarness":"Direct API","rawScore":42.01,"zScore":-1.281112,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:cc1e488b-8ba4-464d-875d-690166b15ad3","observedHarness":"mini-SWE-agent","rawScore":34.85,"zScore":0.284512,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"zhipu:glm-5.3-flash:max:standardized-suite:standardized","provider":"zhipu","modelSlug":"glm-5.3-flash","displayName":"GLM-5.3 Flash","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.551449,"shrinkageFactor":0.990848,"score":41.73,"se":1.99,"samplingSe":1.4296,"residualSe":1.3878,"coverageCount":8,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.58331,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2375,"inputPer1mUsd":0.15,"outputPer1mUsd":0.5,"contextWindow":1048576,"releaseDate":"2026-09-28","paretoProb":0.7894,"benchmarks":[{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:glm-5.3-flash_max:max:","observedHarness":"Proximus","rawScore":18.13,"zScore":-0.85419,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":11.03,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:19496b81-9f41-4214-a77a-1df803b3c5ae","observedHarness":"Stirrup","rawScore":17.61,"zScore":-0.443052,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:glm-5.3-flash","observedHarness":"Direct API","rawScore":20.7,"zScore":-1.352991,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.491289,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:GLM-5.3-Flash","observedHarness":"LMArena Agent Harness","rawScore":1061.4,"zScore":-0.866985,"seModel":"upstream_ci","ciHalfWidth":13.2026,"ciLower":1048.2021,"ciUpper":1074.6073,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:19496b81-9f41-4214-a77a-1df803b3c5ae","observedHarness":"Direct API","rawScore":51.62,"zScore":-0.036607,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:19496b81-9f41-4214-a77a-1df803b3c5ae","observedHarness":"mini-SWE-agent","rawScore":32.83,"zScore":0.194756,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"vals_code_migration","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:code-migration-v1-1:zai/glm-5.3-flash","observedHarness":"Pi","rawScore":51.65,"zScore":-0.231538,"seModel":"upstream_se","seRaw":2.99,"ciHalfWidth":5.8604,"ciLower":null,"ciUpper":null,"measuredCostUsd":8.960582,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/code-migration-v1-1"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:zai/glm-5.3-flash","observedHarness":"Vals SRE Runner","rawScore":0.76,"zScore":-0.706589,"seModel":"upstream_se","seRaw":0.539,"ciHalfWidth":1.0564,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.290191,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"openai:gpt-6-luna:xhigh:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.583158,"shrinkageFactor":0.937105,"score":41.25,"se":4.94,"samplingSe":3.7477,"residualSe":3.2224,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.104093,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.2504,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:rec922KTi4fs6PkF3","observedHarness":"ALE-Agent","rawScore":1576.92,"zScore":0.067809,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.018045,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:19813eb2-460a-475c-af65-810bb8660fec","observedHarness":"Stirrup","rawScore":18.42,"zScore":-0.373523,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:19813eb2-460a-475c-af65-810bb8660fec","observedHarness":"Direct API","rawScore":51.74,"zScore":-0.021067,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:19813eb2-460a-475c-af65-810bb8660fec","observedHarness":"mini-SWE-agent","rawScore":8.08,"zScore":-0.904974,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"google:gemini-3-flash:high:standardized-suite:standardized","provider":"google","modelSlug":"gemini-3-flash","displayName":"Gemini 3 Flash","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.584685,"shrinkageFactor":0.664991,"score":41.23,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.253198,"costLogSd":1.0461,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.125,"inputPer1mUsd":0.5,"outputPer1mUsd":3,"contextWindow":1048576,"releaseDate":"2025-12-15","paretoProb":0.4211,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recfMGg7Kvqpid4qZ","observedHarness":"ALE-Agent","rawScore":1367.2,"zScore":-0.346786,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.091067,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"deepseek:deepseek-v4-pro-0813:max:standardized-suite:standardized","provider":"deepseek","modelSlug":"deepseek-v4-pro-0813","displayName":"DeepSeek V4 Pro (0813)","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.588196,"shrinkageFactor":0.981715,"score":41.18,"se":2.78,"samplingSe":2.0207,"residualSe":1.9142,"coverageCount":7,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.769134,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.98,"inputPer1mUsd":1.32,"outputPer1mUsd":3.96,"contextWindow":1048576,"releaseDate":"2026-08-13","paretoProb":0.0002,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:receNOu1SaASh6NZL","observedHarness":"ALE-Agent","rawScore":1403.15,"zScore":-0.275717,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.072764,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:4sSMLxWLPihKZSUFVUjwTh","observedHarness":"Epoch MirrorCode Scaffold","rawScore":33.33,"zScore":-0.915438,"seModel":"upstream_se","seRaw":10.02642,"ciHalfWidth":19.6518,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-20","publishedDate":"2026-10-20","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:83173329-c09a-41f1-a028-a282a5f908d7","observedHarness":"Stirrup","rawScore":17.07,"zScore":-0.489404,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:83173329-c09a-41f1-a028-a282a5f908d7","observedHarness":"Direct API","rawScore":51.04,"zScore":-0.111718,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:83173329-c09a-41f1-a028-a282a5f908d7","observedHarness":"mini-SWE-agent","rawScore":14.14,"zScore":-0.635707,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"vals_code_migration","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:code-migration-v1-1:deepseek/deepseek-v4-pro-0813","observedHarness":"Pi","rawScore":51.99,"zScore":-0.210415,"seModel":"upstream_se","seRaw":2.235,"ciHalfWidth":4.3806,"ciLower":null,"ciUpper":null,"measuredCostUsd":12.156614,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/code-migration-v1-1"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:deepseek/deepseek-v4-pro-0813","observedHarness":"Vals SRE Runner","rawScore":1.91,"zScore":-0.622329,"seModel":"upstream_se","seRaw":0.847,"ciHalfWidth":1.6601,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.579756,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"moonshot:kimi-k3:max:standardized-suite:standardized","provider":"moonshot","modelSlug":"kimi-k3","displayName":"Kimi K3","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.615598,"shrinkageFactor":0.979638,"score":40.77,"se":2.71,"samplingSe":2.1324,"residualSe":1.6653,"coverageCount":7,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":9.345558,"costLogSd":1.0507,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1048576,"releaseDate":"2026-08-14","paretoProb":0.2661,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recPTGSK6832BNjII","observedHarness":"ALE-Agent","rawScore":1524.47,"zScore":-0.035879,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.281554,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:kimi-k3_max:max:","observedHarness":"Proximus","rawScore":25.87,"zScore":-0.400058,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":109.71,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:beEmcaWF8wxsWTLD5bMYxi","observedHarness":"Epoch MirrorCode Scaffold","rawScore":54.44,"zScore":0.271168,"seModel":"upstream_se","seRaw":11.439589,"ciHalfWidth":22.4216,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-03","publishedDate":"2026-09-03","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:kimi-k3","observedHarness":"Direct API","rawScore":36,"zScore":-0.493416,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":17.67675,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Kimi K3 (Max)","observedHarness":"LMArena Agent Harness","rawScore":1089.84,"zScore":-0.570257,"seModel":"upstream_ci","ciHalfWidth":11.3956,"ciLower":1078.4458,"ciUpper":1101.2369,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:Kimi K3 (Max reasoning)","observedHarness":"Direct API","rawScore":12.2,"zScore":-0.730456,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"},{"benchmarkId":"vals_code_migration","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:code-migration-v1-1:kimi/kimi-k3","observedHarness":"Pi","rawScore":28.59,"zScore":-1.664188,"seModel":"upstream_se","seRaw":3.78,"ciHalfWidth":7.4088,"ciLower":null,"ciUpper":null,"measuredCostUsd":35.748098,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/code-migration-v1-1"}]},{"id":"spacexai:grok-4.6:low:standardized-suite:standardized","provider":"spacexai","modelSlug":"grok-4.6","displayName":"Grok 4.6","effort":"low","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.622405,"shrinkageFactor":0.612857,"score":40.66,"se":10.34,"samplingSe":9.298,"residualSe":4.5271,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.855384,"costLogSd":1.0404,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-08-25","paretoProb":0.4948,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:grok-4.6_low:low:","observedHarness":"Cursor","rawScore":33.1,"zScore":-0.813018,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.76,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"}]},{"id":"anthropic:claude-sonnet-4.6:medium:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-sonnet-4.6","displayName":"Claude Sonnet 4.6","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.637139,"shrinkageFactor":0.664991,"score":40.44,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.831239,"costLogSd":1.0553,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1000000,"releaseDate":"2026-02-17","paretoProb":0.0613,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:rec79auRfHPVtkKyP","observedHarness":"ALE-Agent","rawScore":1327.3,"zScore":-0.425664,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.770753,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"anthropic:claude-opus-4.7:none:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-opus-4.7","displayName":"Claude Opus 4.7","effort":"none","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.642726,"shrinkageFactor":0.664991,"score":40.36,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.977031,"costLogSd":1.049,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-04-16","paretoProb":0.0551,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recy14NQpCEvMvUAx","observedHarness":"ALE-Agent","rawScore":1323.05,"zScore":-0.434066,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.224549,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"zhipu:glm-5.3:high:standardized-suite:standardized","provider":"zhipu","modelSlug":"glm-5.3","displayName":"GLM-5.3","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.650153,"shrinkageFactor":0.664991,"score":40.25,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.162735,"costLogSd":0.5619,"costMinUsd":1.57493,"costMaxUsd":5.009265,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":1048576,"releaseDate":"2026-09-18","paretoProb":0.2969,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recNCRjM56K1cVX2R","observedHarness":"ALE-Agent","rawScore":1317.4,"zScore":-0.445236,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.284001,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"openai:gpt-5.6-terra:xhigh:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.6-terra","displayName":"GPT-5.6 Terra","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.669565,"shrinkageFactor":0.992494,"score":39.96,"se":6.65,"samplingSe":1.2947,"residualSe":6.5274,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":5.29237,"costLogSd":1.0526,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4.5,"inputPer1mUsd":2,"outputPer1mUsd":12,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0009,"benchmarks":[{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:GPT-5.6 Terra (xHigh)","observedHarness":"LMArena Agent Harness","rawScore":1075.53,"zScore":-0.71956,"seModel":"upstream_ci","ciHalfWidth":16.2754,"ciLower":1059.2541,"ciUpper":1091.8049,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"}]},{"id":"google:gemini-3.8-flash:high:standardized-suite:standardized","provider":"google","modelSlug":"gemini-3.8-flash","displayName":"Gemini 3.8 Flash","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.670314,"shrinkageFactor":0.98586,"score":39.95,"se":2.13,"samplingSe":1.777,"residualSe":1.173,"coverageCount":8,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.542136,"costLogSd":1.045,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.5,"inputPer1mUsd":0.75,"outputPer1mUsd":3.75,"contextWindow":1048576,"releaseDate":"2026-08-10","paretoProb":0.3074,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recdGDUA9gtoZhnsC","observedHarness":"ALE-Agent","rawScore":1269.72,"zScore":-0.539494,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.344275,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:gemini-3.8-flash_high:high:","observedHarness":"Proximus","rawScore":19.63,"zScore":-0.76618,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":38.75,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:Fi32v8c7rD82qhgYVcsz7g","observedHarness":"Epoch MirrorCode Scaffold","rawScore":38.89,"zScore":-0.602907,"seModel":"upstream_se","seRaw":11.915339,"ciHalfWidth":23.3541,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-02","publishedDate":"2026-09-02","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:gemini-3.8-flash","observedHarness":"Direct API","rawScore":16.5,"zScore":-1.588953,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.708494,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Gemini 3.8 Flash (High)","observedHarness":"LMArena Agent Harness","rawScore":1079.57,"zScore":-0.677409,"seModel":"upstream_ci","ciHalfWidth":16.7496,"ciLower":1062.8201,"ciUpper":1096.3192,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:Gemini 3.8 Flash (High reasoning)","observedHarness":"Direct API","rawScore":10.6,"zScore":-0.855731,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"},{"benchmarkId":"vals_code_migration","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:code-migration-v1-1:google/gemini-3.8-flash","observedHarness":"Pi","rawScore":38.65,"zScore":-1.03919,"seModel":"upstream_se","seRaw":3.654,"ciHalfWidth":7.1618,"ciLower":null,"ciUpper":null,"measuredCostUsd":15.787246,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/code-migration-v1-1"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:google/gemini-3.8-flash","observedHarness":"Vals SRE Runner","rawScore":11.07,"zScore":0.04882,"seModel":"upstream_se","seRaw":1.942,"ciHalfWidth":3.8063,"ciLower":null,"ciUpper":null,"measuredCostUsd":27.459572,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"meta:muse-spark-1.3:high:standardized-suite:standardized","provider":"meta","modelSlug":"muse-spark-1.3","displayName":"Muse Spark 1.3","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.675419,"shrinkageFactor":0.608624,"score":39.87,"se":10.37,"samplingSe":9.3487,"residualSe":4.4958,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.768757,"costLogSd":1.0444,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.1087,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:muse-spark-1.3_high:high:","observedHarness":"Cursor","rawScore":32.1,"zScore":-0.903813,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.97,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"}]},{"id":"openai:gpt-5.2:high:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.2","displayName":"GPT-5.2","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.681507,"shrinkageFactor":0.664991,"score":39.78,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.898372,"costLogSd":1.0452,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4.8125,"inputPer1mUsd":1.75,"outputPer1mUsd":14,"contextWindow":400000,"releaseDate":"2025-12-11","paretoProb":0.0031,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recbx03MbsAIPxPvj","observedHarness":"ALE-Agent","rawScore":1293.55,"zScore":-0.492385,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.296865,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"anthropic:claude-haiku-5.5:medium:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-haiku-5.5","displayName":"Claude Haiku 5.5","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.681922,"shrinkageFactor":0.945989,"score":39.77,"se":4.89,"samplingSe":3.4729,"residualSe":3.4442,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.091765,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1000000,"releaseDate":"2026-10-07","paretoProb":0.7748,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:claude-haiku-5-5_medium:medium:","observedHarness":"Cursor","rawScore":34.2,"zScore":-0.713144,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.17,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:af26fff4-8120-41be-a5f0-86aaea023f20","observedHarness":"Stirrup","rawScore":16.09,"zScore":-0.573526,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:af26fff4-8120-41be-a5f0-86aaea023f20","observedHarness":"Direct API","rawScore":48.96,"zScore":-0.38108,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:af26fff4-8120-41be-a5f0-86aaea023f20","observedHarness":"mini-SWE-agent","rawScore":15.15,"zScore":-0.590829,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"meta:muse-spark-1.3:medium:standardized-suite:standardized","provider":"meta","modelSlug":"muse-spark-1.3","displayName":"Muse Spark 1.3","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.717209,"shrinkageFactor":0.605141,"score":39.24,"se":10.4,"samplingSe":9.3902,"residualSe":4.4701,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.941523,"costLogSd":1.055,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.1647,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:muse-spark-1.3_medium:medium:","observedHarness":"Cursor","rawScore":31.3,"zScore":-0.976449,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.75,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"}]},{"id":"openai:gpt-5.4:high:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.4","displayName":"GPT-5.4","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.725531,"shrinkageFactor":0.808799,"score":39.12,"se":7.8,"samplingSe":6.5343,"residualSe":4.2544,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":5.273919,"costLogSd":1.0456,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":5.625,"inputPer1mUsd":2.5,"outputPer1mUsd":15,"contextWindow":1050000,"releaseDate":"2026-03-05","paretoProb":0.001,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recJ9Hqsyml44GqRp","observedHarness":"ALE-Agent","rawScore":1607,"zScore":0.127274,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.612174,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:mirrorcode-ml-2l-gpt-5-4","observedHarness":"Epoch MirrorCode Scaffold","rawScore":15.56,"zScore":-1.9143,"seModel":"upstream_se","seRaw":7.70945,"ciHalfWidth":15.1105,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-10","publishedDate":"2026-08-10","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"}]},{"id":"openai:gpt-5.2:medium:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.2","displayName":"GPT-5.2","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.738982,"shrinkageFactor":0.664991,"score":38.92,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.605187,"costLogSd":1.055,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4.8125,"inputPer1mUsd":1.75,"outputPer1mUsd":14,"contextWindow":400000,"releaseDate":"2025-12-11","paretoProb":0.0087,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recVnMSgaT9EDtGvh","observedHarness":"ALE-Agent","rawScore":1249.83,"zScore":-0.578814,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.142947,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"openai:gpt-5.6-luna:max:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.6-luna","displayName":"GPT-5.6 Luna","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.748669,"shrinkageFactor":0.937769,"score":38.77,"se":5.31,"samplingSe":3.7279,"residualSe":3.7826,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.970791,"costLogSd":1.0515,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.45,"inputPer1mUsd":0.2,"outputPer1mUsd":1.2,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0324,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:rechpIYckpxNmorqX","observedHarness":"ALE-Agent","rawScore":1667.4,"zScore":0.246679,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.359171,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:XwpZADrRSwBo2pETMdLVRq","observedHarness":"Epoch MirrorCode Scaffold","rawScore":21.11,"zScore":-1.602332,"seModel":"upstream_se","seRaw":9.82205,"ciHalfWidth":19.2512,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-21","publishedDate":"2026-10-21","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:openai/gpt-5.6-luna","observedHarness":"Vals SRE Runner","rawScore":1.53,"zScore":-0.650172,"seModel":"upstream_se","seRaw":0.759,"ciHalfWidth":1.4876,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.439085,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"anthropic:claude-opus-4.6:max:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-opus-4.6","displayName":"Claude Opus 4.6","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.749783,"shrinkageFactor":0.757592,"score":38.75,"se":9.24,"samplingSe":7.3575,"residualSe":5.5962,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":14.827231,"costLogSd":1.0558,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-02-05","paretoProb":0.0041,"benchmarks":[{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:Q8sNp4aYrjtYQgVS3BRDTd","observedHarness":"Epoch MirrorCode Scaffold","rawScore":33.33,"zScore":-0.915438,"seModel":"upstream_se","seRaw":10.02642,"ciHalfWidth":19.6518,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-21","publishedDate":"2026-10-21","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"}]},{"id":"zhipu:glm-5.2:max:standardized-suite:standardized","provider":"zhipu","modelSlug":"glm-5.2","displayName":"GLM-5.2","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.770952,"shrinkageFactor":0.973469,"score":38.44,"se":3.82,"samplingSe":2.4341,"residualSe":2.9412,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.211024,"costLogSd":1.0457,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":1048576,"releaseDate":"2026-07-02","paretoProb":0.1952,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recjoKzgMKFtXvQGM","observedHarness":"ALE-Agent","rawScore":1010.17,"zScore":-1.052598,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.148957,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:V3sVnMNiy35HcCtBBuwK4J","observedHarness":"Epoch MirrorCode Scaffold","rawScore":36.67,"zScore":-0.727695,"seModel":"upstream_se","seRaw":11.524072,"ciHalfWidth":22.5872,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-21","publishedDate":"2026-10-21","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:GLM-5.2 (Max)","observedHarness":"LMArena Agent Harness","rawScore":1114.96,"zScore":-0.308168,"seModel":"upstream_ci","ciHalfWidth":14.6891,"ciLower":1100.2737,"ciUpper":1129.6519,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:zai/glm-5.2","observedHarness":"Vals SRE Runner","rawScore":0,"zScore":-0.762274,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":26.575264,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"deepseek:deepseek-v4-pro-0813:high:standardized-suite:standardized","provider":"deepseek","modelSlug":"deepseek-v4-pro-0813","displayName":"DeepSeek V4 Pro (0813)","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.77851,"shrinkageFactor":0.950654,"score":38.32,"se":5.16,"samplingSe":3.3196,"residualSe":3.9554,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.848531,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.98,"inputPer1mUsd":1.32,"outputPer1mUsd":3.96,"contextWindow":1048576,"releaseDate":"2026-08-13","paretoProb":0.0047,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:e94c7595-a320-4c88-8f07-d9fd50fc4595","observedHarness":"Stirrup","rawScore":16.67,"zScore":-0.52374,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:e94c7595-a320-4c88-8f07-d9fd50fc4595","observedHarness":"Direct API","rawScore":51.74,"zScore":-0.021067,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:e94c7595-a320-4c88-8f07-d9fd50fc4595","observedHarness":"mini-SWE-agent","rawScore":6.06,"zScore":-0.99473,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-6-luna:high:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.795856,"shrinkageFactor":0.950838,"score":38.06,"se":5.16,"samplingSe":3.3134,"residualSe":3.9562,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.072948,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.1478,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:7d1229bc-deea-4717-ba5b-f59a35d991e3","observedHarness":"Stirrup","rawScore":18.6,"zScore":-0.358072,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:7d1229bc-deea-4717-ba5b-f59a35d991e3","observedHarness":"Direct API","rawScore":50.35,"zScore":-0.201074,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:7d1229bc-deea-4717-ba5b-f59a35d991e3","observedHarness":"mini-SWE-agent","rawScore":4.55,"zScore":-1.061825,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-5.2:xhigh:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.2","displayName":"GPT-5.2","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.798768,"shrinkageFactor":0.760193,"score":38.02,"se":9.22,"samplingSe":7.3179,"residualSe":5.6155,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.006963,"costLogSd":1.0533,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4.8125,"inputPer1mUsd":1.75,"outputPer1mUsd":14,"contextWindow":400000,"releaseDate":"2025-12-11","paretoProb":0.0003,"benchmarks":[{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:MtySoyHFRZQcrHpzg89ic9","observedHarness":"Epoch MirrorCode Scaffold","rawScore":32.22,"zScore":-0.977832,"seModel":"upstream_se","seRaw":9.955811,"ciHalfWidth":19.5134,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-21","publishedDate":"2026-10-21","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"}]},{"id":"anthropic:claude-sonnet-5.5:low:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"low","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.809046,"shrinkageFactor":0.946055,"score":37.86,"se":4.89,"samplingSe":3.4708,"residualSe":3.4444,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.511342,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.0011,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:claude-sonnet-5-5_low:low:","observedHarness":"Cursor","rawScore":31.8,"zScore":-0.931052,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.44,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:bbc2ffea-cf6b-43be-8c41-1769347e234d","observedHarness":"Stirrup","rawScore":7.13,"zScore":-1.342637,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:bbc2ffea-cf6b-43be-8c41-1769347e234d","observedHarness":"Direct API","rawScore":49.07,"zScore":-0.366835,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:bbc2ffea-cf6b-43be-8c41-1769347e234d","observedHarness":"mini-SWE-agent","rawScore":20.71,"zScore":-0.343778,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-5.1:high:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.1","displayName":"GPT-5.1","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.814809,"shrinkageFactor":0.664991,"score":37.78,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.622478,"costLogSd":1.0441,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3.4375,"inputPer1mUsd":1.25,"outputPer1mUsd":10,"contextWindow":400000,"releaseDate":"2025-11-13","paretoProb":0.0033,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recLNNj0GfqyDx4nk","observedHarness":"ALE-Agent","rawScore":1192.15,"zScore":-0.692842,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.315236,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"openai:gpt-5.4-mini:high:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.4-mini","displayName":"GPT-5.4 Mini","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.819503,"shrinkageFactor":0.664991,"score":37.71,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.796841,"costLogSd":1.0444,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.6875,"inputPer1mUsd":0.75,"outputPer1mUsd":4.5,"contextWindow":400000,"releaseDate":"2026-03-17","paretoProb":0.0134,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recJwrUEXk9e8HLtF","observedHarness":"ALE-Agent","rawScore":1188.58,"zScore":-0.699899,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.271408,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"openai:gpt-6-luna:medium:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.826737,"shrinkageFactor":0.983724,"score":37.6,"se":3.69,"samplingSe":1.9065,"residualSe":3.158,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.042202,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.934,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:36667da0-9222-4967-adbb-f7efa15ad213","observedHarness":"Stirrup","rawScore":12.77,"zScore":-0.858509,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:GPT-6 Luna (Medium)","observedHarness":"LMArena Agent Harness","rawScore":1082.3,"zScore":-0.648925,"seModel":"upstream_ci","ciHalfWidth":7.8962,"ciLower":1074.4052,"ciUpper":1090.1976,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:36667da0-9222-4967-adbb-f7efa15ad213","observedHarness":"Direct API","rawScore":50.93,"zScore":-0.125963,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:36667da0-9222-4967-adbb-f7efa15ad213","observedHarness":"mini-SWE-agent","rawScore":2.53,"zScore":-1.15158,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-5.6-sol:medium:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.6-sol","displayName":"GPT-5.6 Sol","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.849019,"shrinkageFactor":0.593268,"score":37.26,"se":10.49,"samplingSe":9.5304,"residualSe":4.3824,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.666736,"costLogSd":1.0569,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0045,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:gpt-5.6-sol_medium:medium:","observedHarness":"Cursor","rawScore":28.7,"zScore":-1.212516,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.97,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"}]},{"id":"spacexai:grok-4.20:none:standardized-suite:standardized","provider":"spacexai","modelSlug":"grok-4.20","displayName":"Grok 4.20","effort":"none","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.869852,"shrinkageFactor":0.664991,"score":36.95,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.409226,"costLogSd":1.0415,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.5625,"inputPer1mUsd":1.25,"outputPer1mUsd":2.5,"contextWindow":2000000,"releaseDate":"2026-03-09","paretoProb":0.6549,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recyOlTS8wLrTAP1n","observedHarness":"ALE-Agent","rawScore":1150.28,"zScore":-0.775615,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.13838,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"anthropic:claude-opus-4.7:high:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-opus-4.7","displayName":"Claude Opus 4.7","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.870394,"shrinkageFactor":0.793694,"score":36.94,"se":8.97,"samplingSe":6.7875,"residualSe":5.8629,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.432844,"costLogSd":1.0492,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-04-16","paretoProb":0.006,"benchmarks":[{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:mirrorcode-ml-2l-claude-opus-4-7","observedHarness":"Epoch MirrorCode Scaffold","rawScore":31.11,"zScore":-1.040226,"seModel":"upstream_se","seRaw":9.036473,"ciHalfWidth":17.7115,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-12","publishedDate":"2026-08-12","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"}]},{"id":"spacexai:grok-4.5:high:standardized-suite:standardized","provider":"spacexai","modelSlug":"grok-4.5","displayName":"Grok 4.5","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.890315,"shrinkageFactor":0.936362,"score":36.65,"se":5.34,"samplingSe":3.7697,"residualSe":3.777,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.430646,"costLogSd":1.0437,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-06-30","paretoProb":0.1396,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recTr2zSEcDvzbwvB","observedHarness":"ALE-Agent","rawScore":1309.28,"zScore":-0.461288,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.176767,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:fiqzdAtVJVP9wdMT2QFeUn","observedHarness":"Epoch MirrorCode Scaffold","rawScore":32.22,"zScore":-0.977832,"seModel":"upstream_se","seRaw":10.723394,"ciHalfWidth":21.0179,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-21","publishedDate":"2026-10-21","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:grok/grok-4.5","observedHarness":"Vals SRE Runner","rawScore":0.76,"zScore":-0.706589,"seModel":"upstream_se","seRaw":0.539,"ciHalfWidth":1.0564,"ciLower":null,"ciUpper":null,"measuredCostUsd":13.415223,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"openai:gpt-5.5:none:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.5","displayName":"GPT-5.5","effort":"none","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.899694,"shrinkageFactor":0.664991,"score":36.5,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.478023,"costLogSd":1.0509,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":11.25,"inputPer1mUsd":5,"outputPer1mUsd":30,"contextWindow":1050000,"releaseDate":"2026-04-23","paretoProb":0.0026,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:rec9k7gkUezxdYAtu","observedHarness":"ALE-Agent","rawScore":1127.58,"zScore":-0.82049,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.073484,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"meta:muse-spark-1.3:low:standardized-suite:standardized","provider":"meta","modelSlug":"muse-spark-1.3","displayName":"Muse Spark 1.3","effort":"low","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.907693,"shrinkageFactor":0.587528,"score":36.38,"se":10.53,"samplingSe":9.5974,"residualSe":4.34,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.621949,"costLogSd":1.0403,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.1968,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:muse-spark-1.3_low:low:","observedHarness":"Cursor","rawScore":27.5,"zScore":-1.32147,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.13,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"}]},{"id":"moonshot:kimi-k2.6:thinking:standardized-suite:standardized","provider":"moonshot","modelSlug":"kimi-k2.6","displayName":"Kimi K2.6","effort":"thinking","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.945588,"shrinkageFactor":0.664991,"score":35.82,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.181727,"costLogSd":1.032,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.7125,"inputPer1mUsd":0.95,"outputPer1mUsd":4,"contextWindow":262144,"releaseDate":"2026-04-10","paretoProb":0.8372,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recgKppN84MLEgXWf","observedHarness":"ALE-Agent","rawScore":1092.67,"zScore":-0.889504,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.196861,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"openai:gpt-5.4:none:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.4","displayName":"GPT-5.4","effort":"none","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.954317,"shrinkageFactor":0.664991,"score":35.69,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.859635,"costLogSd":1.0443,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":5.625,"inputPer1mUsd":2.5,"outputPer1mUsd":15,"contextWindow":1050000,"releaseDate":"2026-03-05","paretoProb":0.0091,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recMra8s8Fm1RC85F","observedHarness":"ALE-Agent","rawScore":1086.03,"zScore":-0.90263,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.052314,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"google:gemini-3.7-flash:high:standardized-suite:standardized","provider":"google","modelSlug":"gemini-3.7-flash","displayName":"Gemini 3.7 Flash","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.974659,"shrinkageFactor":0.975457,"score":35.38,"se":3.55,"samplingSe":2.3411,"residualSe":2.6627,"coverageCount":5,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.542136,"costLogSd":1.045,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.5,"inputPer1mUsd":0.75,"outputPer1mUsd":3.75,"contextWindow":1048576,"releaseDate":"2026-06-18","paretoProb":0.2115,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recocjfk7wUzbsucJ","observedHarness":"ALE-Agent","rawScore":904.3,"zScore":-1.261892,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.243628,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:gemini-3.7-flash_high:high:","observedHarness":"Proximus","rawScore":20.26,"zScore":-0.729216,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":34.14,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:eeq6feqZLzfjfVsx4aeAWL","observedHarness":"Epoch MirrorCode Scaffold","rawScore":41.11,"zScore":-0.47812,"seModel":"upstream_se","seRaw":12.157114,"ciHalfWidth":23.8279,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-21","publishedDate":"2026-10-21","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Gemini 3.7 Flash (High)","observedHarness":"LMArena Agent Harness","rawScore":1032.29,"zScore":-1.170704,"seModel":"upstream_ci","ciHalfWidth":17.1376,"ciLower":1015.1492,"ciUpper":1049.4245,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:google/gemini-3.7-flash","observedHarness":"Vals SRE Runner","rawScore":4.58,"zScore":-0.4267,"seModel":"upstream_se","seRaw":1.294,"ciHalfWidth":2.5362,"ciLower":null,"ciUpper":null,"measuredCostUsd":21.474171,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"anthropic:claude-haiku-5.5:low:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-haiku-5.5","displayName":"Claude Haiku 5.5","effort":"low","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.98254,"shrinkageFactor":0.946694,"score":35.26,"se":4.88,"samplingSe":3.4502,"residualSe":3.4468,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.043758,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1000000,"releaseDate":"2026-10-07","paretoProb":0.9993,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:claude-haiku-5-5_low:low:","observedHarness":"Cursor","rawScore":27.8,"zScore":-1.294232,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.08,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:42908ae3-d3bd-4934-be19-fbff93d3fde8","observedHarness":"Stirrup","rawScore":8.04,"zScore":-1.264524,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:42908ae3-d3bd-4934-be19-fbff93d3fde8","observedHarness":"Direct API","rawScore":49.19,"zScore":-0.351295,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:42908ae3-d3bd-4934-be19-fbff93d3fde8","observedHarness":"mini-SWE-agent","rawScore":12.63,"zScore":-0.702802,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"google:gemini-3.1-pro:low:standardized-suite:standardized","provider":"google","modelSlug":"gemini-3.1-pro","displayName":"Gemini 3.1 Pro","effort":"low","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.995399,"shrinkageFactor":0.664991,"score":35.07,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.521266,"costLogSd":1.0428,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4.5,"inputPer1mUsd":2,"outputPer1mUsd":12,"contextWindow":1048576,"releaseDate":"2026-02-19","paretoProb":0.2507,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recZG8rbQak39ddaG","observedHarness":"ALE-Agent","rawScore":1054.78,"zScore":-0.964408,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.040324,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"zhipu:glm-5.2:high:standardized-suite:standardized","provider":"zhipu","modelSlug":"glm-5.2","displayName":"GLM-5.2","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.005692,"shrinkageFactor":0.664991,"score":34.91,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.836075,"costLogSd":1.0443,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":1048576,"releaseDate":"2026-07-02","paretoProb":0.3479,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:reczG9I4ZtXmjGGEg","observedHarness":"ALE-Agent","rawScore":1046.95,"zScore":-0.979887,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.157666,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"openai:gpt-5.6-sol:high:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.6-sol","displayName":"GPT-5.6 Sol","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.059859,"shrinkageFactor":0.816442,"score":34.1,"se":7.71,"samplingSe":6.4024,"residualSe":4.303,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.896328,"costLogSd":1.0475,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0001,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:gpt-5.6-sol_high:high:","observedHarness":"Cursor","rawScore":33.2,"zScore":-0.803939,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":3,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:mirrorcode-ml-2l-gpt-5-6-sol","observedHarness":"Epoch MirrorCode Scaffold","rawScore":20,"zScore":-1.664725,"seModel":"upstream_se","seRaw":9.229582,"ciHalfWidth":18.09,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-10","publishedDate":"2026-08-10","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"}]},{"id":"openai:gpt-5.4-nano:high:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.4-nano","displayName":"GPT-5.4 Nano","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.061471,"shrinkageFactor":0.664991,"score":34.08,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.576808,"costLogSd":1.0536,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.4625,"inputPer1mUsd":0.2,"outputPer1mUsd":1.25,"contextWindow":400000,"releaseDate":"2026-03-17","paretoProb":0.0579,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:rec9SvDfW1FDfMZ6h","observedHarness":"ALE-Agent","rawScore":1004.52,"zScore":-1.063767,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.027084,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"anthropic:claude-opus-4.6:none:standardized-suite:standardized","provider":"anthropic","modelSlug":"claude-opus-4.6","displayName":"Claude Opus 4.6","effort":"none","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.072015,"shrinkageFactor":0.664991,"score":33.92,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.977031,"costLogSd":1.049,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-02-05","paretoProb":0.018,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recYZMZozZDvmw4y5","observedHarness":"ALE-Agent","rawScore":996.5,"zScore":-1.079622,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.061688,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"openai:o3:high:standardized-suite:standardized","provider":"openai","modelSlug":"o3","displayName":"o3","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.15477,"shrinkageFactor":0.664991,"score":32.68,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.072691,"costLogSd":1.0438,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3.5,"inputPer1mUsd":2,"outputPer1mUsd":8,"contextWindow":200000,"releaseDate":"2025-04-16","paretoProb":0.0015,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recxPLRj9LORsZ4hq","observedHarness":"ALE-Agent","rawScore":933.55,"zScore":-1.204068,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.103474,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"google:gemini-3.5-flash:high:standardized-suite:standardized","provider":"google","modelSlug":"gemini-3.5-flash","displayName":"Gemini 3.5 Flash","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.184388,"shrinkageFactor":0.664991,"score":32.23,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.335721,"costLogSd":1.0439,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3.375,"inputPer1mUsd":1.5,"outputPer1mUsd":9,"contextWindow":1048576,"releaseDate":"2026-04-12","paretoProb":0.0705,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recqmNjSqCngkNswY","observedHarness":"ALE-Agent","rawScore":911.02,"zScore":-1.248607,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.441921,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"google:gemini-3.1-pro:high:standardized-suite:standardized","provider":"google","modelSlug":"gemini-3.1-pro","displayName":"Gemini 3.1 Pro","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.189654,"shrinkageFactor":0.980333,"score":32.16,"se":3.03,"samplingSe":2.0957,"residualSe":2.1879,"coverageCount":6,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.316496,"costLogSd":1.0447,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4.5,"inputPer1mUsd":2,"outputPer1mUsd":12,"contextWindow":1048576,"releaseDate":"2026-02-19","paretoProb":0.03,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recMLdPe4oTagwHaL","observedHarness":"ALE-Agent","rawScore":1160.6,"zScore":-0.755213,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.388403,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:mirrorcode-ml-2l-gemini-3-1-pro-preview","observedHarness":"Epoch MirrorCode Scaffold","rawScore":8.89,"zScore":-2.289225,"seModel":"upstream_se","seRaw":4.561939,"ciHalfWidth":8.9414,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-10","publishedDate":"2026-08-10","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Gemini 3.1 Pro Preview","observedHarness":"LMArena Agent Harness","rawScore":986.33,"zScore":-1.650227,"seModel":"upstream_ci","ciHalfWidth":29.1276,"ciLower":957.2037,"ciUpper":1015.4589,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:bbd93ebe-80da-4594-bb19-61e69d0331df","observedHarness":"Direct API","rawScore":58.68,"zScore":0.877669,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:bbd93ebe-80da-4594-bb19-61e69d0331df","observedHarness":"mini-SWE-agent","rawScore":4.04,"zScore":-1.084486,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:google/gemini-3.1-pro-preview","observedHarness":"Vals SRE Runner","rawScore":0,"zScore":-0.762274,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":11.336961,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"zhipu:glm-5.1:thinking:standardized-suite:standardized","provider":"zhipu","modelSlug":"glm-5.1","displayName":"GLM-5.1","effort":"thinking","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.215834,"shrinkageFactor":0.664991,"score":31.76,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.314752,"costLogSd":1.0316,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":204800,"releaseDate":"2026-04-15","paretoProb":0.3601,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recZnRHVnR4hKTTVy","observedHarness":"ALE-Agent","rawScore":887.1,"zScore":-1.295895,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.12937,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"google:gemini-4-hydrogen:low:standardized-suite:standardized","provider":"google","modelSlug":"gemini-4-hydrogen","displayName":"Gemini 4 Hydrogen","effort":"low","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.248906,"shrinkageFactor":0.959819,"score":31.27,"se":4.99,"samplingSe":2.9955,"residualSe":3.9936,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.292223,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":1.5,"outputPer1mUsd":7.5,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.5677,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:gemini-4-hydrogen_low:low:","observedHarness":"Cursor","rawScore":23.9,"zScore":-1.648333,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.28,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:88c68c50-03c1-458a-9292-bd398c27adc5","observedHarness":"Stirrup","rawScore":9.29,"zScore":-1.157226,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:88c68c50-03c1-458a-9292-bd398c27adc5","observedHarness":"mini-SWE-agent","rawScore":6.57,"zScore":-0.972069,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-5.4:xhigh:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.4","displayName":"GPT-5.4","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.256496,"shrinkageFactor":0.827332,"score":31.15,"se":8.71,"samplingSe":6.2096,"residualSe":6.1114,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.468237,"costLogSd":1.0538,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":5.625,"inputPer1mUsd":2.5,"outputPer1mUsd":15,"contextWindow":1050000,"releaseDate":"2026-03-05","paretoProb":0.0001,"benchmarks":[{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:RBdTQudLgW7a5iaooxsGeg","observedHarness":"Epoch MirrorCode Scaffold","rawScore":23.33,"zScore":-1.477544,"seModel":"upstream_se","seRaw":8.099905,"ciHalfWidth":15.8758,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-29","publishedDate":"2026-10-29","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"}]},{"id":"moonshot:kimi-k2.7-code:max:standardized-suite:standardized","provider":"moonshot","modelSlug":"kimi-k2.7-code","displayName":"Kimi K2.7 Code","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.257511,"shrinkageFactor":0.908618,"score":31.14,"se":5.84,"samplingSe":4.5174,"residualSe":3.6961,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.884912,"costLogSd":1.0458,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.7125,"inputPer1mUsd":0.95,"outputPer1mUsd":4,"contextWindow":262144,"releaseDate":"2026-06-15","paretoProb":0.3967,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:rec4wsb1ZD4xGsic4","observedHarness":"ALE-Agent","rawScore":886.23,"zScore":-1.297614,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.184743,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:8d0cb231-7303-452c-9923-a9620b948475","observedHarness":"Direct API","rawScore":47.8,"zScore":-0.531301,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:8d0cb231-7303-452c-9923-a9620b948475","observedHarness":"mini-SWE-agent","rawScore":1.01,"zScore":-1.219119,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-6-luna:low:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"low","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.274192,"shrinkageFactor":0.985323,"score":30.89,"se":3.64,"samplingSe":1.8104,"residualSe":3.1631,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.005115,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":1,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:bf9708d8-933d-44f6-affa-b696a7a650c9","observedHarness":"Stirrup","rawScore":2.2,"zScore":-1.765819,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:GPT-6 Luna (Low)","observedHarness":"LMArena Agent Harness","rawScore":1037.75,"zScore":-1.113737,"seModel":"upstream_ci","ciHalfWidth":13.2927,"ciLower":1024.4596,"ciUpper":1051.045,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:bf9708d8-933d-44f6-affa-b696a7a650c9","observedHarness":"Direct API","rawScore":46.88,"zScore":-0.650442,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:bf9708d8-933d-44f6-affa-b696a7a650c9","observedHarness":"mini-SWE-agent","rawScore":0,"zScore":-1.263997,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-5:minimal:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5","displayName":"GPT-5","effort":"minimal","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.32028,"shrinkageFactor":0.664991,"score":30.2,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.276104,"costLogSd":1.0418,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3.4375,"inputPer1mUsd":1.25,"outputPer1mUsd":10,"contextWindow":400000,"releaseDate":"2025-08-07","paretoProb":0.01,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:rec39nSc7yf62OCDn","observedHarness":"ALE-Agent","rawScore":807.65,"zScore":-1.452959,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.043744,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"alibaba:qwen3.7-max:thinking:standardized-suite:standardized","provider":"alibaba","modelSlug":"qwen3.7-max","displayName":"Qwen3.7 Max","effort":"thinking","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.320992,"shrinkageFactor":0.937385,"score":30.19,"se":5.73,"samplingSe":3.7393,"residualSe":4.3474,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.123992,"costLogSd":1.0308,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3.75,"inputPer1mUsd":2.5,"outputPer1mUsd":7.5,"contextWindow":1000000,"releaseDate":"2026-05-20","paretoProb":0.5654,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recA9I9t2L4RQrsQs","observedHarness":"ALE-Agent","rawScore":1189.4,"zScore":-0.698278,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.3845,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Qwen3.7-Max","observedHarness":"LMArena Agent Harness","rawScore":991.03,"zScore":-1.601189,"seModel":"upstream_ci","ciHalfWidth":21.798,"ciLower":969.2289,"ciUpper":1012.8249,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"}]},{"id":"deepseek:deepseek-v4.1-flash-1017:none:standardized-suite:standardized","provider":"deepseek","modelSlug":"deepseek-v4.1-flash-1017","displayName":"DeepSeek V4.1 Flash (1017)","effort":"none","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.326127,"shrinkageFactor":0.954815,"score":30.11,"se":5.09,"samplingSe":3.1765,"residualSe":3.9727,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.42542,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.525,"inputPer1mUsd":0.3,"outputPer1mUsd":1.2,"contextWindow":1048576,"releaseDate":"2026-10-17","paretoProb":1,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:5720fd2f-873a-4765-8640-a153b38bd64d","observedHarness":"Stirrup","rawScore":6.1,"zScore":-1.43105,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:5720fd2f-873a-4765-8640-a153b38bd64d","observedHarness":"Direct API","rawScore":38.54,"zScore":-1.73048,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:5720fd2f-873a-4765-8640-a153b38bd64d","observedHarness":"mini-SWE-agent","rawScore":9.09,"zScore":-0.860096,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"deepseek:deepseek-v4-pro-0813:none:standardized-suite:standardized","provider":"deepseek","modelSlug":"deepseek-v4-pro-0813","displayName":"DeepSeek V4 Pro (0813)","effort":"none","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.340305,"shrinkageFactor":0.956876,"score":29.9,"se":5.05,"samplingSe":3.1032,"residualSe":3.9813,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.976988,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.98,"inputPer1mUsd":1.32,"outputPer1mUsd":3.96,"contextWindow":1048576,"releaseDate":"2026-08-13","paretoProb":0,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:d4f5b250-be37-4801-85e6-32fda4179337","observedHarness":"Stirrup","rawScore":10.31,"zScore":-1.069671,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:d4f5b250-be37-4801-85e6-32fda4179337","observedHarness":"Direct API","rawScore":39.93,"zScore":-1.550474,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:d4f5b250-be37-4801-85e6-32fda4179337","observedHarness":"mini-SWE-agent","rawScore":1.52,"zScore":-1.196458,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"zhipu:glm-5:thinking:standardized-suite:standardized","provider":"zhipu","modelSlug":"glm-5","displayName":"GLM-5","effort":"thinking","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.375534,"shrinkageFactor":0.664991,"score":29.37,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.989151,"costLogSd":1.033,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.55,"inputPer1mUsd":1,"outputPer1mUsd":3.2,"contextWindow":204800,"releaseDate":"2026-02-01","paretoProb":0.4089,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:rec1dnhYFrGJo63GA","observedHarness":"ALE-Agent","rawScore":765.62,"zScore":-1.536048,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.073284,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"google:gemini-3.5-flash-lite:high:standardized-suite:standardized","provider":"google","modelSlug":"gemini-3.5-flash-lite","displayName":"Gemini 3.5 Flash-Lite","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.375994,"shrinkageFactor":0.664991,"score":29.36,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.529863,"costLogSd":0.6488,"costMinUsd":0.692902,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.85,"inputPer1mUsd":0.3,"outputPer1mUsd":2.5,"contextWindow":1048576,"releaseDate":"2026-04-12","paretoProb":0.0038,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recQXW49hzii6cT10","observedHarness":"ALE-Agent","rawScore":765.27,"zScore":-1.53674,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.045726,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"openai:gpt-5:high:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5","displayName":"GPT-5","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.38074,"shrinkageFactor":0.820922,"score":29.29,"se":7.66,"samplingSe":6.3238,"residualSe":4.3182,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.622478,"costLogSd":1.0441,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3.4375,"inputPer1mUsd":1.25,"outputPer1mUsd":10,"contextWindow":400000,"releaseDate":"2025-08-07","paretoProb":0.0003,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:rec6WynYilo1cLLBN","observedHarness":"ALE-Agent","rawScore":1162.45,"zScore":-0.751556,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.244948,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:Th39ihgvLFnsDGMed7Q55b","observedHarness":"Epoch MirrorCode Scaffold","rawScore":4.44,"zScore":-2.539362,"seModel":"upstream_se","seRaw":3.437525,"ciHalfWidth":6.7375,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-21","publishedDate":"2026-10-21","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"}]},{"id":"meta:muse-spark-1.2:xhigh:standardized-suite:standardized","provider":"meta","modelSlug":"muse-spark-1.2","displayName":"Muse Spark 1.2","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.391613,"shrinkageFactor":0.995715,"score":29.13,"se":3.82,"samplingSe":0.9783,"residualSe":3.6934,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.165655,"costLogSd":1.051,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.0117,"benchmarks":[{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:muse-spark-1.2_xhigh:xhigh:","observedHarness":"Proximus","rawScore":11.97,"zScore":-1.215618,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":27.81,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Muse Spark 1.2 (xHigh)","observedHarness":"LMArena Agent Harness","rawScore":971.81,"zScore":-1.801721,"seModel":"upstream_ci","ciHalfWidth":17.4586,"ciLower":954.3546,"ciUpper":989.2719,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:meta/muse_spark_1_2","observedHarness":"Vals SRE Runner","rawScore":0,"zScore":-0.762274,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":23.427477,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"google:gemini-3.5-flash-lite:minimal:standardized-suite:standardized","provider":"google","modelSlug":"gemini-3.5-flash-lite","displayName":"Gemini 3.5 Flash-Lite","effort":"minimal","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.426336,"shrinkageFactor":0.958578,"score":28.6,"se":5.02,"samplingSe":3.0414,"residualSe":3.9884,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.692902,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.85,"inputPer1mUsd":0.3,"outputPer1mUsd":2.5,"contextWindow":1048576,"releaseDate":"2026-04-12","paretoProb":0.0647,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:093b9df2-ebee-4309-943f-5a7e88360adf","observedHarness":"Stirrup","rawScore":6.03,"zScore":-1.437059,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:093b9df2-ebee-4309-943f-5a7e88360adf","observedHarness":"Direct API","rawScore":41.32,"zScore":-1.370467,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:093b9df2-ebee-4309-943f-5a7e88360adf","observedHarness":"mini-SWE-agent","rawScore":1.01,"zScore":-1.219119,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-6-luna:none:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"none","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.486567,"shrinkageFactor":0.95991,"score":27.7,"se":4.99,"samplingSe":2.9921,"residualSe":3.9939,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.071417,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0024,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:63cb48ea-0b0c-4e07-8ded-6d28dec4fa28","observedHarness":"Stirrup","rawScore":1.58,"zScore":-1.819039,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:63cb48ea-0b0c-4e07-8ded-6d28dec4fa28","observedHarness":"Direct API","rawScore":43.06,"zScore":-1.145136,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:63cb48ea-0b0c-4e07-8ded-6d28dec4fa28","observedHarness":"mini-SWE-agent","rawScore":1.52,"zScore":-1.196458,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"google:gemini-3.6-flash:high:standardized-suite:standardized","provider":"google","modelSlug":"gemini-3.6-flash","displayName":"Gemini 3.6 Flash","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.519933,"shrinkageFactor":0.801841,"score":27.2,"se":7.88,"samplingSe":6.6522,"residualSe":4.2178,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.542136,"costLogSd":1.045,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.5,"inputPer1mUsd":0.75,"outputPer1mUsd":3.75,"contextWindow":1048576,"releaseDate":"2026-05-05","paretoProb":0.1384,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recjoXKmlx7gYuFVy","observedHarness":"ALE-Agent","rawScore":715.52,"zScore":-1.635091,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.244468,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:ftpz5JnywQdGtFDCHyjMt4","observedHarness":"Epoch MirrorCode Scaffold","rawScore":23.33,"zScore":-1.477544,"seModel":"upstream_se","seRaw":9.315175,"ciHalfWidth":18.2577,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-21","publishedDate":"2026-10-21","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"}]},{"id":"meta:muse-spark-1.1:high:standardized-suite:standardized","provider":"meta","modelSlug":"muse-spark-1.1","displayName":"Muse Spark 1.1","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.884412,"shrinkageFactor":0.995273,"score":21.73,"se":6.63,"samplingSe":1.0274,"residualSe":6.5457,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.768757,"costLogSd":1.0444,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.0151,"benchmarks":[{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Muse Spark 1.1","observedHarness":"LMArena Agent Harness","rawScore":958.64,"zScore":-1.93913,"seModel":"upstream_ci","ciHalfWidth":12.8979,"ciLower":945.7414,"ciUpper":971.5373,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"}]},{"id":"zhipu:glm-5.3-flash:high:standardized-suite:standardized","provider":"zhipu","modelSlug":"glm-5.3-flash","displayName":"GLM-5.3 Flash","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.982979,"shrinkageFactor":0.664991,"score":20.26,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.298808,"costLogSd":0.6488,"costMaxUsd":0.58331,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2375,"inputPer1mUsd":0.15,"outputPer1mUsd":0.5,"contextWindow":1048576,"releaseDate":"2026-09-28","paretoProb":0.626,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recuPsUOAayyoYdDf","observedHarness":"ALE-Agent","rawScore":303.55,"zScore":-2.449513,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.010869,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"openai:gpt-5.5:high:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.5","displayName":"GPT-5.5","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-2.010878,"shrinkageFactor":0.89778,"score":19.84,"se":8.17,"samplingSe":4.7777,"residualSe":6.6318,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":9.848117,"costLogSd":1.0508,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":11.25,"inputPer1mUsd":5,"outputPer1mUsd":30,"contextWindow":1050000,"releaseDate":"2026-04-23","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:mirrorcode-ml-2l-gpt-5-5","observedHarness":"Epoch MirrorCode Scaffold","rawScore":10,"zScore":-2.226831,"seModel":"upstream_se","seRaw":6.04218,"ciHalfWidth":11.8427,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-10","publishedDate":"2026-08-10","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"}]}],"provisionalConfigs":[{"id":"meta:muse-wonder-full:max:standardized-suite:standardized","provider":"meta","modelSlug":"muse-wonder-full","displayName":"Muse Wonder Full","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":0.633137,"shrinkageFactor":0.989243,"score":59.5,"se":2.32,"samplingSe":1.5499,"residualSe":1.7235,"coverageCount":6,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2","observedHarness":"Stirrup","rawScore":37.76,"zScore":1.286589,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Muse Wonder Full (Max)","observedHarness":"LMArena Agent Harness","rawScore":1212.27,"zScore":0.707115,"seModel":"upstream_ci","ciHalfWidth":16.6432,"ciLower":1195.6228,"ciUpper":1228.9092,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2","observedHarness":"Direct API","rawScore":62.62,"zScore":1.387903,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2","observedHarness":"mini-SWE-agent","rawScore":57.07,"zScore":1.271826,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"vals_code_migration","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:code-migration-v1-1:meta/muse-wonder-full","observedHarness":"Pi","rawScore":62.79,"zScore":0.460557,"seModel":"upstream_se","seRaw":3.175,"ciHalfWidth":6.223,"ciLower":null,"ciUpper":null,"measuredCostUsd":23.584618,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/code-migration-v1-1"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:meta/muse-wonder-full","observedHarness":"Vals SRE Runner","rawScore":11.83,"zScore":0.104504,"seModel":"upstream_se","seRaw":1.999,"ciHalfWidth":3.918,"ciLower":null,"ciUpper":null,"measuredCostUsd":33.810651,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"alibaba:qwen3.8-max:xhigh:standardized-suite:standardized","provider":"alibaba","modelSlug":"qwen3.8-max","displayName":"Qwen3.8 Max","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.240407,"shrinkageFactor":0.975257,"score":46.39,"se":3.95,"samplingSe":2.3506,"residualSe":3.1729,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-08-03","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_frontierswe:qwen3.8-max_xhigh:xhigh:","observedHarness":"Proximus","rawScore":15.81,"zScore":-0.990312,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":55.14,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.frontierswe.com/"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:RQRuLNXoCHzNMEbJCcUVVD","observedHarness":"Epoch MirrorCode Scaffold","rawScore":52.22,"zScore":0.14638,"seModel":"upstream_se","seRaw":10.894718,"ciHalfWidth":21.3536,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-21","publishedDate":"2026-10-21","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Qwen3.8-Max","observedHarness":"LMArena Agent Harness","rawScore":1094.71,"zScore":-0.519446,"seModel":"upstream_ci","ciHalfWidth":12.7079,"ciLower":1081.9981,"ciUpper":1107.414,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:sudo-l7:Qwen 3.8 Max (xHigh reasoning)","observedHarness":"Direct API","rawScore":16.7,"zScore":-0.378123,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/sudo-l7"}]},{"id":"deepseek:deepseek-v4.1-flash:max:standardized-suite:standardized","provider":"deepseek","modelSlug":"deepseek-v4.1-flash","displayName":"DeepSeek V4.1 Flash","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.358642,"shrinkageFactor":0.971003,"score":44.62,"se":3.87,"samplingSe":2.5447,"residualSe":2.9131,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-10","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recgLH5qA0YS3ftkG","observedHarness":"ALE-Agent","rawScore":1092.35,"zScore":-0.890136,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.081663,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_mirrorcode:nRrYnwt7v5xuS4K29sTubp","observedHarness":"Epoch MirrorCode Scaffold","rawScore":64.44,"zScore":0.833274,"seModel":"upstream_se","seRaw":11.37,"ciHalfWidth":22.2852,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-21","publishedDate":"2026-10-21","sourceUrl":"https://epoch.ai/benchmarks/mirrorcode"},{"benchmarkId":"vals_code_migration","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:code-migration-v1-1:deepseek/deepseek-v4.1-flash","observedHarness":"Pi","rawScore":58.94,"zScore":0.221368,"seModel":"upstream_se","seRaw":2.656,"ciHalfWidth":5.2058,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.088847,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/code-migration-v1-1"},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:sre_bench:deepseek/deepseek-v4.1-flash","observedHarness":"Vals SRE Runner","rawScore":0.76,"zScore":-0.706589,"seModel":"upstream_se","seRaw":0.539,"ciHalfWidth":1.0564,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.548583,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/sre_bench"}]},{"id":"alibaba:qwen3.8-flash-next:xhigh:standardized-suite:standardized","provider":"alibaba","modelSlug":"qwen3.8-flash-next","displayName":"Qwen3.8 Flash Next","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.415472,"shrinkageFactor":0.977989,"score":43.77,"se":3.84,"samplingSe":2.2171,"residualSe":3.1395,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":0.569381,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-15","paretoProb":0,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:9ac3908c-9eb2-44db-8c01-3ec7da6fc2ae","observedHarness":"Stirrup","rawScore":27.14,"zScore":0.374987,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Qwen3.8-Flash-Next","observedHarness":"LMArena Agent Harness","rawScore":1063.68,"zScore":-0.843197,"seModel":"upstream_ci","ciHalfWidth":13.881,"ciLower":1049.7959,"ciUpper":1077.558,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:9ac3908c-9eb2-44db-8c01-3ec7da6fc2ae","observedHarness":"Direct API","rawScore":50.58,"zScore":-0.171288,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:9ac3908c-9eb2-44db-8c01-3ec7da6fc2ae","observedHarness":"mini-SWE-agent","rawScore":25.25,"zScore":-0.14205,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"alibaba:qwen3.8-2.4t-a95b:xhigh:standardized-suite:standardized","provider":"alibaba","modelSlug":"qwen3.8-2.4t-a95b","displayName":"Qwen3.8 2.4T A95B","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.585302,"shrinkageFactor":0.910119,"score":41.22,"se":6.57,"samplingSe":4.4801,"residualSe":4.8016,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-12","paretoProb":0,"benchmarks":[{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:8df710d3-9dae-4498-9b4e-9818238e6f31","observedHarness":"Direct API","rawScore":54.05,"zScore":0.27808,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:8df710d3-9dae-4498-9b4e-9818238e6f31","observedHarness":"mini-SWE-agent","rawScore":11.11,"zScore":-0.770341,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"alibaba:qwen3.8-max:max:standardized-suite:standardized","provider":"alibaba","modelSlug":"qwen3.8-max","displayName":"Qwen3.8 Max","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.591876,"shrinkageFactor":0.738078,"score":41.12,"se":9.36,"samplingSe":7.6479,"residualSe":5.3911,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-08-03","paretoProb":0,"benchmarks":[{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:qwen3.8-max","observedHarness":"Direct API","rawScore":26.6,"zScore":-1.021521,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":16.590033,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"}]},{"id":"openai:gpt-5.5-instant:none:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.5-instant","displayName":"GPT-5.5 Instant","effort":"none","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.594933,"shrinkageFactor":0.908484,"score":41.08,"se":6.59,"samplingSe":4.5207,"residualSe":4.793,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-06-01","paretoProb":0,"benchmarks":[{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:c4169d75-9d24-4794-a9c3-69e1fe364de5","observedHarness":"Direct API","rawScore":52.55,"zScore":0.083828,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:c4169d75-9d24-4794-a9c3-69e1fe364de5","observedHarness":"mini-SWE-agent","rawScore":12.63,"zScore":-0.702802,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"deepseek:deepseek-v4-flash:max:standardized-suite:standardized","provider":"deepseek","modelSlug":"deepseek-v4-flash","displayName":"DeepSeek V4 Flash","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.665035,"shrinkageFactor":0.664991,"score":40.02,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-31","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recuPYbCCpYAt7G1G","observedHarness":"ALE-Agent","rawScore":1306.08,"zScore":-0.467614,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.019526,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"deepseek:deepseek-v4-flash-vision:max:standardized-suite:standardized","provider":"deepseek","modelSlug":"deepseek-v4-flash-vision","displayName":"DeepSeek V4 Flash Vision","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.712041,"shrinkageFactor":0.908768,"score":39.32,"se":6.58,"samplingSe":4.5137,"residualSe":4.7945,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-10","paretoProb":0,"benchmarks":[{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:7707a46f-895f-4247-b7a8-ddab87c9a11d","observedHarness":"Direct API","rawScore":49.65,"zScore":-0.291724,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:7707a46f-895f-4247-b7a8-ddab87c9a11d","observedHarness":"mini-SWE-agent","rawScore":12.12,"zScore":-0.725463,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"google:gemini-3-pro:high:standardized-suite:standardized","provider":"google","modelSlug":"gemini-3-pro","displayName":"Gemini 3 Pro","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.835055,"shrinkageFactor":0.664991,"score":37.47,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-11-18","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recPGiw7I44Wc9L5u","observedHarness":"ALE-Agent","rawScore":1176.75,"zScore":-0.723286,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.259385,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"alibaba:qwen3.8-27b:xhigh:standardized-suite:standardized","provider":"alibaba","modelSlug":"qwen3.8-27b","displayName":"Qwen3.8 27B","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-0.997559,"shrinkageFactor":0.981912,"score":35.04,"se":3.74,"samplingSe":2.0098,"residualSe":3.1521,"coverageCount":4,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":3.529549,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-05","paretoProb":0,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:b01dee41-c62b-48ed-8d16-984adc405e5c","observedHarness":"Stirrup","rawScore":14.91,"zScore":-0.674815,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Qwen3.8-27B","observedHarness":"LMArena Agent Harness","rawScore":1041.48,"zScore":-1.07482,"seModel":"upstream_ci","ciHalfWidth":16.2417,"ciLower":1025.2405,"ciUpper":1057.724,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:b01dee41-c62b-48ed-8d16-984adc405e5c","observedHarness":"Direct API","rawScore":46.64,"zScore":-0.681522,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:b01dee41-c62b-48ed-8d16-984adc405e5c","observedHarness":"mini-SWE-agent","rawScore":5.56,"zScore":-1.016947,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"meta:muse-spark-open:xhigh:standardized-suite:standardized","provider":"meta","modelSlug":"muse-spark-open","displayName":"Muse Spark Open","effort":"xhigh","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.02677,"shrinkageFactor":0.968888,"score":34.6,"se":3.7,"samplingSe":2.6358,"residualSe":2.5909,"coverageCount":5,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":2.017668,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recpfB5OsaYt2uwRy","observedHarness":"ALE-Agent","rawScore":1590.12,"zScore":0.093904,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.325474,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:3661c8fb-caf7-4118-aa18-65b84fea8e44","observedHarness":"Stirrup","rawScore":7.6,"zScore":-1.302293,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:3661c8fb-caf7-4118-aa18-65b84fea8e44","observedHarness":"Direct API","rawScore":58.45,"zScore":0.847884,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:3661c8fb-caf7-4118-aa18-65b84fea8e44","observedHarness":"mini-SWE-agent","rawScore":8.08,"zScore":-0.904974,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"vals_code_migration","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:code-migration-v1-1:meta/muse-spark-open","observedHarness":"Pi","rawScore":26.89,"zScore":-1.769804,"seModel":"upstream_se","seRaw":3.229,"ciHalfWidth":6.3288,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.767449,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/code-migration-v1-1"}]},{"id":"deepseek:deepseek-v4-pro:high:standardized-suite:standardized","provider":"deepseek","modelSlug":"deepseek-v4-pro","displayName":"DeepSeek V4 Pro (0424)","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.05942,"shrinkageFactor":0.664991,"score":34.11,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-04-24","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recRdiZQ9HF3Kug6B","observedHarness":"ALE-Agent","rawScore":1006.08,"zScore":-1.060683,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.148062,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"google:gemini-3-pro:low:standardized-suite:standardized","provider":"google","modelSlug":"gemini-3-pro","displayName":"Gemini 3 Pro","effort":"low","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.082886,"shrinkageFactor":0.664991,"score":33.76,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-11-18","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recviUDCuJAokY7Al","observedHarness":"ALE-Agent","rawScore":988.23,"zScore":-1.095971,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.099547,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"alibaba:qwen3.8-27b:max:standardized-suite:standardized","provider":"alibaba","modelSlug":"qwen3.8-27b","displayName":"Qwen3.8 27B","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.09286,"shrinkageFactor":0.795349,"score":33.61,"se":8.91,"samplingSe":6.7602,"residualSe":5.8094,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-05","paretoProb":0,"benchmarks":[{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:qwen3.8-27b","observedHarness":"Direct API","rawScore":15.9,"zScore":-1.622662,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.769956,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"}]},{"id":"openai:gpt-5.6-sol:low:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-5.6-sol","displayName":"GPT-5.6 Sol","effort":"low","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.11483,"shrinkageFactor":0.564792,"score":33.28,"se":10.7,"samplingSe":9.8583,"residualSe":4.1721,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":2.433362,"costLogSd":1.0467,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:gpt-5.6-sol_low:low:","observedHarness":"Cursor","rawScore":23,"zScore":-1.730048,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.98,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"}]},{"id":"google:gemma-4-26b-a4b:none:standardized-suite:standardized","provider":"google","modelSlug":"gemma-4-26b-a4b","displayName":"Gemma 4 26B A4B","effort":"none","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.163157,"shrinkageFactor":0.664991,"score":32.55,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-01","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recatyI2kjiCX3JZJ","observedHarness":"ALE-Agent","rawScore":927.17,"zScore":-1.21668,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.009725,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"meta:muse-spark-1.3:minimal:standardized-suite:standardized","provider":"meta","modelSlug":"muse-spark-1.3","displayName":"Muse Spark 1.3","effort":"minimal","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.187591,"shrinkageFactor":0.555794,"score":32.19,"se":10.77,"samplingSe":9.9597,"residualSe":4.1056,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":0.621949,"costLogSd":1.0403,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_cursorbench:muse-spark-1.3_minimal:minimal:","observedHarness":"Cursor","rawScore":21.3,"zScore":-1.8844,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.69,"evalDate":null,"publishedDate":null,"sourceUrl":"https://cursor.com/cursorbench"}]},{"id":"alibaba:qwen3.8-27b:medium:standardized-suite:standardized","provider":"alibaba","modelSlug":"qwen3.8-27b","displayName":"Qwen3.8 27B","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.259222,"shrinkageFactor":0.954648,"score":31.11,"se":5.09,"samplingSe":3.1824,"residualSe":3.972,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":2.160005,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-05","paretoProb":0,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:6657d7de-a2a9-40bc-a32f-86e80ad63698","observedHarness":"Stirrup","rawScore":11.5,"zScore":-0.967523,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:6657d7de-a2a9-40bc-a32f-86e80ad63698","observedHarness":"Direct API","rawScore":39,"zScore":-1.670909,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:6657d7de-a2a9-40bc-a32f-86e80ad63698","observedHarness":"mini-SWE-agent","rawScore":5.05,"zScore":-1.039608,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"alibaba:qwen3.8-27b:low:standardized-suite:standardized","provider":"alibaba","modelSlug":"qwen3.8-27b","displayName":"Qwen3.8 27B","effort":"low","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.262507,"shrinkageFactor":0.955461,"score":31.06,"se":5.07,"samplingSe":3.1537,"residualSe":3.9754,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":1.905905,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-05","paretoProb":0,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:7e30585f-fad7-40df-a8cb-03c1d96df38a","observedHarness":"Stirrup","rawScore":12.37,"zScore":-0.892844,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:7e30585f-fad7-40df-a8cb-03c1d96df38a","observedHarness":"Direct API","rawScore":40.05,"zScore":-1.534933,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:7e30585f-fad7-40df-a8cb-03c1d96df38a","observedHarness":"mini-SWE-agent","rawScore":2.53,"zScore":-1.15158,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"meta:muse-spark-open:max:standardized-suite:standardized","provider":"meta","modelSlug":"muse-spark-open","displayName":"Muse Spark Open","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.297164,"shrinkageFactor":0.823774,"score":30.54,"se":8.69,"samplingSe":6.2732,"residualSe":6.017,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:muse-spark-open","observedHarness":"Direct API","rawScore":12.1,"zScore":-1.836152,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.497439,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"}]},{"id":"alibaba:qwen3.5-122b-a10b:thinking:standardized-suite:standardized","provider":"alibaba","modelSlug":"qwen3.5-122b-a10b","displayName":"Qwen3.5 122B A10B","effort":"thinking","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.425488,"shrinkageFactor":0.924433,"score":28.62,"se":6.38,"samplingSe":4.1079,"residualSe":4.8771,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-03-20","paretoProb":0,"benchmarks":[{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:2236df45-0699-40d1-b5cc-69ee345d2257","observedHarness":"Direct API","rawScore":39.7,"zScore":-1.580259,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:2236df45-0699-40d1-b5cc-69ee345d2257","observedHarness":"mini-SWE-agent","rawScore":0,"zScore":-1.263997,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-oss-20b:high:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-oss-20b","displayName":"gpt-oss-20b","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.45524,"shrinkageFactor":0.924911,"score":28.17,"se":6.37,"samplingSe":4.0949,"residualSe":4.8797,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-08-05","paretoProb":0,"benchmarks":[{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:36f73aaf-d38a-4b56-a2b3-d04d17186910","observedHarness":"Direct API","rawScore":38.89,"zScore":-1.685155,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:36f73aaf-d38a-4b56-a2b3-d04d17186910","observedHarness":"mini-SWE-agent","rawScore":0,"zScore":-1.263997,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"meta:muse-glimmer:high:standardized-suite:standardized","provider":"meta","modelSlug":"muse-glimmer","displayName":"Muse Glimmer","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.46293,"shrinkageFactor":0.960268,"score":28.06,"se":4.98,"samplingSe":2.9787,"residualSe":3.9954,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":0.178208,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:583f98fb-c4b8-4df3-8d40-60ac0ed69882","observedHarness":"Stirrup","rawScore":1.47,"zScore":-1.828481,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:583f98fb-c4b8-4df3-8d40-60ac0ed69882","observedHarness":"Direct API","rawScore":44.91,"zScore":-0.905559,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:583f98fb-c4b8-4df3-8d40-60ac0ed69882","observedHarness":"mini-SWE-agent","rawScore":0.51,"zScore":-1.241336,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"deepseek:deepseek-v4-flash:high:standardized-suite:standardized","provider":"deepseek","modelSlug":"deepseek-v4-flash","displayName":"DeepSeek V4 Flash","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.490458,"shrinkageFactor":0.664991,"score":27.64,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-31","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recYO1qmn1ztCHit3","observedHarness":"ALE-Agent","rawScore":678.2,"zScore":-1.708868,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.005179,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"google:gemma-4-31b:thinking:standardized-suite:standardized","provider":"google","modelSlug":"gemma-4-31b","displayName":"Gemma 4 31B","effort":"thinking","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.497326,"shrinkageFactor":0.961223,"score":27.54,"se":4.97,"samplingSe":2.9427,"residualSe":3.9994,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-01","paretoProb":0,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:cd26a386-4873-46ff-b853-d239050025a2","observedHarness":"Stirrup","rawScore":0,"zScore":-1.954663,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:cd26a386-4873-46ff-b853-d239050025a2","observedHarness":"Direct API","rawScore":45.49,"zScore":-0.830448,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:cd26a386-4873-46ff-b853-d239050025a2","observedHarness":"mini-SWE-agent","rawScore":0,"zScore":-1.263997,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"google:gemma-4-31b:max:standardized-suite:standardized","provider":"google","modelSlug":"gemma-4-31b","displayName":"Gemma 4 31B","effort":"max","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.509023,"shrinkageFactor":0.855726,"score":27.36,"se":8.44,"samplingSe":5.6761,"residualSe":6.2504,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-01","paretoProb":0,"benchmarks":[{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","sourceId":"livebench","sourceRowId":"livebench:20261106:long_agentic:gemma-4-31b","observedHarness":"Direct API","rawScore":8.5,"zScore":-2.038405,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.968522,"evalDate":null,"publishedDate":"2026-11-06","sourceUrl":"https://livebench.ai/"}]},{"id":"alibaba:qwen3.5-397b-a17b:thinking:standardized-suite:standardized","provider":"alibaba","modelSlug":"qwen3.5-397b-a17b","displayName":"Qwen3.5 397B A17B","effort":"thinking","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.515643,"shrinkageFactor":0.909346,"score":27.27,"se":5.82,"samplingSe":4.4993,"residualSe":3.699,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-03-20","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recKClj8NjtsOOnGa","observedHarness":"ALE-Agent","rawScore":621.92,"zScore":-1.820128,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.03981,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:0e66bae9-41f1-42fc-9276-ce8cb6f72919","observedHarness":"Direct API","rawScore":44.79,"zScore":-0.921099,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:0e66bae9-41f1-42fc-9276-ce8cb6f72919","observedHarness":"mini-SWE-agent","rawScore":0,"zScore":-1.263997,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"alibaba:qwen3.6-35b-a3b:thinking:standardized-suite:standardized","provider":"alibaba","modelSlug":"qwen3.6-35b-a3b","displayName":"Qwen3.6 35B A3B","effort":"thinking","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.540897,"shrinkageFactor":0.926481,"score":26.89,"se":6.35,"samplingSe":4.0519,"residualSe":4.888,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-05-20","paretoProb":0,"benchmarks":[{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:8665ca00-c687-44c7-875c-22618cb31c4f","observedHarness":"Direct API","rawScore":36.57,"zScore":-1.985597,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:8665ca00-c687-44c7-875c-22618cb31c4f","observedHarness":"mini-SWE-agent","rawScore":0,"zScore":-1.263997,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"alibaba:qwen3-coder-next:none:standardized-suite:standardized","provider":"alibaba","modelSlug":"qwen3-coder-next","displayName":"Qwen3 Coder Next","effort":"none","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.553509,"shrinkageFactor":0.926737,"score":26.7,"se":6.35,"samplingSe":4.0448,"residualSe":4.8893,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-02-03","paretoProb":0,"benchmarks":[{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:fc92f822-04b7-420d-9c07-a21af5e9aac7","observedHarness":"Direct API","rawScore":36.23,"zScore":-2.029627,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:fc92f822-04b7-420d-9c07-a21af5e9aac7","observedHarness":"mini-SWE-agent","rawScore":0,"zScore":-1.263997,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"alibaba:qwen3.7-plus:thinking:standardized-suite:standardized","provider":"alibaba","modelSlug":"qwen3.7-plus","displayName":"Qwen3.7 Plus","effort":"thinking","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.558032,"shrinkageFactor":0.974358,"score":26.63,"se":4.43,"samplingSe":2.3929,"residualSe":3.7224,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-10","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","sourceId":"lmarena","sourceRowId":"lmarena:agent_capability:code:Qwen3.7-Plus","observedHarness":"LMArena Agent Harness","rawScore":955.33,"zScore":-1.973665,"seModel":"upstream_ci","ciHalfWidth":27.9508,"ciLower":927.3781,"ciUpper":983.2798,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://arena.ai/leaderboard/agent/code"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:a87fce42-eea3-4e45-a96d-fe057814f371","observedHarness":"Direct API","rawScore":46.06,"zScore":-0.756633,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:a87fce42-eea3-4e45-a96d-fe057814f371","observedHarness":"mini-SWE-agent","rawScore":1.01,"zScore":-1.219119,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-oss-120b:medium:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-oss-120b","displayName":"gpt-oss-120b","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.625311,"shrinkageFactor":0.664991,"score":25.62,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-08-05","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:rect7OMebdGAo6fqi","observedHarness":"ALE-Agent","rawScore":575.62,"zScore":-1.911659,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.001303,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"openai:gpt-oss-20b:medium:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-oss-20b","displayName":"gpt-oss-20b","effort":"medium","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.637892,"shrinkageFactor":0.664991,"score":25.43,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-08-05","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recnLZ6QnnjGuQZTM","observedHarness":"ALE-Agent","rawScore":566.05,"zScore":-1.930577,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.000717,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"google:gemma-4-26b-a4b:thinking:standardized-suite:standardized","provider":"google","modelSlug":"gemma-4-26b-a4b","displayName":"Gemma 4 26B A4B","effort":"thinking","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.639194,"shrinkageFactor":0.962336,"score":25.41,"se":4.94,"samplingSe":2.9001,"residualSe":4.004,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-01","paretoProb":0,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:6e1b44ff-c227-496b-aef4-19b70cd18c76","observedHarness":"Stirrup","rawScore":0,"zScore":-1.954663,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:6e1b44ff-c227-496b-aef4-19b70cd18c76","observedHarness":"Direct API","rawScore":40.05,"zScore":-1.534933,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:6e1b44ff-c227-496b-aef4-19b70cd18c76","observedHarness":"mini-SWE-agent","rawScore":0,"zScore":-1.263997,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"alibaba:qwen3.8-27b:none:standardized-suite:standardized","provider":"alibaba","modelSlug":"qwen3.8-27b","displayName":"Qwen3.8 27B","effort":"none","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.641641,"shrinkageFactor":0.961844,"score":25.38,"se":4.95,"samplingSe":2.919,"residualSe":4.002,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":3.772944,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-05","paretoProb":0,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:353c92f3-2148-4c2d-9231-aa7e1322a1fd","observedHarness":"Stirrup","rawScore":3.72,"zScore":-1.635345,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:353c92f3-2148-4c2d-9231-aa7e1322a1fd","observedHarness":"Direct API","rawScore":36.23,"zScore":-2.029627,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:353c92f3-2148-4c2d-9231-aa7e1322a1fd","observedHarness":"mini-SWE-agent","rawScore":0,"zScore":-1.263997,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"deepseek:deepseek-v4-pro:none:standardized-suite:standardized","provider":"deepseek","modelSlug":"deepseek-v4-pro","displayName":"DeepSeek V4 Pro (0424)","effort":"none","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.696235,"shrinkageFactor":0.664991,"score":24.56,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-04-24","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recgLipv7iMGBxsY3","observedHarness":"ALE-Agent","rawScore":521.67,"zScore":-2.018312,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.0166,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]},{"id":"meta:llama-4-maverick:none:standardized-suite:standardized","provider":"meta","modelSlug":"llama-4-maverick","displayName":"Llama 4 Maverick","effort":"none","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.722769,"shrinkageFactor":0.930761,"score":24.16,"se":6.29,"samplingSe":3.9322,"residualSe":4.9105,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-04-05","paretoProb":0,"benchmarks":[{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:922c69c7-9037-43c6-8bcf-a1c555e7f3eb","observedHarness":"Direct API","rawScore":31.71,"zScore":-2.614971,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:922c69c7-9037-43c6-8bcf-a1c555e7f3eb","observedHarness":"mini-SWE-agent","rawScore":0,"zScore":-1.263997,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"alibaba:qwen3.5-9b:thinking:standardized-suite:standardized","provider":"alibaba","modelSlug":"qwen3.5-9b","displayName":"Qwen3.5 9B","effort":"thinking","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.790556,"shrinkageFactor":0.932515,"score":23.14,"se":6.27,"samplingSe":3.882,"residualSe":4.9198,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-03-20","paretoProb":0,"benchmarks":[{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:2d28a13a-096e-475a-beb8-26bbd1c7d51c","observedHarness":"Direct API","rawScore":29.51,"zScore":-2.899873,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:2d28a13a-096e-475a-beb8-26bbd1c7d51c","observedHarness":"mini-SWE-agent","rawScore":0.51,"zScore":-1.241336,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-oss-120b:high:standardized-suite:standardized","provider":"openai","modelSlug":"gpt-oss-120b","displayName":"gpt-oss-120b","effort":"high","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.798097,"shrinkageFactor":0.964551,"score":23.03,"se":4.9,"samplingSe":2.8136,"residualSe":4.0132,"coverageCount":3,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":0.345342,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-08-05","paretoProb":0,"benchmarks":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:frontiercode_2_0:f0083258-8646-45b8-8082-7aaf6c2ea82a","observedHarness":"Stirrup","rawScore":0,"zScore":-1.954663,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/frontiercode-2-0"},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:f0083258-8646-45b8-8082-7aaf6c2ea82a","observedHarness":"Direct API","rawScore":34.03,"zScore":-2.314529,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:f0083258-8646-45b8-8082-7aaf6c2ea82a","observedHarness":"mini-SWE-agent","rawScore":0,"zScore":-1.263997,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"meta:llama-4-scout:none:standardized-suite:standardized","provider":"meta","modelSlug":"llama-4-scout","displayName":"Llama 4 Scout","effort":"none","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.853327,"shrinkageFactor":0.944607,"score":22.2,"se":6.1,"samplingSe":3.5171,"residualSe":4.9836,"coverageCount":2,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-04-05","paretoProb":0,"benchmarks":[{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:scicode_aa:adf9a85e-abc3-4f28-937b-db6655cc5238","observedHarness":"Direct API","rawScore":21.3,"zScore":-3,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_4_0:adf9a85e-abc3-4f28-937b-db6655cc5238","observedHarness":"mini-SWE-agent","rawScore":0,"zScore":-1.263997,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"deepseek:deepseek-v4-flash:none:standardized-suite:standardized","provider":"deepseek","modelSlug":"deepseek-v4-flash","displayName":"DeepSeek V4 Flash","effort":"none","harness":"standardized-suite","harnessRegime":"standardized","domainId":"coding","theta":-1.954807,"shrinkageFactor":0.664991,"score":20.68,"se":9.86,"samplingSe":8.6494,"residualSe":4.7321,"coverageCount":1,"activeBenchmarkCount":12,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-31","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_ale_bench:recWsgZQjBL5E6mbZ","observedHarness":"ALE-Agent","rawScore":324.98,"zScore":-2.407148,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.000882,"evalDate":null,"publishedDate":null,"sourceUrl":"https://github.com/SakanaAI/ALE-Bench"}]}],"frontiers":[{"provider":"anthropic","harnessRegime":"standardized","harnessFilter":"all","frontier":[{"configId":"anthropic:claude-haiku-5.5:low:standardized-suite:standardized","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9993,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"anthropic:claude-haiku-5.5:medium:standardized-suite:standardized","zStepFromPrev":0.773,"pBetterThanPrev":0.7801,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.7748,"marginalScorePerDollar":93.945,"costMultipleFromPrev":2.1},{"configId":"anthropic:claude-haiku-5.5:high:standardized-suite:standardized","zStepFromPrev":1.199,"pBetterThanPrev":0.8848,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.8586,"marginalScorePerDollar":100.906,"costMultipleFromPrev":1.73},{"configId":"anthropic:claude-haiku-5.5:xhigh:standardized-suite:standardized","zStepFromPrev":0.779,"pBetterThanPrev":0.7821,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.7508,"marginalScorePerDollar":49.554,"costMultipleFromPrev":1.57},{"configId":"anthropic:claude-haiku-5.5:max:standardized-suite:standardized","zStepFromPrev":0.337,"pBetterThanPrev":0.6319,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.5764,"marginalScorePerDollar":11.128,"costMultipleFromPrev":1.65},{"configId":"anthropic:claude-sonnet-5.5:high:standardized-suite:standardized","zStepFromPrev":0.941,"pBetterThanPrev":0.8265,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.4991,"marginalScorePerDollar":3.436,"costMultipleFromPrev":3.98},{"configId":"anthropic:claude-opus-5.5:medium:standardized-suite:standardized","zStepFromPrev":0.692,"pBetterThanPrev":0.7554,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.5179,"marginalScorePerDollar":5.154,"costMultipleFromPrev":1.5},{"configId":"anthropic:claude-fable-5.2:low:standardized-suite:standardized","zStepFromPrev":0.355,"pBetterThanPrev":0.6387,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.4309,"marginalScorePerDollar":3.776,"costMultipleFromPrev":1.26},{"configId":"anthropic:claude-opus-5.5:high:standardized-suite:standardized","zStepFromPrev":0.838,"pBetterThanPrev":0.7991,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.6045,"marginalScorePerDollar":13.527,"costMultipleFromPrev":1.12},{"configId":"anthropic:claude-opus-5.5:xhigh:standardized-suite:standardized","zStepFromPrev":0.179,"pBetterThanPrev":0.5711,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.3581,"marginalScorePerDollar":0.391,"costMultipleFromPrev":1.75},{"configId":"anthropic:claude-fable-5.2:high:standardized-suite:standardized","zStepFromPrev":0.036,"pBetterThanPrev":0.5143,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.2583,"marginalScorePerDollar":0.079,"costMultipleFromPrev":1.5},{"configId":"anthropic:claude-sonnet-5.5:max:standardized-suite:standardized","zStepFromPrev":1.494,"pBetterThanPrev":0.9324,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.7871,"marginalScorePerDollar":8.289,"costMultipleFromPrev":1.1},{"configId":"anthropic:claude-opus-5.5:max:standardized-suite:standardized","zStepFromPrev":0.512,"pBetterThanPrev":0.6957,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.6439,"marginalScorePerDollar":116.537,"costMultipleFromPrev":1},{"configId":"anthropic:claude-fable-5.2:max:standardized-suite:standardized","zStepFromPrev":1.568,"pBetterThanPrev":0.9415,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9152,"marginalScorePerDollar":0.396,"costMultipleFromPrev":2.64}],"pruned":[{"configId":"anthropic:claude-sonnet-5.5:low:standardized-suite:standardized","dominatedById":"anthropic:claude-haiku-5.5:max:standardized-suite:standardized","scoreDeficit":14.91,"costRatio":1.24,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5.5:medium:standardized-suite:standardized","dominatedById":"anthropic:claude-haiku-5.5:max:standardized-suite:standardized","scoreDeficit":9.65,"costRatio":1.9,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5.5:low:standardized-suite:standardized","dominatedById":"anthropic:claude-haiku-5.5:max:standardized-suite:standardized","scoreDeficit":4.11,"costRatio":2.37,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-4.6:medium:standardized-suite:standardized","dominatedById":"anthropic:claude-opus-5.5:medium:standardized-suite:standardized","scoreDeficit":20.74,"costRatio":1.16,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.8:none:standardized-suite:standardized","dominatedById":"anthropic:claude-opus-5.5:medium:standardized-suite:standardized","scoreDeficit":19.07,"costRatio":1.21,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.7:none:standardized-suite:standardized","dominatedById":"anthropic:claude-opus-5.5:medium:standardized-suite:standardized","scoreDeficit":20.82,"costRatio":1.21,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.6:none:standardized-suite:standardized","dominatedById":"anthropic:claude-opus-5.5:medium:standardized-suite:standardized","scoreDeficit":27.26,"costRatio":1.21,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5:high:standardized-suite:standardized","dominatedById":"anthropic:claude-opus-5.5:high:standardized-suite:standardized","scoreDeficit":23.03,"costRatio":1.07,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.2:medium:standardized-suite:standardized","dominatedById":"anthropic:claude-opus-5.5:high:standardized-suite:standardized","scoreDeficit":3.15,"costRatio":1.1,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5.5:xhigh:standardized-suite:standardized","dominatedById":"anthropic:claude-opus-5.5:high:standardized-suite:standardized","scoreDeficit":6.12,"costRatio":1.18,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5:max:standardized-suite:standardized","dominatedById":"anthropic:claude-opus-5.5:xhigh:standardized-suite:standardized","scoreDeficit":27.4,"costRatio":1.07,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5:high:standardized-suite:standardized","dominatedById":"anthropic:claude-opus-5.5:xhigh:standardized-suite:standardized","scoreDeficit":12.65,"costRatio":1.39,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.8:high:standardized-suite:standardized","dominatedById":"anthropic:claude-opus-5.5:xhigh:standardized-suite:standardized","scoreDeficit":24.49,"costRatio":1.39,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.7:high:standardized-suite:standardized","dominatedById":"anthropic:claude-opus-5.5:xhigh:standardized-suite:standardized","scoreDeficit":32.66,"costRatio":1.39,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5:max:standardized-suite:standardized","dominatedById":"anthropic:claude-opus-5.5:max:standardized-suite:standardized","scoreDeficit":16.69,"costRatio":1.48,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.8:max:standardized-suite:standardized","dominatedById":"anthropic:claude-opus-5.5:max:standardized-suite:standardized","scoreDeficit":20.7,"costRatio":1.48,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.7:max:standardized-suite:standardized","dominatedById":"anthropic:claude-opus-5.5:max:standardized-suite:standardized","scoreDeficit":34.36,"costRatio":1.48,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.6:max:standardized-suite:standardized","dominatedById":"anthropic:claude-opus-5.5:max:standardized-suite:standardized","scoreDeficit":40.74,"costRatio":1.48,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.2:xhigh:standardized-suite:standardized","dominatedById":"anthropic:claude-opus-5.5:max:standardized-suite:standardized","scoreDeficit":4.97,"costRatio":1.48,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.1:high:standardized-suite:standardized","dominatedById":"anthropic:claude-opus-5.5:max:standardized-suite:standardized","scoreDeficit":18.39,"costRatio":1.57,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5:high:standardized-suite:standardized","dominatedById":"anthropic:claude-opus-5.5:max:standardized-suite:standardized","scoreDeficit":21.51,"costRatio":1.57,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.1:max:standardized-suite:standardized","dominatedById":"anthropic:claude-fable-5.2:max:standardized-suite:standardized","scoreDeficit":13.34,"costRatio":1.05,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5:max:standardized-suite:standardized","dominatedById":"anthropic:claude-fable-5.2:max:standardized-suite:standardized","scoreDeficit":22.99,"costRatio":1.05,"costComparisonNote":null}],"verdicts":{"budgetId":"anthropic:claude-haiku-5.5:low:standardized-suite:standardized","defaultId":"anthropic:claude-opus-5.5:max:standardized-suite:standardized","maxId":"anthropic:claude-fable-5.2:max:standardized-suite:standardized"}},{"provider":"openai","harnessRegime":"standardized","harnessFilter":"all","frontier":[{"configId":"openai:gpt-6-luna:low:standardized-suite:standardized","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"openai:gpt-6-luna:medium:standardized-suite:standardized","zStepFromPrev":1.632,"pBetterThanPrev":0.9487,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.934,"marginalScorePerDollar":180.926,"costMultipleFromPrev":8.25},{"configId":"openai:gpt-6.1-terra:low:standardized-suite:standardized","zStepFromPrev":1.065,"pBetterThanPrev":0.8566,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.8324,"marginalScorePerDollar":231.672,"costMultipleFromPrev":1.69},{"configId":"openai:gpt-6.1-terra:medium:standardized-suite:standardized","zStepFromPrev":1.808,"pBetterThanPrev":0.9647,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9038,"marginalScorePerDollar":111.422,"costMultipleFromPrev":2.37},{"configId":"openai:gpt-6.2-sol:low:standardized-suite:standardized","zStepFromPrev":0.046,"pBetterThanPrev":0.5183,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.4873,"marginalScorePerDollar":20.974,"costMultipleFromPrev":1.09},{"configId":"openai:gpt-6.1-terra:high:standardized-suite:standardized","zStepFromPrev":0.186,"pBetterThanPrev":0.5738,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.3879,"marginalScorePerDollar":18.759,"costMultipleFromPrev":1.41},{"configId":"openai:gpt-6.1-terra:xhigh:standardized-suite:standardized","zStepFromPrev":0.271,"pBetterThanPrev":0.6066,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.3588,"marginalScorePerDollar":12.729,"costMultipleFromPrev":1.55},{"configId":"openai:gpt-6.2-sol:medium:standardized-suite:standardized","zStepFromPrev":1.121,"pBetterThanPrev":0.8688,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.7508,"marginalScorePerDollar":1602.385,"costMultipleFromPrev":1.01},{"configId":"openai:gpt-6.2-sol:high:standardized-suite:standardized","zStepFromPrev":0.066,"pBetterThanPrev":0.5262,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.4369,"marginalScorePerDollar":2.729,"costMultipleFromPrev":1.39},{"configId":"openai:gpt-6.2-sol:xhigh:standardized-suite:standardized","zStepFromPrev":0.034,"pBetterThanPrev":0.5134,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.3145,"marginalScorePerDollar":2.609,"costMultipleFromPrev":1.15},{"configId":"openai:gpt-6.1-terra:max:standardized-suite:standardized","zStepFromPrev":0.066,"pBetterThanPrev":0.5262,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.2154,"marginalScorePerDollar":1.373,"costMultipleFromPrev":1.48},{"configId":"openai:gpt-6.2-sol:max:standardized-suite:standardized","zStepFromPrev":0.731,"pBetterThanPrev":0.7675,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.3959,"marginalScorePerDollar":65.701,"costMultipleFromPrev":1.06},{"configId":"openai:gpt-6.2-astra:xhigh:standardized-suite:standardized","zStepFromPrev":0.333,"pBetterThanPrev":0.6305,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.3187,"marginalScorePerDollar":1.467,"costMultipleFromPrev":2.49},{"configId":"openai:gpt-6.2-astra:max:standardized-suite:standardized","zStepFromPrev":0.194,"pBetterThanPrev":0.5771,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.2804,"marginalScorePerDollar":0.833,"costMultipleFromPrev":1.59},{"configId":"openai:gpt-6-astra:max:standardized-suite:standardized","zStepFromPrev":0.442,"pBetterThanPrev":0.6706,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.3696,"marginalScorePerDollar":0.076,"costMultipleFromPrev":6.96}],"pruned":[{"configId":"openai:gpt-6-luna:none:standardized-suite:standardized","dominatedById":"openai:gpt-6.1-terra:low:standardized-suite:standardized","scoreDeficit":16.65,"costRatio":1,"costComparisonNote":null},{"configId":"openai:gpt-6-luna:high:standardized-suite:standardized","dominatedById":"openai:gpt-6.1-terra:low:standardized-suite:standardized","scoreDeficit":6.29,"costRatio":1.02,"costComparisonNote":null},{"configId":"openai:gpt-6-luna:xhigh:standardized-suite:standardized","dominatedById":"openai:gpt-6.1-terra:low:standardized-suite:standardized","scoreDeficit":3.1,"costRatio":1.46,"costComparisonNote":null},{"configId":"openai:gpt-6-luna:max:standardized-suite:standardized","dominatedById":"openai:gpt-6.1-terra:low:standardized-suite:standardized","scoreDeficit":2.05,"costRatio":2.13,"costComparisonNote":null},{"configId":"openai:gpt-6.1-terra:none:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-sol:low:standardized-suite:standardized","scoreDeficit":6.09,"costRatio":1.23,"costComparisonNote":null},{"configId":"openai:gpt-5.4-nano:high:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-sol:high:standardized-suite:standardized","scoreDeficit":33.69,"costRatio":1.02,"costComparisonNote":null},{"configId":"openai:gpt-6.2-astra:low:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-sol:xhigh:standardized-suite:standardized","scoreDeficit":6.78,"costRatio":1.02,"costComparisonNote":null},{"configId":"openai:gpt-5.6-luna:max:standardized-suite:standardized","dominatedById":"openai:gpt-6.1-terra:max:standardized-suite:standardized","scoreDeficit":29.65,"costRatio":1.01,"costComparisonNote":null},{"configId":"openai:gpt-5:minimal:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-sol:max:standardized-suite:standardized","scoreDeficit":41.72,"costRatio":1.26,"costComparisonNote":null},{"configId":"openai:gpt-6.2-astra:medium:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-sol:max:standardized-suite:standardized","scoreDeficit":1.12,"costRatio":1.59,"costComparisonNote":null},{"configId":"openai:gpt-5.4-mini:high:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-sol:max:standardized-suite:standardized","scoreDeficit":34.21,"costRatio":1.77,"costComparisonNote":null},{"configId":"openai:gpt-5.4:none:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-sol:max:standardized-suite:standardized","scoreDeficit":36.23,"costRatio":1.83,"costComparisonNote":null},{"configId":"openai:gpt-6.1-sol:medium:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-sol:max:standardized-suite:standardized","scoreDeficit":7.94,"costRatio":1.94,"costComparisonNote":null},{"configId":"openai:gpt-6.2-astra:high:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-sol:max:standardized-suite:standardized","scoreDeficit":0.25,"costRatio":2.04,"costComparisonNote":null},{"configId":"openai:gpt-5.2:medium:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:xhigh:standardized-suite:standardized","scoreDeficit":35.21,"costRatio":1.03,"costComparisonNote":null},{"configId":"openai:gpt-5.4:medium:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:xhigh:standardized-suite:standardized","scoreDeficit":29.87,"costRatio":1.11,"costComparisonNote":null},{"configId":"openai:o3:high:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:xhigh:standardized-suite:standardized","scoreDeficit":41.45,"costRatio":1.22,"costComparisonNote":null},{"configId":"openai:gpt-5.5:none:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:xhigh:standardized-suite:standardized","scoreDeficit":37.63,"costRatio":1.38,"costComparisonNote":null},{"configId":"openai:gpt-5.1:high:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:xhigh:standardized-suite:standardized","scoreDeficit":36.35,"costRatio":1.44,"costComparisonNote":null},{"configId":"openai:gpt-5:high:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:xhigh:standardized-suite:standardized","scoreDeficit":44.84,"costRatio":1.44,"costComparisonNote":null},{"configId":"openai:gpt-5.6-sol:medium:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:xhigh:standardized-suite:standardized","scoreDeficit":36.87,"costRatio":1.45,"costComparisonNote":null},{"configId":"openai:gpt-5.2:high:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:max:standardized-suite:standardized","scoreDeficit":35.58,"costRatio":1.23,"costComparisonNote":null},{"configId":"openai:gpt-5.5:medium:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:max:standardized-suite:standardized","scoreDeficit":29.75,"costRatio":1.31,"costComparisonNote":null},{"configId":"openai:gpt-5.4:high:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:max:standardized-suite:standardized","scoreDeficit":36.24,"costRatio":1.32,"costComparisonNote":null},{"configId":"openai:gpt-5.6-terra:xhigh:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:max:standardized-suite:standardized","scoreDeficit":35.4,"costRatio":1.32,"costComparisonNote":null},{"configId":"openai:gpt-5.3-codex:xhigh:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:max:standardized-suite:standardized","scoreDeficit":28.45,"costRatio":1.5,"costComparisonNote":null},{"configId":"openai:gpt-5.2:xhigh:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:max:standardized-suite:standardized","scoreDeficit":37.34,"costRatio":1.5,"costComparisonNote":null},{"configId":"openai:gpt-5.4:xhigh:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:max:standardized-suite:standardized","scoreDeficit":44.21,"costRatio":1.62,"costComparisonNote":null},{"configId":"openai:gpt-6.1-sol:max:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:max:standardized-suite:standardized","scoreDeficit":2.15,"costRatio":1.62,"costComparisonNote":null},{"configId":"openai:gpt-6-sol:max:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:max:standardized-suite:standardized","scoreDeficit":11.39,"costRatio":1.62,"costComparisonNote":null},{"configId":"openai:gpt-5.6-sol:high:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:max:standardized-suite:standardized","scoreDeficit":41.26,"costRatio":1.72,"costComparisonNote":null},{"configId":"openai:gpt-5.6-terra:max:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:max:standardized-suite:standardized","scoreDeficit":28.74,"costRatio":1.89,"costComparisonNote":null},{"configId":"openai:gpt-5.6-sol:xhigh:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:max:standardized-suite:standardized","scoreDeficit":25.18,"costRatio":2.12,"costComparisonNote":null},{"configId":"openai:gpt-5.5:high:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:max:standardized-suite:standardized","scoreDeficit":55.52,"costRatio":2.46,"costComparisonNote":null},{"configId":"openai:gpt-5.5:xhigh:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:max:standardized-suite:standardized","scoreDeficit":32.57,"costRatio":3.02,"costComparisonNote":null},{"configId":"openai:gpt-5.6-sol:max:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:max:standardized-suite:standardized","scoreDeficit":15.3,"costRatio":3.03,"costComparisonNote":null},{"configId":"openai:gpt-6-astra:high:standardized-suite:standardized","dominatedById":"openai:gpt-6.2-astra:max:standardized-suite:standardized","scoreDeficit":28.22,"costRatio":3.95,"costComparisonNote":null}],"verdicts":{"budgetId":"openai:gpt-6-luna:low:standardized-suite:standardized","defaultId":"openai:gpt-6.2-sol:medium:standardized-suite:standardized","maxId":"openai:gpt-6-astra:max:standardized-suite:standardized"}},{"provider":"google","harnessRegime":"standardized","harnessFilter":"all","frontier":[{"configId":"google:gemini-4-hydrogen:low:standardized-suite:standardized","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.5677,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"google:gemini-4-neon:low:standardized-suite:standardized","zStepFromPrev":3.031,"pBetterThanPrev":0.9988,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.8727,"marginalScorePerDollar":45.651,"costMultipleFromPrev":2.55},{"configId":"google:gemini-4-hydrogen:medium:standardized-suite:standardized","zStepFromPrev":0.129,"pBetterThanPrev":0.5513,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.5157,"marginalScorePerDollar":17.059,"costMultipleFromPrev":1.07},{"configId":"google:gemini-4-hydrogen:high:standardized-suite:standardized","zStepFromPrev":2.171,"pBetterThanPrev":0.985,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9548,"marginalScorePerDollar":11.531,"costMultipleFromPrev":2.35},{"configId":"google:gemini-4-neon:medium:standardized-suite:standardized","zStepFromPrev":0.062,"pBetterThanPrev":0.5246,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.5186,"marginalScorePerDollar":12.163,"costMultipleFromPrev":1.02},{"configId":"google:gemini-4-neon:high:standardized-suite:standardized","zStepFromPrev":1.605,"pBetterThanPrev":0.9458,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.908,"marginalScorePerDollar":5.19,"costMultipleFromPrev":1.97},{"configId":"google:gemini-4-neon:xhigh:standardized-suite:standardized","zStepFromPrev":0.129,"pBetterThanPrev":0.5513,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.545,"marginalScorePerDollar":0.266,"costMultipleFromPrev":1.72}],"pruned":[{"configId":"google:gemini-3.5-flash-lite:minimal:standardized-suite:standardized","dominatedById":"google:gemini-4-hydrogen:low:standardized-suite:standardized","scoreDeficit":2.67,"costRatio":2.37,"costComparisonNote":null},{"configId":"google:gemini-3-flash:high:standardized-suite:standardized","dominatedById":"google:gemini-4-hydrogen:medium:standardized-suite:standardized","scoreDeficit":11.62,"costRatio":1.57,"costComparisonNote":null},{"configId":"google:gemini-3.1-pro:low:standardized-suite:standardized","dominatedById":"google:gemini-4-hydrogen:medium:standardized-suite:standardized","scoreDeficit":17.78,"costRatio":1.91,"costComparisonNote":null},{"configId":"google:gemini-3.8-flash:high:standardized-suite:standardized","dominatedById":"google:gemini-4-hydrogen:medium:standardized-suite:standardized","scoreDeficit":12.9,"costRatio":1.93,"costComparisonNote":null},{"configId":"google:gemini-3.7-flash:high:standardized-suite:standardized","dominatedById":"google:gemini-4-hydrogen:medium:standardized-suite:standardized","scoreDeficit":17.47,"costRatio":1.93,"costComparisonNote":null},{"configId":"google:gemini-3.6-flash:high:standardized-suite:standardized","dominatedById":"google:gemini-4-hydrogen:medium:standardized-suite:standardized","scoreDeficit":25.65,"costRatio":1.93,"costComparisonNote":null},{"configId":"google:gemini-3.5-flash-lite:high:standardized-suite:standardized","dominatedById":"google:gemini-4-neon:medium:standardized-suite:standardized","scoreDeficit":36.27,"costRatio":1.33,"costComparisonNote":null},{"configId":"google:gemini-4-hydrogen:xhigh:standardized-suite:standardized","dominatedById":"google:gemini-4-neon:medium:standardized-suite:standardized","scoreDeficit":0.51,"costRatio":1.66,"costComparisonNote":null},{"configId":"google:gemini-3.5-flash:high:standardized-suite:standardized","dominatedById":"google:gemini-4-neon:medium:standardized-suite:standardized","scoreDeficit":33.4,"costRatio":1.75,"costComparisonNote":null},{"configId":"google:gemini-3.1-pro:high:standardized-suite:standardized","dominatedById":"google:gemini-4-neon:high:standardized-suite:standardized","scoreDeficit":43.01,"costRatio":1.15,"costComparisonNote":null},{"configId":"google:gemini-4-argon:low:standardized-suite:standardized","dominatedById":"google:gemini-4-neon:high:standardized-suite:standardized","scoreDeficit":24.49,"costRatio":1.17,"costComparisonNote":null},{"configId":"google:gemini-4-carbon:medium:standardized-suite:standardized","dominatedById":"google:gemini-4-neon:xhigh:standardized-suite:standardized","scoreDeficit":6.73,"costRatio":1.02,"costComparisonNote":null},{"configId":"google:gemini-4-carbon:low:standardized-suite:standardized","dominatedById":"google:gemini-4-neon:xhigh:standardized-suite:standardized","scoreDeficit":11.25,"costRatio":1.12,"costComparisonNote":null},{"configId":"google:gemini-4-carbon:high:standardized-suite:standardized","dominatedById":"google:gemini-4-neon:xhigh:standardized-suite:standardized","scoreDeficit":4.24,"costRatio":1.2,"costComparisonNote":null},{"configId":"google:gemini-4-argon:medium:standardized-suite:standardized","dominatedById":"google:gemini-4-neon:xhigh:standardized-suite:standardized","scoreDeficit":15.41,"costRatio":1.34,"costComparisonNote":null},{"configId":"google:gemini-4-argon:high:standardized-suite:standardized","dominatedById":"google:gemini-4-neon:xhigh:standardized-suite:standardized","scoreDeficit":11.98,"costRatio":1.54,"costComparisonNote":null}],"verdicts":{"budgetId":"google:gemini-4-hydrogen:low:standardized-suite:standardized","defaultId":"google:gemini-4-neon:medium:standardized-suite:standardized","maxId":"google:gemini-4-neon:xhigh:standardized-suite:standardized"}},{"provider":"spacexai","harnessRegime":"standardized","harnessFilter":"all","frontier":[{"configId":"spacexai:grok-4.20:none:standardized-suite:standardized","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.6549,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"spacexai:grok-4.6:low:standardized-suite:standardized","zStepFromPrev":0.26,"pBetterThanPrev":0.6024,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.4948,"marginalScorePerDollar":8.315,"costMultipleFromPrev":2.09},{"configId":"spacexai:grok-4.6:medium:standardized-suite:standardized","zStepFromPrev":0.368,"pBetterThanPrev":0.6437,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.5368,"marginalScorePerDollar":11.047,"costMultipleFromPrev":1.51},{"configId":"spacexai:grok-4.6:high:standardized-suite:standardized","zStepFromPrev":0.16,"pBetterThanPrev":0.5635,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.4806,"marginalScorePerDollar":1.847,"costMultipleFromPrev":1.88},{"configId":"spacexai:grok-4.7:medium:standardized-suite:standardized","zStepFromPrev":0.077,"pBetterThanPrev":0.5306,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.3349,"marginalScorePerDollar":0.586,"costMultipleFromPrev":1.74},{"configId":"spacexai:grok-4.7:high:standardized-suite:standardized","zStepFromPrev":0.235,"pBetterThanPrev":0.593,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.2733,"marginalScorePerDollar":1.321,"costMultipleFromPrev":1.46}],"pruned":[{"configId":"spacexai:grok-4.5:high:standardized-suite:standardized","dominatedById":"spacexai:grok-4.6:high:standardized-suite:standardized","scoreDeficit":10.95,"costRatio":1,"costComparisonNote":null},{"configId":"spacexai:grok-4.7:low:standardized-suite:standardized","dominatedById":"spacexai:grok-4.6:high:standardized-suite:standardized","scoreDeficit":4.51,"costRatio":1.2,"costComparisonNote":null},{"configId":"spacexai:grok-4.6:xhigh:standardized-suite:standardized","dominatedById":"spacexai:grok-4.6:high:standardized-suite:standardized","scoreDeficit":4.34,"costRatio":1.22,"costComparisonNote":null},{"configId":"spacexai:grok-4.7:xhigh:standardized-suite:standardized","dominatedById":"spacexai:grok-4.7:high:standardized-suite:standardized","scoreDeficit":2.7,"costRatio":1.26,"costComparisonNote":null}],"verdicts":{"budgetId":"spacexai:grok-4.20:none:standardized-suite:standardized","defaultId":null,"maxId":"spacexai:grok-4.7:high:standardized-suite:standardized"}},{"provider":"meta","harnessRegime":"standardized","harnessFilter":"all","frontier":[{"configId":"meta:muse-ember:xhigh:standardized-suite:standardized","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.962,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"meta:muse-ember:max:standardized-suite:standardized","zStepFromPrev":0.06,"pBetterThanPrev":0.5239,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.5031,"marginalScorePerDollar":18.774,"costMultipleFromPrev":1.17},{"configId":"meta:muse-wonder:high:standardized-suite:standardized","zStepFromPrev":0.45,"pBetterThanPrev":0.6737,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.4202,"marginalScorePerDollar":0.689,"costMultipleFromPrev":33.55},{"configId":"meta:muse-wonder:max:standardized-suite:standardized","zStepFromPrev":0.479,"pBetterThanPrev":0.6839,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.4666,"marginalScorePerDollar":1.196,"costMultipleFromPrev":1.52}],"pruned":[{"configId":"meta:muse-spark-1.3:low:standardized-suite:standardized","dominatedById":"meta:muse-ember:max:standardized-suite:standardized","scoreDeficit":14.01,"costRatio":4.58,"costComparisonNote":null},{"configId":"meta:muse-spark-1.3:medium:standardized-suite:standardized","dominatedById":"meta:muse-ember:max:standardized-suite:standardized","scoreDeficit":11.15,"costRatio":6.93,"costComparisonNote":null},{"configId":"meta:muse-spark-1.3:high:standardized-suite:standardized","dominatedById":"meta:muse-ember:max:standardized-suite:standardized","scoreDeficit":10.52,"costRatio":13.01,"costComparisonNote":null},{"configId":"meta:muse-spark-1.1:high:standardized-suite:standardized","dominatedById":"meta:muse-ember:max:standardized-suite:standardized","scoreDeficit":28.66,"costRatio":13.01,"costComparisonNote":null},{"configId":"meta:muse-spark-1.3:xhigh:standardized-suite:standardized","dominatedById":"meta:muse-ember:max:standardized-suite:standardized","scoreDeficit":4.91,"costRatio":15.93,"costComparisonNote":null},{"configId":"meta:muse-spark-1.2:xhigh:standardized-suite:standardized","dominatedById":"meta:muse-ember:max:standardized-suite:standardized","scoreDeficit":21.26,"costRatio":15.93,"costComparisonNote":null},{"configId":"meta:muse-spark-1.3:max:standardized-suite:standardized","dominatedById":"meta:muse-ember:max:standardized-suite:standardized","scoreDeficit":3.43,"costRatio":22.75,"costComparisonNote":null}],"verdicts":{"budgetId":"meta:muse-ember:xhigh:standardized-suite:standardized","defaultId":null,"maxId":"meta:muse-wonder:max:standardized-suite:standardized"}},{"provider":"deepseek","harnessRegime":"standardized","harnessFilter":"all","frontier":[{"configId":"deepseek:deepseek-v4.1-flash-1017:none:standardized-suite:standardized","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"deepseek:deepseek-v4.1-flash-1017:high:standardized-suite:standardized","zStepFromPrev":2.39,"pBetterThanPrev":0.9916,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9926,"marginalScorePerDollar":482.875,"costMultipleFromPrev":1.07},{"configId":"deepseek:deepseek-v4.1-flash-1017:max:standardized-suite:standardized","zStepFromPrev":1.319,"pBetterThanPrev":0.9063,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9094,"marginalScorePerDollar":53.8,"costMultipleFromPrev":1.32}],"pruned":[{"configId":"deepseek:deepseek-v4-pro-0813:high:standardized-suite:standardized","dominatedById":"deepseek:deepseek-v4.1-flash-1017:max:standardized-suite:standardized","scoreDeficit":14.5,"costRatio":1.41,"costComparisonNote":null},{"configId":"deepseek:deepseek-v4-pro-0813:none:standardized-suite:standardized","dominatedById":"deepseek:deepseek-v4.1-flash-1017:max:standardized-suite:standardized","scoreDeficit":22.92,"costRatio":1.62,"costComparisonNote":null},{"configId":"deepseek:deepseek-v4-pro-0813:max:standardized-suite:standardized","dominatedById":"deepseek:deepseek-v4.1-flash-1017:max:standardized-suite:standardized","scoreDeficit":11.64,"costRatio":2.94,"costComparisonNote":null}],"verdicts":{"budgetId":"deepseek:deepseek-v4.1-flash-1017:none:standardized-suite:standardized","defaultId":"deepseek:deepseek-v4.1-flash-1017:high:standardized-suite:standardized","maxId":"deepseek:deepseek-v4.1-flash-1017:max:standardized-suite:standardized"}},{"provider":"moonshot","harnessRegime":"standardized","harnessFilter":"all","frontier":[{"configId":"moonshot:kimi-k2.6:thinking:standardized-suite:standardized","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.8372,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"moonshot:kimi-k3.1-pro:max:standardized-suite:standardized","zStepFromPrev":2.13,"pBetterThanPrev":0.9834,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9846,"marginalScorePerDollar":8.638,"costMultipleFromPrev":3.12}],"pruned":[{"configId":"moonshot:kimi-k2.7-code:max:standardized-suite:standardized","dominatedById":"moonshot:kimi-k2.6:thinking:standardized-suite:standardized","scoreDeficit":4.68,"costRatio":2.44,"costComparisonNote":null},{"configId":"moonshot:kimi-k3:max:standardized-suite:standardized","dominatedById":"moonshot:kimi-k3.1-pro:max:standardized-suite:standardized","scoreDeficit":16.64,"costRatio":2.54,"costComparisonNote":null}],"verdicts":{"budgetId":"moonshot:kimi-k2.6:thinking:standardized-suite:standardized","defaultId":null,"maxId":"moonshot:kimi-k3.1-pro:max:standardized-suite:standardized"}},{"provider":"alibaba","harnessRegime":"standardized","harnessFilter":"all","frontier":[{"configId":"alibaba:qwen3.7-max:thinking:standardized-suite:standardized","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.5654,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"alibaba:qwen3.8-max-0902:xhigh:standardized-suite:standardized","zStepFromPrev":2.506,"pBetterThanPrev":0.9939,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.7533,"marginalScorePerDollar":21.639,"costMultipleFromPrev":1.4},{"configId":"alibaba:qwen4-max:xhigh:standardized-suite:standardized","zStepFromPrev":1.255,"pBetterThanPrev":0.8952,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9207,"marginalScorePerDollar":24.57,"costMultipleFromPrev":1.1}],"pruned":[],"verdicts":{"budgetId":"alibaba:qwen3.7-max:thinking:standardized-suite:standardized","defaultId":null,"maxId":"alibaba:qwen4-max:xhigh:standardized-suite:standardized"}},{"provider":"zhipu","harnessRegime":"standardized","harnessFilter":"all","frontier":[{"configId":"zhipu:glm-5.3-flash:high:standardized-suite:standardized","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.626,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"zhipu:glm-5.3-flash:max:standardized-suite:standardized","zStepFromPrev":2.103,"pBetterThanPrev":0.9823,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.7894,"marginalScorePerDollar":75.465,"costMultipleFromPrev":1.95},{"configId":"zhipu:glm-5.3:low:standardized-suite:standardized","zStepFromPrev":0.035,"pBetterThanPrev":0.5141,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.3988,"marginalScorePerDollar":0.202,"costMultipleFromPrev":2.7},{"configId":"zhipu:glm-5.3:max:standardized-suite:standardized","zStepFromPrev":1.115,"pBetterThanPrev":0.8676,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.5712,"marginalScorePerDollar":1.799,"costMultipleFromPrev":3.18}],"pruned":[{"configId":"zhipu:glm-5:thinking:standardized-suite:standardized","dominatedById":"zhipu:glm-5.3-flash:max:standardized-suite:standardized","scoreDeficit":12.36,"costRatio":1.7,"costComparisonNote":null},{"configId":"zhipu:glm-5.1:thinking:standardized-suite:standardized","dominatedById":"zhipu:glm-5.3-flash:max:standardized-suite:standardized","scoreDeficit":9.97,"costRatio":2.25,"costComparisonNote":null},{"configId":"zhipu:glm-5.2:high:standardized-suite:standardized","dominatedById":"zhipu:glm-5.3:low:standardized-suite:standardized","scoreDeficit":7.02,"costRatio":1.17,"costComparisonNote":null},{"configId":"zhipu:glm-5.3:high:standardized-suite:standardized","dominatedById":"zhipu:glm-5.3:low:standardized-suite:standardized","scoreDeficit":1.68,"costRatio":2.01,"costComparisonNote":null},{"configId":"zhipu:glm-5.2:max:standardized-suite:standardized","dominatedById":"zhipu:glm-5.3:low:standardized-suite:standardized","scoreDeficit":3.49,"costRatio":2.04,"costComparisonNote":null}],"verdicts":{"budgetId":"zhipu:glm-5.3-flash:high:standardized-suite:standardized","defaultId":"zhipu:glm-5.3-flash:max:standardized-suite:standardized","maxId":"zhipu:glm-5.3:max:standardized-suite:standardized"}}]},"writing":{"domainId":"writing","epochVersion":"writing-v2.0.0","costUnit":"usd_per_task","activeBenchmarkIds":["lmarena_creative_writing","lechmazur_writing","eqbench_creative_writing","eqbench_longform_writing","surge_complex_constraints"],"candidateBenchmarkIds":[],"benchmarkVersions":{"lmarena_creative_writing":"text_style_control","lechmazur_writing":"2026.11","eqbench_creative_writing":"3.0","eqbench_longform_writing":"3.0","surge_complex_constraints":"1.1"},"coverageFloor":1,"tauResid":0.5568,"regimeMeta":{"native":{"activeBenchmarkIds":["lmarena_creative_writing","lechmazur_writing","eqbench_creative_writing","eqbench_longform_writing","surge_complex_constraints"],"costSource":"list","measuredCostBenchmarkIds":[],"coverageFloor":1,"tauResid":0.5568,"calibrationTauResidSq":0.31001825,"priorMean":0.17189,"tauModelSq":0.796141,"anchorLabs":9,"withheldBenchmarkIds":[],"linkJointCov":{"benchmarkIds":["surge_complex_constraints"],"covMatrix":[[3.2749981119460676,-0.04838158460639],[-0.04838158460639,0.0007197143459082477]]},"epochLink":{"previousEpoch":"writing-v1.0.0","linked":true,"overlapCount":129,"overlapLabs":9,"slope":0.810152,"intercept":0.03412,"pearsonR":0.9876,"reason":"Verified Mean-Sigma link (n=129, labs=9, r=0.988)"}}},"anchorStats":{"lmarena_creative_writing":{"benchmarkId":"lmarena_creative_writing","mean":1432.453591,"sd":52.463576,"anchorCount":61,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":0.157031,"reliability":0.9886,"discrimination":0.8125,"redundancyFactor":0.5192,"effectiveWeight":0.417,"taskFamily":"creative_prose","datasetLineageId":"lmarena-text-creative-writing","introducedAt":"2026-06-01","invertScore":false},"lechmazur_writing":{"benchmarkId":"lechmazur_writing","mean":-0.259907,"sd":2.124674,"anchorCount":31,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":-0.260737,"reliability":0.9994,"discrimination":0.9375,"redundancyFactor":0.3763,"effectiveWeight":0.3526,"taskFamily":"creative_prose","datasetLineageId":"lechmazur-creative-writing","introducedAt":"2026-04-01","invertScore":false},"eqbench_creative_writing":{"benchmarkId":"eqbench_creative_writing","mean":14.635246,"sd":1.599277,"anchorCount":51,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":0.086813,"reliability":0.1,"discrimination":0.0134,"redundancyFactor":0.3882,"effectiveWeight":0.1,"taskFamily":"creative_prose","datasetLineageId":"eqbench-creative-writing-v3","introducedAt":"2026-05-01","invertScore":false,"popoviciuSe":1.767767,"boundedMax":20,"sampleSize":32},"eqbench_longform_writing":{"benchmarkId":"eqbench_longform_writing","mean":70.18627,"sd":13.492269,"anchorCount":53,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":0.108753,"reliability":0.1,"discrimination":0.0194,"redundancyFactor":0.6934,"effectiveWeight":0.1,"taskFamily":"longform_narrative","datasetLineageId":"eqbench-longform-writing-v3","introducedAt":"2026-02-19","invertScore":false,"popoviciuSe":14.433757,"boundedMax":100,"sampleSize":12},"surge_complex_constraints":{"benchmarkId":"surge_complex_constraints","mean":50.081679,"sd":15.9832,"anchorCount":14,"anchorLabs":8,"linkedViaMeanSigma":true,"raschDelta":0.068973,"reliability":0.4707,"discrimination":0.3876,"redundancyFactor":0.5,"effectiveWeight":0.1,"taskFamily":"constrained_professional_writing","datasetLineageId":"surge-complex-constraints-1.1","introducedAt":"2026-06-03","invertScore":false,"popoviciuSe":5.773503,"boundedMax":100,"sampleSize":75,"linkVarIntercept":3.2749981119460676,"linkVarSlope":0.0007197143459082477,"linkCovIS":-0.04838158460639}},"configs":[{"id":"openai:gpt-6.2-astra:high:direct:native","provider":"openai","modelSlug":"gpt-6.2-astra","displayName":"GPT-6.2 Astra","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":2.406861,"shrinkageFactor":0.99956,"score":86.1,"se":6.62,"samplingSe":0.2809,"residualSe":6.6134,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.24,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.8628,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:GPT-6.2 Astra (high)","observedHarness":"Direct API","rawScore":4.302,"zScore":2.147109,"seModel":"upstream_ci","ciHalfWidth":0.078,"ciLower":4.223,"ciUpper":4.379,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"}]},{"id":"anthropic:claude-fable-5.2:high:direct:native","provider":"anthropic","modelSlug":"claude-fable-5.2","displayName":"Claude Fable 5.2","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":2.059848,"shrinkageFactor":0.999655,"score":80.9,"se":6.62,"samplingSe":0.2485,"residualSe":6.6141,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.24,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1000000,"releaseDate":"2026-11-09","paretoProb":0.466,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Claude Fable 5.2 (high)","observedHarness":"Direct API","rawScore":3.564,"zScore":1.799762,"seModel":"upstream_ci","ciHalfWidth":0.069,"ciLower":3.498,"ciUpper":3.636,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"}]},{"id":"alibaba:qwen4-max:high:direct:native","provider":"alibaba","modelSlug":"qwen4-max","displayName":"Qwen4 Max","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":2.05214,"shrinkageFactor":0.99956,"score":80.78,"se":6.62,"samplingSe":0.2809,"residualSe":6.6134,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.033,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":2.25,"inputPer1mUsd":1.5,"outputPer1mUsd":4.5,"contextWindow":1000000,"releaseDate":"2026-10-22","paretoProb":0.9794,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Qwen 4 Max (high)⁂","observedHarness":"Direct API","rawScore":3.548,"zScore":1.792231,"seModel":"upstream_ci","ciHalfWidth":0.078,"ciLower":3.469,"ciUpper":3.625,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"}]},{"id":"anthropic:claude-opus-5:xhigh:direct:native","provider":"anthropic","modelSlug":"claude-opus-5","displayName":"Claude Opus 5","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"writing","theta":2.003039,"shrinkageFactor":0.999722,"score":80.05,"se":6.62,"samplingSe":0.2233,"residualSe":6.6145,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.15,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-07-28","paretoProb":0.7031,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Claude Opus 5 (xhigh)","observedHarness":"Direct API","rawScore":3.443,"zScore":1.742812,"seModel":"upstream_ci","ciHalfWidth":0.062,"ciLower":3.378,"ciUpper":3.502,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"}]},{"id":"openai:gpt-6-astra:high:direct:native","provider":"openai","modelSlug":"gpt-6-astra","displayName":"GPT-6 Astra","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":1.872732,"shrinkageFactor":0.999739,"score":78.09,"se":6.62,"samplingSe":0.2161,"residualSe":6.6146,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.3,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.1544,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:GPT-6 Astra (high)","observedHarness":"Direct API","rawScore":3.166,"zScore":1.612439,"seModel":"upstream_ci","ciHalfWidth":0.06,"ciLower":3.105,"ciUpper":3.225,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"}]},{"id":"anthropic:claude-opus-5.5:high:direct:native","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":1.671193,"shrinkageFactor":0.995969,"score":75.07,"se":2.3,"samplingSe":0.8498,"residualSe":2.1343,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.12,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.7906,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Claude Opus 5.5 (high)","observedHarness":"Direct API","rawScore":3.355,"zScore":1.701394,"seModel":"upstream_ci","ciHalfWidth":0.0695,"ciLower":3.286,"ciUpper":3.425,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Claude Opus 5.5 (High)","observedHarness":"Direct API","rawScore":1516.05,"zScore":1.593418,"seModel":"upstream_ci","ciHalfWidth":10.675,"ciLower":1505.3778,"ciUpper":1526.7279,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-fable-5.1:high:direct:native","provider":"anthropic","modelSlug":"claude-fable-5.1","displayName":"Claude Fable 5.1","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":1.568504,"shrinkageFactor":0.944333,"score":73.53,"se":5.18,"samplingSe":3.1578,"residualSe":4.107,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.3,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-09-12","paretoProb":0.051,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:claude-fable-5-1","observedHarness":"Direct API","rawScore":16.27,"zScore":1.022183,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:claude-fable-5-1","observedHarness":"Direct API","rawScore":85.3,"zScore":1.120177,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Claude Fable 5.1 (high)","observedHarness":"Direct API","rawScore":3.51,"zScore":1.774346,"seModel":"upstream_ci","ciHalfWidth":0.08,"ciLower":3.428,"ciUpper":3.588,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"}]},{"id":"moonshot:kimi-k3.1-pro:high:direct:native","provider":"moonshot","modelSlug":"kimi-k3.1-pro","displayName":"Kimi K3.1 Pro","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":1.496108,"shrinkageFactor":0.999593,"score":72.44,"se":6.62,"samplingSe":0.2701,"residualSe":6.6137,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.06,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-10-10","paretoProb":0.9574,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Kimi K3.1 Pro (high)","observedHarness":"Direct API","rawScore":2.366,"zScore":1.235911,"seModel":"upstream_ci","ciHalfWidth":0.075,"ciLower":2.29,"ciUpper":2.44,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"}]},{"id":"anthropic:claude-fable-5.2:max:direct:native","provider":"anthropic","modelSlug":"claude-fable-5.2","displayName":"Claude Fable 5.2","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":1.46995,"shrinkageFactor":0.966715,"score":72.05,"se":5.11,"samplingSe":2.4418,"residualSe":4.4938,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.24,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1000000,"releaseDate":"2026-11-09","paretoProb":0.0586,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Claude Fable 5.2 (Max)","observedHarness":"Direct API","rawScore":1528.73,"zScore":1.83511,"seModel":"upstream_ci","ciHalfWidth":18.8595,"ciLower":1509.8687,"ciUpper":1547.5877,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:Claude Fable 5.2 (Adaptive/Max)","observedHarness":"Direct API","rawScore":64.5,"zScore":0.902092,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"openai:gpt-5.5:xhigh:direct:native","provider":"openai","modelSlug":"gpt-5.5","displayName":"GPT-5.5","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"writing","theta":1.403556,"shrinkageFactor":0.999699,"score":71.05,"se":6.62,"samplingSe":0.2323,"residualSe":6.6144,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.17,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":11.25,"inputPer1mUsd":5,"outputPer1mUsd":30,"contextWindow":1050000,"releaseDate":"2026-04-23","paretoProb":0.2817,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:GPT-5.5 (xhigh)","observedHarness":"Direct API","rawScore":2.169,"zScore":1.14319,"seModel":"upstream_ci","ciHalfWidth":0.0645,"ciLower":2.112,"ciUpper":2.241,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"}]},{"id":"openai:gpt-5.6-sol:high:direct:native","provider":"openai","modelSlug":"gpt-5.6-sol","displayName":"GPT-5.6 Sol","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":1.329104,"shrinkageFactor":0.999604,"score":69.94,"se":6.62,"samplingSe":0.2665,"residualSe":6.6137,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.12,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.3149,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:GPT-5.6 Sol (high)","observedHarness":"Direct API","rawScore":2.011,"zScore":1.068826,"seModel":"upstream_ci","ciHalfWidth":0.074,"ciLower":1.935,"ciUpper":2.083,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"}]},{"id":"google:gemini-4-carbon:high:direct:native","provider":"google","modelSlug":"gemini-4-carbon","displayName":"Gemini 4 Carbon","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":1.273018,"shrinkageFactor":0.986673,"score":69.1,"se":2.13,"samplingSe":1.5451,"residualSe":1.4679,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.06,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-10-12","paretoProb":0.3115,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Gemini 4 Carbon (high)","observedHarness":"Direct API","rawScore":1.618,"zScore":0.883857,"seModel":"upstream_ci","ciHalfWidth":0.085,"ciLower":1.535,"ciUpper":1.705,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Gemini 4 Carbon (High)","observedHarness":"Direct API","rawScore":1515.3,"zScore":1.579122,"seModel":"upstream_ci","ciHalfWidth":20.1448,"ciLower":1495.1503,"ciUpper":1535.4399,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:Gemini 4 Carbon (High reasoning)","observedHarness":"Direct API","rawScore":70.9,"zScore":1.302513,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"anthropic:claude-fable-5:high:direct:native","provider":"anthropic","modelSlug":"claude-fable-5","displayName":"Claude Fable 5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":1.248154,"shrinkageFactor":0.979568,"score":68.72,"se":2.11,"samplingSe":1.9131,"residualSe":0.8849,"coverageCount":4,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.3,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-06-09","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:claude-fable-5","observedHarness":"Direct API","rawScore":16.38,"zScore":1.090964,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:claude-fable-5","observedHarness":"Direct API","rawScore":83,"zScore":0.949709,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Claude Fable 5 (high)§","observedHarness":"Direct API","rawScore":2.585,"zScore":1.338985,"seModel":"upstream_ci","ciHalfWidth":0.0655,"ciLower":2.522,"ciUpper":2.653,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Claude Fable 5 (High)","observedHarness":"Direct API","rawScore":1501.51,"zScore":1.316273,"seModel":"upstream_ci","ciHalfWidth":7.1593,"ciLower":1494.3484,"ciUpper":1508.667,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-4-hydrogen:xhigh:direct:native","provider":"google","modelSlug":"gemini-4-hydrogen","displayName":"Gemini 4 Hydrogen","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"writing","theta":1.207439,"shrinkageFactor":0.837258,"score":68.11,"se":9.11,"samplingSe":5.3993,"residualSe":7.3337,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.045,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":3,"inputPer1mUsd":1.5,"outputPer1mUsd":7.5,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.567,"benchmarks":[{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:Gemini 4 Hydrogen (xHigh reasoning)","observedHarness":"Direct API","rawScore":73.7,"zScore":1.477697,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"openai:gpt-5.4:medium:direct:native","provider":"openai","modelSlug":"gpt-5.4","displayName":"GPT-5.4","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"writing","theta":1.178078,"shrinkageFactor":0.999132,"score":67.67,"se":6.62,"samplingSe":0.3942,"residualSe":6.6106,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.085,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":5.625,"inputPer1mUsd":2.5,"outputPer1mUsd":15,"contextWindow":1050000,"releaseDate":"2026-03-05","paretoProb":0.2604,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:GPT-5.4 (medium)","observedHarness":"Direct API","rawScore":1.691,"zScore":0.918215,"seModel":"upstream_ci","ciHalfWidth":0.1095,"ciLower":1.58,"ciUpper":1.799,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"}]},{"id":"openai:gpt-5.4:xhigh:direct:native","provider":"openai","modelSlug":"gpt-5.4","displayName":"GPT-5.4","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"writing","theta":1.175334,"shrinkageFactor":0.99921,"score":67.63,"se":6.62,"samplingSe":0.3763,"residualSe":6.6111,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.085,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":5.625,"inputPer1mUsd":2.5,"outputPer1mUsd":15,"contextWindow":1050000,"releaseDate":"2026-03-05","paretoProb":0.2561,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:GPT-5.4 (xhigh)","observedHarness":"Direct API","rawScore":1.685,"zScore":0.915391,"seModel":"upstream_ci","ciHalfWidth":0.1045,"ciLower":1.58,"ciUpper":1.789,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"}]},{"id":"google:gemini-4-neon:high:direct:native","provider":"google","modelSlug":"gemini-4-neon","displayName":"Gemini 4 Neon","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":1.16517,"shrinkageFactor":0.96702,"score":67.48,"se":5.11,"samplingSe":2.4306,"residualSe":4.4952,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.09,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1048576,"releaseDate":"2026-11-18","paretoProb":0.1958,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Gemini 4 Neon (High)","observedHarness":"Direct API","rawScore":1501.59,"zScore":1.317798,"seModel":"upstream_ci","ciHalfWidth":18.7168,"ciLower":1482.8726,"ciUpper":1520.3062,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:Gemini 4 Neon (High reasoning)","observedHarness":"Direct API","rawScore":72.9,"zScore":1.427644,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"meta:muse-wonder:high:direct:native","provider":"meta","modelSlug":"muse-wonder","displayName":"Muse Wonder","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":1.139994,"shrinkageFactor":0.999625,"score":67.1,"se":6.62,"samplingSe":0.2593,"residualSe":6.6139,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.12,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.5558,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Muse Wonder (high)","observedHarness":"Direct API","rawScore":1.609,"zScore":0.879621,"seModel":"upstream_ci","ciHalfWidth":0.072,"ciLower":1.537,"ciUpper":1.681,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"}]},{"id":"anthropic:claude-opus-5:high:direct:native","provider":"anthropic","modelSlug":"claude-opus-5","displayName":"Claude Opus 5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":1.099988,"shrinkageFactor":0.98065,"score":66.5,"se":2.06,"samplingSe":1.8618,"residualSe":0.8859,"coverageCount":4,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.15,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-07-28","paretoProb":0.0001,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:claude-opus-5","observedHarness":"Direct API","rawScore":16.26,"zScore":1.01593,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:claude-opus-5","observedHarness":"Direct API","rawScore":86.3,"zScore":1.194294,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Claude Opus 5 (high)","observedHarness":"Direct API","rawScore":3.049,"zScore":1.557372,"seModel":"upstream_ci","ciHalfWidth":0.1015,"ciLower":2.945,"ciUpper":3.148,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Claude Opus 5 (High)","observedHarness":"Direct API","rawScore":1471.11,"zScore":0.736824,"seModel":"upstream_ci","ciHalfWidth":6.4042,"ciLower":1464.7044,"ciUpper":1477.5129,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-4-argon:high:direct:native","provider":"google","modelSlug":"gemini-4-argon","displayName":"Gemini 4 Argon","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":1.099528,"shrinkageFactor":0.995269,"score":66.49,"se":1.74,"samplingSe":0.9206,"residualSe":1.4807,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.06,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.0462,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Gemini 4 Argon (high)","observedHarness":"Direct API","rawScore":0.454,"zScore":0.336008,"seModel":"upstream_ci","ciHalfWidth":0.0985,"ciLower":0.354,"ciUpper":0.551,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Gemini 4 Argon (High)","observedHarness":"Direct API","rawScore":1519.95,"zScore":1.667755,"seModel":"upstream_ci","ciHalfWidth":9.2348,"ciLower":1510.711,"ciUpper":1529.1805,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:Gemini 4 Argon (High reasoning)","observedHarness":"Direct API","rawScore":70.3,"zScore":1.264973,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"anthropic:claude-opus-4.6:high:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.6","displayName":"Claude Opus 4.6","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":1.090982,"shrinkageFactor":0.995639,"score":66.36,"se":5.99,"samplingSe":0.8838,"residualSe":5.9219,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.15,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-02-05","paretoProb":0.0226,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Claude Opus 4.6 (High)","observedHarness":"Direct API","rawScore":1498.14,"zScore":1.252038,"seModel":"upstream_ci","ciHalfWidth":6.072,"ciLower":1492.0719,"ciUpper":1504.2159,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-6.2-astra:max:direct:native","provider":"openai","modelSlug":"gpt-6.2-astra","displayName":"GPT-6.2 Astra","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":1.007697,"shrinkageFactor":0.852591,"score":65.12,"se":9.07,"samplingSe":5.1386,"residualSe":7.468,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.24,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.0231,"benchmarks":[{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:GPT 6.2 Astra (Max reasoning)","observedHarness":"Direct API","rawScore":69.6,"zScore":1.221177,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"openai:gpt-6.2-sol:max:direct:native","provider":"openai","modelSlug":"gpt-6.2-sol","displayName":"GPT-6.2 Sol","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":1.002484,"shrinkageFactor":0.852715,"score":65.04,"se":9.06,"samplingSe":5.1365,"residualSe":7.4691,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.06,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.5096,"benchmarks":[{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:GPT 6.2 Sol (Max reasoning)","observedHarness":"Direct API","rawScore":69.5,"zScore":1.214921,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"anthropic:claude-opus-4.7:high:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.7","displayName":"Claude Opus 4.7","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":1.000752,"shrinkageFactor":0.998151,"score":65.01,"se":2.22,"samplingSe":0.5756,"residualSe":2.1389,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.15,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-04-16","paretoProb":0,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Claude Opus 4.7 (high)†","observedHarness":"Direct API","rawScore":1.496,"zScore":0.826436,"seModel":"upstream_ci","ciHalfWidth":0.1065,"ciLower":1.389,"ciUpper":1.602,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Claude Opus 4.7 (High)","observedHarness":"Direct API","rawScore":1489.51,"zScore":1.087543,"seModel":"upstream_ci","ciHalfWidth":6.9414,"ciLower":1482.5691,"ciUpper":1496.452,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-5.6-sol:xhigh:direct:native","provider":"openai","modelSlug":"gpt-5.6-sol","displayName":"GPT-5.6 Sol","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.959469,"shrinkageFactor":0.997986,"score":64.39,"se":2.22,"samplingSe":0.6007,"residualSe":2.1386,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.12,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0106,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:GPT-5.6 Sol (xhigh)","observedHarness":"Direct API","rawScore":2.199,"zScore":1.15731,"seModel":"upstream_ci","ciHalfWidth":0.084,"ciLower":2.112,"ciUpper":2.28,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GPT-5.6 Sol (xHigh)","observedHarness":"Direct API","rawScore":1470.84,"zScore":0.731677,"seModel":"upstream_ci","ciHalfWidth":7.4024,"ciLower":1463.4393,"ciUpper":1478.2441,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-4-hydrogen:high:direct:native","provider":"google","modelSlug":"gemini-4-hydrogen","displayName":"Gemini 4 Hydrogen","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.91671,"shrinkageFactor":0.955635,"score":63.75,"se":6.34,"samplingSe":2.8191,"residualSe":5.6839,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.045,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":3,"inputPer1mUsd":1.5,"outputPer1mUsd":7.5,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.263,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Gemini 4 Hydrogen (High)","observedHarness":"Direct API","rawScore":1490.6,"zScore":1.10832,"seModel":"upstream_ci","ciHalfWidth":19.769,"ciLower":1470.8305,"ciUpper":1510.3685,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-fable-5.1:max:direct:native","provider":"anthropic","modelSlug":"claude-fable-5.1","displayName":"Claude Fable 5.1","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.894589,"shrinkageFactor":0.984625,"score":63.42,"se":4.87,"samplingSe":1.6595,"residualSe":4.5771,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.3,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-09-12","paretoProb":0.0005,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Claude Fable 5.1 (Max)","observedHarness":"Direct API","rawScore":1492.36,"zScore":1.141867,"seModel":"upstream_ci","ciHalfWidth":9.5873,"ciLower":1482.7725,"ciUpper":1501.947,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:Claude Fable 5.1 (Adaptive/Max)","observedHarness":"Direct API","rawScore":60.4,"zScore":0.645573,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"meta:muse-wonder:max:direct:native","provider":"meta","modelSlug":"muse-wonder","displayName":"Muse Wonder","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.883216,"shrinkageFactor":0.964537,"score":63.25,"se":4.46,"samplingSe":2.5204,"residualSe":3.6793,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.12,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.2347,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:muse-wonder","observedHarness":"Direct API","rawScore":16.22,"zScore":0.990919,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Muse Wonder (Max)","observedHarness":"Direct API","rawScore":1486.75,"zScore":1.034935,"seModel":"upstream_ci","ciHalfWidth":11.024,"ciLower":1475.7287,"ciUpper":1497.7767,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:Muse Wonder 1.0 (Max reasoning)","observedHarness":"Direct API","rawScore":67.9,"zScore":1.114816,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"anthropic:claude-sonnet-5.5:max:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.879353,"shrinkageFactor":0.852204,"score":63.19,"se":9.07,"samplingSe":5.1454,"residualSe":7.4646,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.06,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.7289,"benchmarks":[{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:Claude Sonnet 5.5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":67.2,"zScore":1.07102,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"zhipu:glm-5.3:max:direct:native","provider":"zhipu","modelSlug":"glm-5.3","displayName":"GLM-5.3","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.822667,"shrinkageFactor":0.979845,"score":62.34,"se":1.9,"samplingSe":1.9001,"residualSe":0,"coverageCount":5,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0316,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":1048576,"releaseDate":"2026-09-18","paretoProb":0.7686,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:GLM-5.3","observedHarness":"Direct API","rawScore":16.44,"zScore":1.128481,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:GLM-5.3","observedHarness":"Direct API","rawScore":81.8,"zScore":0.860769,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:GLM-5.3 (max)","observedHarness":"Direct API","rawScore":2.613,"zScore":1.352164,"seModel":"upstream_ci","ciHalfWidth":0.083,"ciLower":2.529,"ciUpper":2.695,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GLM-5.3 (Max)","observedHarness":"Direct API","rawScore":1456.55,"zScore":0.459298,"seModel":"upstream_ci","ciHalfWidth":7.9075,"ciLower":1448.6399,"ciUpper":1464.4548,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:GLM 5.3 (Max reasoning)","observedHarness":"Direct API","rawScore":54.4,"zScore":0.270179,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"anthropic:claude-opus-4.7:none:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.7","displayName":"Claude Opus 4.7","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.760163,"shrinkageFactor":0.949326,"score":61.4,"se":4.71,"samplingSe":3.0129,"residualSe":3.6213,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.15,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-04-16","paretoProb":0.0009,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:claude-opus-4-7","observedHarness":"Direct API","rawScore":16.13,"zScore":0.934644,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:claude-opus-4-7","observedHarness":"Direct API","rawScore":81.8,"zScore":0.860769,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Claude Opus 4.7","observedHarness":"Direct API","rawScore":1482.01,"zScore":0.944587,"seModel":"upstream_ci","ciHalfWidth":6.8758,"ciLower":1475.1377,"ciUpper":1488.8894,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"moonshot:kimi-k3:max:direct:native","provider":"moonshot","modelSlug":"kimi-k3","displayName":"Kimi K3","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.733303,"shrinkageFactor":0.978594,"score":61,"se":1.96,"samplingSe":1.9582,"residualSe":0,"coverageCount":5,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.09,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1048576,"releaseDate":"2026-08-14","paretoProb":0.0361,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:kimi-k3","observedHarness":"Direct API","rawScore":16.35,"zScore":1.072206,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:kimi-k3","observedHarness":"Direct API","rawScore":79.6,"zScore":0.697713,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Kimi K3","observedHarness":"Direct API","rawScore":2.104,"zScore":1.112598,"seModel":"upstream_ci","ciHalfWidth":0.0695,"ciLower":2.034,"ciUpper":2.173,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Kimi K3 (Max)","observedHarness":"Direct API","rawScore":1460.36,"zScore":0.53192,"seModel":"upstream_ci","ciHalfWidth":7.2939,"ciLower":1453.0632,"ciUpper":1467.6509,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:Kimi K3 (Max reasoning)","observedHarness":"Direct API","rawScore":51.1,"zScore":0.063712,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"anthropic:claude-opus-5.5:max:direct:native","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.724012,"shrinkageFactor":0.84297,"score":60.86,"se":9.09,"samplingSe":5.3037,"residualSe":7.3837,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.12,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.0668,"benchmarks":[{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:Claude Opus 5.5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":64.4,"zScore":0.895836,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"google:gemini-3.8-flash:high:direct:native","provider":"google","modelSlug":"gemini-3.8-flash","displayName":"Gemini 3.8 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.650062,"shrinkageFactor":0.996176,"score":59.75,"se":1.7,"samplingSe":0.8277,"residualSe":1.4821,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.0225,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":1.5,"inputPer1mUsd":0.75,"outputPer1mUsd":3.75,"contextWindow":1048576,"releaseDate":"2026-08-10","paretoProb":0.3377,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Gemini 3.8 Flash (high)","observedHarness":"Direct API","rawScore":-0.157,"zScore":0.048434,"seModel":"upstream_ci","ciHalfWidth":0.091,"ciLower":-0.243,"ciUpper":-0.061,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Gemini 3.8 Flash (High)","observedHarness":"Direct API","rawScore":1488.84,"zScore":1.074773,"seModel":"upstream_ci","ciHalfWidth":6.4911,"ciLower":1482.3462,"ciUpper":1495.3285,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:Gemini 3.8 Flash (High reasoning)","observedHarness":"Direct API","rawScore":63.2,"zScore":0.820757,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"anthropic:claude-opus-5.5:medium:direct:native","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.638637,"shrinkageFactor":0.681482,"score":59.58,"se":8.75,"samplingSe":7.5536,"residualSe":4.4084,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.12,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.0455,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:claude-opus-5-5","observedHarness":"Direct API","rawScore":15.98,"zScore":0.840851,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:claude-opus-5-5","observedHarness":"Direct API","rawScore":84.6,"zScore":1.068295,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"}]},{"id":"anthropic:claude-opus-4.8:xhigh:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.8","displayName":"Claude Opus 4.8","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.618046,"shrinkageFactor":0.999217,"score":59.27,"se":6.62,"samplingSe":0.3745,"residualSe":6.6112,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.15,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-06-12","paretoProb":0.0013,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Claude Opus 4.8 (xhigh)","observedHarness":"Direct API","rawScore":0.5,"zScore":0.357658,"seModel":"upstream_ci","ciHalfWidth":0.104,"ciLower":0.398,"ciUpper":0.606,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"}]},{"id":"anthropic:claude-opus-4.6:none:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.6","displayName":"Claude Opus 4.6","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.614903,"shrinkageFactor":0.945221,"score":59.22,"se":4.78,"samplingSe":3.1325,"residualSe":3.6057,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.15,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-02-05","paretoProb":0.0002,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:claude-opus-4-6","observedHarness":"Direct API","rawScore":15.66,"zScore":0.640761,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:claude-opus-4-6","observedHarness":"Direct API","rawScore":77.7,"zScore":0.556892,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Claude Opus 4.6","observedHarness":"Direct API","rawScore":1477.81,"zScore":0.864531,"seModel":"upstream_ci","ciHalfWidth":5.9161,"ciLower":1471.8928,"ciUpper":1483.725,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-opus-5:max:direct:native","provider":"anthropic","modelSlug":"claude-opus-5","displayName":"Claude Opus 5","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.612311,"shrinkageFactor":0.992527,"score":59.18,"se":6.02,"samplingSe":1.157,"residualSe":5.9034,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.15,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-07-28","paretoProb":0.0034,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Claude Opus 5 (Max)","observedHarness":"Direct API","rawScore":1472.99,"zScore":0.772658,"seModel":"upstream_ci","ciHalfWidth":7.9614,"ciLower":1465.0263,"ciUpper":1480.9491,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"meta:muse-spark-1.3:max:direct:native","provider":"meta","modelSlug":"muse-spark-1.3","displayName":"Muse Spark 1.3","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.61219,"shrinkageFactor":0.987721,"score":59.18,"se":4.83,"samplingSe":1.4831,"residualSe":4.5915,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0295,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.3268,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Muse Spark 1.3 (Max)","observedHarness":"Direct API","rawScore":1470.15,"zScore":0.718525,"seModel":"upstream_ci","ciHalfWidth":8.4829,"ciLower":1461.6684,"ciUpper":1478.6342,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:Muse Spark 1.3 (Max reasoning)","observedHarness":"Direct API","rawScore":64.8,"zScore":0.920862,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"openai:gpt-6-sol:medium:direct:native","provider":"openai","modelSlug":"gpt-6-sol","displayName":"GPT-6 Sol","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.604607,"shrinkageFactor":0.676036,"score":59.07,"se":8.78,"samplingSe":7.6179,"residualSe":4.3732,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.06,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.2102,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:gpt-6-sol","observedHarness":"Direct API","rawScore":16.05,"zScore":0.884621,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:gpt-6-sol","observedHarness":"Direct API","rawScore":82.8,"zScore":0.934886,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"}]},{"id":"openai:gpt-5.6-sol:medium:direct:native","provider":"openai","modelSlug":"gpt-5.6-sol","displayName":"GPT-5.6 Sol","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.601425,"shrinkageFactor":0.674485,"score":59.02,"se":8.79,"samplingSe":7.6361,"residualSe":4.3631,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.12,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0528,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:gpt-5.6-sol","observedHarness":"Direct API","rawScore":16.17,"zScore":0.959655,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:gpt-5.6-sol","observedHarness":"Direct API","rawScore":81.7,"zScore":0.853358,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"}]},{"id":"google:gemini-3.6-flash:high:direct:native","provider":"google","modelSlug":"gemini-3.6-flash","displayName":"Gemini 3.6 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.583491,"shrinkageFactor":0.992371,"score":58.75,"se":6.02,"samplingSe":1.169,"residualSe":5.9024,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0225,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":1.5,"inputPer1mUsd":0.75,"outputPer1mUsd":3.75,"contextWindow":1048576,"releaseDate":"2026-05-05","paretoProb":0.3242,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Gemini 3.6 Flash (High)","observedHarness":"Direct API","rawScore":1471.47,"zScore":0.743686,"seModel":"upstream_ci","ciHalfWidth":8.0444,"ciLower":1463.4257,"ciUpper":1479.5145,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-6-astra:medium:direct:native","provider":"openai","modelSlug":"gpt-6-astra","displayName":"GPT-6 Astra","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.580045,"shrinkageFactor":0.673868,"score":58.7,"se":8.8,"samplingSe":7.6433,"residualSe":4.3591,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.3,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0031,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:gpt-6-astra","observedHarness":"Direct API","rawScore":15.94,"zScore":0.81584,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:gpt-6-astra","observedHarness":"Direct API","rawScore":82.8,"zScore":0.934886,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"}]},{"id":"google:gemini-3.5-flash:medium:direct:native","provider":"google","modelSlug":"gemini-3.5-flash","displayName":"Gemini 3.5 Flash","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.543263,"shrinkageFactor":0.993551,"score":58.15,"se":6.01,"samplingSe":1.0748,"residualSe":5.9095,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.051,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":3.375,"inputPer1mUsd":1.5,"outputPer1mUsd":9,"contextWindow":1048576,"releaseDate":"2026-04-12","paretoProb":0.052,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Gemini 3.5 Flash (Medium)","observedHarness":"Direct API","rawScore":1469.32,"zScore":0.702705,"seModel":"upstream_ci","ciHalfWidth":7.3921,"ciLower":1461.9234,"ciUpper":1476.7075,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"meta:muse-spark-1.3:high:direct:native","provider":"meta","modelSlug":"muse-spark-1.3","displayName":"Muse Spark 1.3","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.538282,"shrinkageFactor":0.938772,"score":58.07,"se":5.26,"samplingSe":3.3118,"residualSe":4.0828,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0295,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.2644,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:muse-spark-1.3","observedHarness":"Direct API","rawScore":15.64,"zScore":0.628255,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:muse-spark-1.3","observedHarness":"Direct API","rawScore":82.8,"zScore":0.934886,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Muse Spark 1.3 (high)","observedHarness":"Direct API","rawScore":0.234,"zScore":0.232462,"seModel":"upstream_ci","ciHalfWidth":0.079,"ciLower":0.151,"ciUpper":0.309,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"}]},{"id":"spacexai:grok-4.7:high:direct:native","provider":"spacexai","modelSlug":"grok-4.7","displayName":"Grok 4.7","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.516361,"shrinkageFactor":0.939936,"score":57.75,"se":5.24,"samplingSe":3.2801,"residualSe":4.0879,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.044,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-10-18","paretoProb":0.6708,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:grok-4.7","observedHarness":"Direct API","rawScore":15.95,"zScore":0.822093,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:grok-4.7","observedHarness":"Direct API","rawScore":83,"zScore":0.949709,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Grok 4.7 (high)","observedHarness":"Direct API","rawScore":0.029,"zScore":0.135977,"seModel":"upstream_ci","ciHalfWidth":0.162,"ciLower":-0.146,"ciUpper":0.178,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"}]},{"id":"anthropic:claude-opus-4.8:none:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.8","displayName":"Claude Opus 4.8","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.503548,"shrinkageFactor":0.948062,"score":57.55,"se":4.73,"samplingSe":3.0502,"residualSe":3.6165,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.15,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-06-12","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:claude-opus-4-8","observedHarness":"Direct API","rawScore":15.94,"zScore":0.81584,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:claude-opus-4-8","observedHarness":"Direct API","rawScore":80.8,"zScore":0.786653,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Claude Opus 4.8","observedHarness":"Direct API","rawScore":1463.49,"zScore":0.59158,"seModel":"upstream_ci","ciHalfWidth":6.5021,"ciLower":1456.9913,"ciUpper":1469.9955,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-opus-4.8:high:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.8","displayName":"Claude Opus 4.8","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.475494,"shrinkageFactor":0.998297,"score":57.13,"se":2.21,"samplingSe":0.5523,"residualSe":2.1393,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.15,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-06-12","paretoProb":0,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Claude Opus 4.8 (high)‡","observedHarness":"Direct API","rawScore":-0.013,"zScore":0.116209,"seModel":"upstream_ci","ciHalfWidth":0.1165,"ciLower":-0.137,"ciUpper":0.096,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Claude Opus 4.8 (High)","observedHarness":"Direct API","rawScore":1470.06,"zScore":0.71681,"seModel":"upstream_ci","ciHalfWidth":6.5575,"ciLower":1463.5066,"ciUpper":1476.6217,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-4-argon:medium:direct:native","provider":"google","modelSlug":"gemini-4-argon","displayName":"Gemini 4 Argon","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.446871,"shrinkageFactor":0.660258,"score":56.7,"se":8.89,"samplingSe":7.8012,"residualSe":4.2711,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.06,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.0469,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:gemini-4-argon","observedHarness":"Direct API","rawScore":15.56,"zScore":0.578233,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:gemini-4-argon","observedHarness":"Direct API","rawScore":80.9,"zScore":0.794064,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"}]},{"id":"openai:gpt-5.6-terra:medium:direct:native","provider":"openai","modelSlug":"gpt-5.6-terra","displayName":"GPT-5.6 Terra","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.444861,"shrinkageFactor":0.657279,"score":56.67,"se":8.91,"samplingSe":7.8353,"residualSe":4.2518,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.068,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":4.5,"inputPer1mUsd":2,"outputPer1mUsd":12,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.1216,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:gpt-5.6-terra","observedHarness":"Direct API","rawScore":15.9,"zScore":0.790829,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:gpt-5.6-terra","observedHarness":"Direct API","rawScore":78,"zScore":0.579126,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"}]},{"id":"alibaba:qwen4-max:xhigh:direct:native","provider":"alibaba","modelSlug":"qwen4-max","displayName":"Qwen4 Max","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.431278,"shrinkageFactor":0.802129,"score":56.47,"se":9.21,"samplingSe":5.9536,"residualSe":7.026,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.033,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":2.25,"inputPer1mUsd":1.5,"outputPer1mUsd":4.5,"contextWindow":1000000,"releaseDate":"2026-10-22","paretoProb":0.0206,"benchmarks":[{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:Qwen 4 Max (xHigh reasoning)","observedHarness":"Direct API","rawScore":59.1,"zScore":0.564238,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"google:gemini-3.7-flash:high:direct:native","provider":"google","modelSlug":"gemini-3.7-flash","displayName":"Gemini 3.7 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.418921,"shrinkageFactor":0.983193,"score":56.28,"se":2.27,"samplingSe":1.7351,"residualSe":1.4628,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0225,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":1.5,"inputPer1mUsd":0.75,"outputPer1mUsd":3.75,"contextWindow":1048576,"releaseDate":"2026-06-18","paretoProb":0.0322,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:gemini-3.7-flash","observedHarness":"Direct API","rawScore":15.22,"zScore":0.365636,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Gemini 3.7 Flash (high)","observedHarness":"Direct API","rawScore":-0.995,"zScore":-0.345979,"seModel":"upstream_ci","ciHalfWidth":0.1165,"ciLower":-1.101,"ciUpper":-0.868,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Gemini 3.7 Flash (High)","observedHarness":"Direct API","rawScore":1487.26,"zScore":1.044656,"seModel":"upstream_ci","ciHalfWidth":8.2039,"ciLower":1479.0606,"ciUpper":1495.4684,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"meta:muse-ember:max:direct:native","provider":"meta","modelSlug":"muse-ember","displayName":"Muse Ember","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.415279,"shrinkageFactor":0.799034,"score":56.23,"se":9.22,"samplingSe":6,"residualSe":6.9989,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.003,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1048576,"releaseDate":null,"paretoProb":1,"benchmarks":[{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:Muse Ember 1.0 (Max reasoning)","observedHarness":"Direct API","rawScore":58.8,"zScore":0.545468,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"openai:gpt-5.2:medium:direct:native","provider":"openai","modelSlug":"gpt-5.2","displayName":"GPT-5.2","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.40894,"shrinkageFactor":0.997972,"score":56.13,"se":6.63,"samplingSe":0.6027,"residualSe":6.6029,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0735,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":4.8125,"inputPer1mUsd":1.75,"outputPer1mUsd":14,"contextWindow":400000,"releaseDate":"2025-12-11","paretoProb":0.0646,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:GPT-5.2 (medium)","observedHarness":"Direct API","rawScore":0.056,"zScore":0.148685,"seModel":"upstream_ci","ciHalfWidth":0.1675,"ciLower":-0.103,"ciUpper":0.232,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"}]},{"id":"moonshot:kimi-k3.1-pro:max:direct:native","provider":"moonshot","modelSlug":"kimi-k3.1-pro","displayName":"Kimi K3.1 Pro","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.398851,"shrinkageFactor":0.939401,"score":55.98,"se":4.87,"samplingSe":3.2947,"residualSe":3.5835,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.06,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-10-10","paretoProb":0.0422,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:kimi-k3.1-pro","observedHarness":"Direct API","rawScore":16.63,"zScore":1.247285,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:kimi-k3.1-pro","observedHarness":"Direct API","rawScore":79.9,"zScore":0.719948,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Kimi K3.1 Pro (Max)","observedHarness":"Direct API","rawScore":1450.5,"zScore":0.34398,"seModel":"upstream_ci","ciHalfWidth":16.4053,"ciLower":1434.0956,"ciUpper":1466.9061,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-6-astra:max:direct:native","provider":"openai","modelSlug":"gpt-6-astra","displayName":"GPT-6 Astra","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.389513,"shrinkageFactor":0.98587,"score":55.84,"se":4.85,"samplingSe":1.591,"residualSe":4.5829,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.3,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GPT-6 Astra (Max)","observedHarness":"Direct API","rawScore":1450.79,"zScore":0.349507,"seModel":"upstream_ci","ciHalfWidth":10.0067,"ciLower":1440.7813,"ciUpper":1460.7948,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:GPT 6 Astra (Max reasoning)","observedHarness":"Direct API","rawScore":70.8,"zScore":1.296256,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"openai:gpt-6.1-sol:max:direct:native","provider":"openai","modelSlug":"gpt-6.1-sol","displayName":"GPT-6.1 Sol","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.374865,"shrinkageFactor":0.98327,"score":55.62,"se":4.89,"samplingSe":1.7312,"residualSe":4.5708,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.06,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.0413,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GPT-6.1 Sol (Max)","observedHarness":"Direct API","rawScore":1450.41,"zScore":0.342264,"seModel":"upstream_ci","ciHalfWidth":11.6591,"ciLower":1438.7488,"ciUpper":1462.0671,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:GPT 6.1 Sol (Max reasoning)","observedHarness":"Direct API","rawScore":70.1,"zScore":1.25246,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"anthropic:claude-sonnet-5.5:xhigh:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.368896,"shrinkageFactor":0.98622,"score":55.53,"se":6.07,"samplingSe":1.5711,"residualSe":5.8659,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.06,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.2269,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Claude Sonnet 5.5 (xHigh)","observedHarness":"Direct API","rawScore":1460.19,"zScore":0.528679,"seModel":"upstream_ci","ciHalfWidth":10.8453,"ciLower":1449.3435,"ciUpper":1471.0342,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"meta:muse-spark-1.2:high:direct:native","provider":"meta","modelSlug":"muse-spark-1.2","displayName":"Muse Spark 1.2","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.357291,"shrinkageFactor":0.93724,"score":55.36,"se":5.28,"samplingSe":3.353,"residualSe":4.0761,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0295,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.1045,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:muse-spark-1.2","observedHarness":"Direct API","rawScore":15.59,"zScore":0.596991,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:muse-spark-1.2","observedHarness":"Direct API","rawScore":81.5,"zScore":0.838534,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Muse Spark 1.2 (high)","observedHarness":"Direct API","rawScore":-0.33,"zScore":-0.03299,"seModel":"upstream_ci","ciHalfWidth":0.104,"ciLower":-0.445,"ciUpper":-0.237,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"}]},{"id":"google:gemini-3.8-flash:medium:direct:native","provider":"google","modelSlug":"gemini-3.8-flash","displayName":"Gemini 3.8 Flash","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.351563,"shrinkageFactor":0.64854,"score":55.27,"se":8.98,"samplingSe":7.9346,"residualSe":4.1953,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0225,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":1.5,"inputPer1mUsd":0.75,"outputPer1mUsd":3.75,"contextWindow":1048576,"releaseDate":"2026-08-10","paretoProb":0.2381,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:gemini-3.8-flash","observedHarness":"Direct API","rawScore":15.6,"zScore":0.603244,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:gemini-3.8-flash","observedHarness":"Direct API","rawScore":76.8,"zScore":0.490187,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"}]},{"id":"openai:gpt-5.6-luna:medium:direct:native","provider":"openai","modelSlug":"gpt-5.6-luna","displayName":"GPT-5.6 Luna","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.338065,"shrinkageFactor":0.646585,"score":55.07,"se":8.99,"samplingSe":7.9566,"residualSe":4.1827,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0068,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":0.45,"inputPer1mUsd":0.2,"outputPer1mUsd":1.2,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.8405,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:gpt-5.6-luna","observedHarness":"Direct API","rawScore":15.69,"zScore":0.659519,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:gpt-5.6-luna","observedHarness":"Direct API","rawScore":75.5,"zScore":0.393835,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"}]},{"id":"zhipu:glm-5.2:thinking:direct:native","provider":"zhipu","modelSlug":"glm-5.2","displayName":"GLM-5.2","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.320873,"shrinkageFactor":0.647004,"score":54.81,"se":8.99,"samplingSe":7.9519,"residualSe":4.1854,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0316,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":1048576,"releaseDate":"2026-07-02","paretoProb":0.2065,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:zai-org/GLM-5.2","observedHarness":"Direct API","rawScore":15.32,"zScore":0.428165,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:zai-org/GLM-5.2","observedHarness":"Direct API","rawScore":77.9,"zScore":0.571715,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"}]},{"id":"zhipu:glm-5.2:max:direct:native","provider":"zhipu","modelSlug":"glm-5.2","displayName":"GLM-5.2","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.319609,"shrinkageFactor":0.998243,"score":54.79,"se":2.21,"samplingSe":0.5611,"residualSe":2.1391,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0316,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":1048576,"releaseDate":"2026-07-02","paretoProb":0.0036,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:GLM-5.2 (max)","observedHarness":"Direct API","rawScore":0.182,"zScore":0.207988,"seModel":"upstream_ci","ciHalfWidth":0.095,"ciLower":0.086,"ciUpper":0.276,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GLM-5.2 (Max)","observedHarness":"Direct API","rawScore":1450.87,"zScore":0.351032,"seModel":"upstream_ci","ciHalfWidth":6.8221,"ciLower":1444.0434,"ciUpper":1457.6875,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"meta:muse-spark-1.1:high:direct:native","provider":"meta","modelSlug":"muse-spark-1.1","displayName":"Muse Spark 1.1","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.310335,"shrinkageFactor":0.975446,"score":54.66,"se":2.27,"samplingSe":2.0972,"residualSe":0.8812,"coverageCount":4,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0295,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.0135,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:muse-spark-1.1","observedHarness":"Direct API","rawScore":15.97,"zScore":0.834598,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:muse-spark-1.1","observedHarness":"Direct API","rawScore":70.6,"zScore":0.030664,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Muse Spark 1.1 (high)","observedHarness":"Direct API","rawScore":0.495,"zScore":0.355305,"seModel":"upstream_ci","ciHalfWidth":0.1085,"ciLower":0.382,"ciUpper":0.599,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Muse Spark 1.1","observedHarness":"Direct API","rawScore":1443.22,"zScore":0.205217,"seModel":"upstream_ci","ciHalfWidth":7.1638,"ciLower":1436.0562,"ciUpper":1450.3838,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-opus-4.5:none:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.5","displayName":"Claude Opus 4.5","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.291179,"shrinkageFactor":0.939886,"score":54.37,"se":4.86,"samplingSe":3.2815,"residualSe":3.5853,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.15,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":200000,"releaseDate":"2025-11-24","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:claude-opus-4-5-20251101","observedHarness":"Direct API","rawScore":14.99,"zScore":0.221821,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:claude-opus-4-5-20251101","observedHarness":"Direct API","rawScore":73.1,"zScore":0.215956,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Claude Opus 4.5 (20251101)","observedHarness":"Direct API","rawScore":1460.84,"zScore":0.541069,"seModel":"upstream_ci","ciHalfWidth":6.3566,"ciLower":1454.4802,"ciUpper":1467.1934,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-5.5:medium:direct:native","provider":"openai","modelSlug":"gpt-5.5","displayName":"GPT-5.5","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.274117,"shrinkageFactor":0.946056,"score":54.11,"se":4.76,"samplingSe":3.1086,"residualSe":3.6088,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.17,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":11.25,"inputPer1mUsd":5,"outputPer1mUsd":30,"contextWindow":1050000,"releaseDate":"2026-04-23","paretoProb":0.0005,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:gpt-5.5","observedHarness":"Direct API","rawScore":15.89,"zScore":0.784576,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:gpt-5.5","observedHarness":"Direct API","rawScore":78.2,"zScore":0.59395,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GPT-5.5","observedHarness":"Direct API","rawScore":1447.54,"zScore":0.28756,"seModel":"upstream_ci","ciHalfWidth":6.5567,"ciLower":1440.9792,"ciUpper":1454.0925,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-sonnet-4.6:none:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-4.6","displayName":"Claude Sonnet 4.6","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.272545,"shrinkageFactor":0.945815,"score":54.09,"se":4.77,"samplingSe":3.1155,"residualSe":3.6079,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.09,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1000000,"releaseDate":"2026-02-17","paretoProb":0.0832,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:claude-sonnet-4-6","observedHarness":"Direct API","rawScore":15.53,"zScore":0.559474,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:claude-sonnet-4-6","observedHarness":"Direct API","rawScore":79.9,"zScore":0.719948,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Claude Sonnet 4.6","observedHarness":"Direct API","rawScore":1448.66,"zScore":0.308908,"seModel":"upstream_ci","ciHalfWidth":6.5304,"ciLower":1442.128,"ciUpper":1455.1889,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"deepseek:deepseek-v4-pro-0813:high:direct:native","provider":"deepseek","modelSlug":"deepseek-v4-pro-0813","displayName":"DeepSeek V4 Pro (0813)","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.245131,"shrinkageFactor":0.993063,"score":53.68,"se":2.4,"samplingSe":1.1148,"residualSe":2.128,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.02904,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":1.98,"inputPer1mUsd":1.32,"outputPer1mUsd":3.96,"contextWindow":1048576,"releaseDate":"2026-08-13","paretoProb":1,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:DeepSeek V4 Pro (high)","observedHarness":"Direct API","rawScore":0.14,"zScore":0.18822,"seModel":"upstream_ci","ciHalfWidth":0.0875,"ciLower":0.054,"ciUpper":0.229,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:DeepSeek-V4-Pro (High, 20260813)","observedHarness":"Direct API","rawScore":1444.56,"zScore":0.230758,"seModel":"upstream_ci","ciHalfWidth":14.0346,"ciLower":1430.5281,"ciUpper":1458.5972,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-5.5:high:direct:native","provider":"openai","modelSlug":"gpt-5.5","displayName":"GPT-5.5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.178907,"shrinkageFactor":0.994893,"score":52.68,"se":5.99,"samplingSe":0.9564,"residualSe":5.9174,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.17,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":11.25,"inputPer1mUsd":5,"outputPer1mUsd":30,"contextWindow":1050000,"releaseDate":"2026-04-23","paretoProb":0.0014,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GPT-5.5 (High)","observedHarness":"Direct API","rawScore":1450.08,"zScore":0.335974,"seModel":"upstream_ci","ciHalfWidth":6.5733,"ciLower":1443.5079,"ciUpper":1456.6546,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-6.1-terra:max:direct:native","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.160949,"shrinkageFactor":0.972978,"score":52.41,"se":6.19,"samplingSe":2.2001,"residualSe":5.7871,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.03,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.2726,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GPT-6.1 Terra (Max)","observedHarness":"Direct API","rawScore":1449.12,"zScore":0.317676,"seModel":"upstream_ci","ciHalfWidth":15.2904,"ciLower":1433.832,"ciUpper":1464.4127,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-sonnet-4.5:none:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-4.5","displayName":"Claude Sonnet 4.5","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.143512,"shrinkageFactor":0.937497,"score":52.15,"se":4.9,"samplingSe":3.3461,"residualSe":3.5762,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.09,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1000000,"releaseDate":"2025-09-29","paretoProb":0.0447,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:claude-sonnet-4.5","observedHarness":"Direct API","rawScore":14.44,"zScore":-0.122084,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:claude-sonnet-4.5","observedHarness":"Direct API","rawScore":71.4,"zScore":0.089957,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Claude Sonnet 4.5 (20250929)","observedHarness":"Direct API","rawScore":1454.55,"zScore":0.421176,"seModel":"upstream_ci","ciHalfWidth":6.0173,"ciLower":1448.533,"ciUpper":1460.5677,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-3-flash:minimal:direct:native","provider":"google","modelSlug":"gemini-3-flash","displayName":"Gemini 3 Flash","effort":"minimal","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.117407,"shrinkageFactor":0.995953,"score":51.76,"se":5.98,"samplingSe":0.8515,"residualSe":5.9237,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.017,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":1.125,"inputPer1mUsd":0.5,"outputPer1mUsd":3,"contextWindow":1048576,"releaseDate":"2025-12-15","paretoProb":0.4612,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Gemini 3 Flash (Minimal)","observedHarness":"Direct API","rawScore":1446.84,"zScore":0.274217,"seModel":"upstream_ci","ciHalfWidth":5.8487,"ciLower":1440.9962,"ciUpper":1452.6937,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"moonshot:kimi-k2.6:thinking:direct:native","provider":"moonshot","modelSlug":"kimi-k2.6","displayName":"Kimi K2.6","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.103653,"shrinkageFactor":0.975348,"score":51.55,"se":2.28,"samplingSe":2.1014,"residualSe":0.8811,"coverageCount":4,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0255,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":1.7125,"inputPer1mUsd":0.95,"outputPer1mUsd":4,"contextWindow":262144,"releaseDate":"2026-04-10","paretoProb":1,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:moonshotai/Kimi-K2.6","observedHarness":"Direct API","rawScore":14.93,"zScore":0.184305,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:moonshotai/Kimi-K2.6","observedHarness":"Direct API","rawScore":78.5,"zScore":0.616185,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Kimi K2.6","observedHarness":"Direct API","rawScore":-0.191,"zScore":0.032432,"seModel":"upstream_ci","ciHalfWidth":0.1155,"ciLower":-0.315,"ciUpper":-0.084,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Kimi K2.6","observedHarness":"Direct API","rawScore":1432.51,"zScore":0.001075,"seModel":"upstream_ci","ciHalfWidth":8.253,"ciLower":1424.2588,"ciUpper":1440.7647,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"spacexai:grok-4.20:none:direct:native","provider":"spacexai","modelSlug":"grok-4.20","displayName":"Grok 4.20","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.097108,"shrinkageFactor":0.93018,"score":51.46,"se":5.01,"samplingSe":3.5365,"residualSe":3.5483,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0225,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":1.5625,"inputPer1mUsd":1.25,"outputPer1mUsd":2.5,"contextWindow":2000000,"releaseDate":"2026-03-09","paretoProb":0.5151,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:grok-4.20-beta","observedHarness":"Direct API","rawScore":13.14,"zScore":-0.934951,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:grok-4.20-beta","observedHarness":"Direct API","rawScore":68.5,"zScore":-0.12498,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Grok 4.20 Beta 1","observedHarness":"Direct API","rawScore":1463.59,"zScore":0.593486,"seModel":"upstream_ci","ciHalfWidth":9.724,"ciLower":1453.865,"ciUpper":1473.313,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"spacexai:grok-4.20:thinking:direct:native","provider":"spacexai","modelSlug":"grok-4.20","displayName":"Grok 4.20","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.088667,"shrinkageFactor":0.994579,"score":51.33,"se":6,"samplingSe":0.9854,"residualSe":5.9156,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0225,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":1.5625,"inputPer1mUsd":1.25,"outputPer1mUsd":2.5,"contextWindow":2000000,"releaseDate":"2026-03-09","paretoProb":0.4848,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Grok 4.20 Beta (Reasoning, 0309)","observedHarness":"Direct API","rawScore":1445.32,"zScore":0.245245,"seModel":"upstream_ci","ciHalfWidth":6.7738,"ciLower":1438.5502,"ciUpper":1452.0979,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"meta:muse-spark-1.2:xhigh:direct:native","provider":"meta","modelSlug":"muse-spark-1.2","displayName":"Muse Spark 1.2","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.086519,"shrinkageFactor":0.986548,"score":51.3,"se":6.07,"samplingSe":1.5523,"residualSe":5.8678,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0295,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.0444,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Muse Spark 1.2 (xHigh)","observedHarness":"Direct API","rawScore":1445.17,"zScore":0.242385,"seModel":"upstream_ci","ciHalfWidth":10.7137,"ciLower":1434.453,"ciUpper":1455.8804,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-5.4:none:direct:native","provider":"openai","modelSlug":"gpt-5.4","displayName":"GPT-5.4","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.081154,"shrinkageFactor":0.945404,"score":51.22,"se":4.77,"samplingSe":3.1273,"residualSe":3.6064,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.085,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":5.625,"inputPer1mUsd":2.5,"outputPer1mUsd":15,"contextWindow":1050000,"releaseDate":"2026-03-05","paretoProb":0.0014,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:gpt-5.4","observedHarness":"Direct API","rawScore":15.75,"zScore":0.697036,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:gpt-5.4","observedHarness":"Direct API","rawScore":78.3,"zScore":0.601361,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GPT-5.4","observedHarness":"Direct API","rawScore":1432.71,"zScore":0.004887,"seModel":"upstream_ci","ciHalfWidth":6.8107,"ciLower":1425.9004,"ciUpper":1439.5217,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-5.4-mini:none:direct:native","provider":"openai","modelSlug":"gpt-5.4-mini","displayName":"GPT-5.4 Mini","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.044428,"shrinkageFactor":0.623107,"score":50.67,"se":9.15,"samplingSe":8.2167,"residualSe":4.0308,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0255,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":1.6875,"inputPer1mUsd":0.75,"outputPer1mUsd":4.5,"contextWindow":400000,"releaseDate":"2026-03-17","paretoProb":0.3352,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:gpt-5.4-mini","observedHarness":"Direct API","rawScore":14.51,"zScore":-0.078314,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:gpt-5.4-mini","observedHarness":"Direct API","rawScore":73,"zScore":0.208544,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"}]},{"id":"openai:gpt-5:medium:direct:native","provider":"openai","modelSlug":"gpt-5","displayName":"GPT-5","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.043744,"shrinkageFactor":0.622925,"score":50.66,"se":9.15,"samplingSe":8.2186,"residualSe":4.0296,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0525,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":3.4375,"inputPer1mUsd":1.25,"outputPer1mUsd":10,"contextWindow":400000,"releaseDate":"2025-08-07","paretoProb":0.1986,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:gpt-5-2025-08-07","observedHarness":"Direct API","rawScore":14.53,"zScore":-0.065808,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:gpt-5-2025-08-07","observedHarness":"Direct API","rawScore":72.8,"zScore":0.193721,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"}]},{"id":"anthropic:claude-sonnet-5:high:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-5","displayName":"Claude Sonnet 5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.039584,"shrinkageFactor":0.943542,"score":50.59,"se":4.8,"samplingSe":3.1802,"residualSe":3.5992,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.06,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-07-15","paretoProb":0.0442,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:claude-sonnet-5","observedHarness":"Direct API","rawScore":15.26,"zScore":0.390648,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:claude-sonnet-5","observedHarness":"Direct API","rawScore":78.3,"zScore":0.601361,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Claude Sonnet 5 (High)","observedHarness":"Direct API","rawScore":1433.13,"zScore":0.012893,"seModel":"upstream_ci","ciHalfWidth":6.8677,"ciLower":1426.2648,"ciUpper":1440.0002,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-3-flash:high:direct:native","provider":"google","modelSlug":"gemini-3-flash","displayName":"Gemini 3 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.021632,"shrinkageFactor":0.943494,"score":50.32,"se":5.42,"samplingSe":3.1815,"residualSe":4.3859,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.017,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":1.125,"inputPer1mUsd":0.5,"outputPer1mUsd":3,"contextWindow":1048576,"releaseDate":"2025-12-15","paretoProb":0.2972,"benchmarks":[{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:gemini-3-flash-preview","observedHarness":"Direct API","rawScore":55.7,"zScore":-1.073672,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Gemini 3 Flash","observedHarness":"Direct API","rawScore":1456.39,"zScore":0.456248,"seModel":"upstream_ci","ciHalfWidth":9.3151,"ciLower":1447.0703,"ciUpper":1465.7005,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"zhipu:glm-5:thinking:direct:native","provider":"zhipu","modelSlug":"glm-5","displayName":"GLM-5","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.0172,"shrinkageFactor":0.934228,"score":50.26,"se":4.95,"samplingSe":3.4325,"residualSe":3.5637,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0228,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":1.55,"inputPer1mUsd":1,"outputPer1mUsd":3.2,"contextWindow":204800,"releaseDate":"2026-02-01","paretoProb":0.6021,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:zai-org/GLM-5","observedHarness":"Direct API","rawScore":14.12,"zScore":-0.322174,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:zai-org/GLM-5","observedHarness":"Direct API","rawScore":70.9,"zScore":0.052899,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GLM-5","observedHarness":"Direct API","rawScore":1447.03,"zScore":0.277839,"seModel":"upstream_ci","ciHalfWidth":8.9909,"ciLower":1438.0436,"ciUpper":1456.0253,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-3.1-pro:high:direct:native","provider":"google","modelSlug":"gemini-3.1-pro","displayName":"Gemini 3.1 Pro","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.000491,"shrinkageFactor":0.97292,"score":49.99,"se":2.37,"samplingSe":2.2025,"residualSe":0.8789,"coverageCount":4,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.068,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":4.5,"inputPer1mUsd":2,"outputPer1mUsd":12,"contextWindow":1048576,"releaseDate":"2026-02-19","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:gemini-3.1-pro-preview","observedHarness":"Direct API","rawScore":14.23,"zScore":-0.253393,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:gemini-3.1-pro-preview","observedHarness":"Direct API","rawScore":68.2,"zScore":-0.147215,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Gemini 3.1 Pro Preview","observedHarness":"Direct API","rawScore":-2.441,"zScore":-1.026554,"seModel":"upstream_ci","ciHalfWidth":0.0985,"ciLower":-2.533,"ciUpper":-2.336,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Gemini 3.1 Pro Preview","observedHarness":"Direct API","rawScore":1481.52,"zScore":0.935247,"seModel":"upstream_ci","ciHalfWidth":5.2241,"ciLower":1476.2921,"ciUpper":1486.7402,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-5.4:high:direct:native","provider":"openai","modelSlug":"gpt-5.4","displayName":"GPT-5.4","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.002468,"shrinkageFactor":0.994293,"score":49.96,"se":6,"samplingSe":1.0111,"residualSe":5.9139,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.085,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":5.625,"inputPer1mUsd":2.5,"outputPer1mUsd":15,"contextWindow":1050000,"releaseDate":"2026-03-05","paretoProb":0.0023,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GPT-5.4 (High)","observedHarness":"Direct API","rawScore":1440.51,"zScore":0.153562,"seModel":"upstream_ci","ciHalfWidth":6.9513,"ciLower":1433.5597,"ciUpper":1447.4622,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"zhipu:glm-5.1:thinking:direct:native","provider":"zhipu","modelSlug":"glm-5.1","displayName":"GLM-5.1","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.039409,"shrinkageFactor":0.973839,"score":49.41,"se":2.34,"samplingSe":2.1648,"residualSe":0.8797,"coverageCount":4,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0316,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":204800,"releaseDate":"2026-04-15","paretoProb":0.0001,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:zai-org/GLM-5.1","observedHarness":"Direct API","rawScore":14.44,"zScore":-0.122084,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:zai-org/GLM-5.1","observedHarness":"Direct API","rawScore":73.5,"zScore":0.245602,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:GLM-5.1","observedHarness":"Direct API","rawScore":-1.302,"zScore":-0.490472,"seModel":"upstream_ci","ciHalfWidth":0.176,"ciLower":-1.471,"ciUpper":-1.119,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GLM-5.1","observedHarness":"Direct API","rawScore":1446.29,"zScore":0.263734,"seModel":"upstream_ci","ciHalfWidth":6.243,"ciLower":1440.0478,"ciUpper":1452.5338,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-3.5-flash:high:direct:native","provider":"google","modelSlug":"gemini-3.5-flash","displayName":"Gemini 3.5 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.044784,"shrinkageFactor":0.982866,"score":49.33,"se":2.28,"samplingSe":1.7519,"residualSe":1.4623,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.051,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":3.375,"inputPer1mUsd":1.5,"outputPer1mUsd":9,"contextWindow":1048576,"releaseDate":"2026-04-12","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:gemini-3.5-flash","observedHarness":"Direct API","rawScore":71.8,"zScore":0.119604,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Gemini 3.5 Flash","observedHarness":"Direct API","rawScore":-2.253,"zScore":-0.93807,"seModel":"upstream_ci","ciHalfWidth":0.08,"ciLower":-2.335,"ciUpper":-2.175,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Gemini 3.5 Flash (High)","observedHarness":"Direct API","rawScore":1465.29,"zScore":0.62589,"seModel":"upstream_ci","ciHalfWidth":7.2263,"ciLower":1458.0645,"ciUpper":1472.5171,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-3.5-flash-lite:minimal:direct:native","provider":"google","modelSlug":"gemini-3.5-flash-lite","displayName":"Gemini 3.5 Flash-Lite","effort":"minimal","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.086796,"shrinkageFactor":0.950915,"score":48.7,"se":5.32,"samplingSe":2.9653,"residualSe":4.4204,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.013,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":0.85,"inputPer1mUsd":0.3,"outputPer1mUsd":2.5,"contextWindow":1048576,"releaseDate":"2026-04-12","paretoProb":1,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:gemini-3.5-flash-lite","observedHarness":"Direct API","rawScore":14.47,"zScore":-0.103325,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Gemini 3.5 Flash-Lite","observedHarness":"Direct API","rawScore":1436.57,"zScore":0.078462,"seModel":"upstream_ci","ciHalfWidth":8.036,"ciLower":1428.5369,"ciUpper":1444.6088,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"zhipu:glm-5.3-flash:max:direct:native","provider":"zhipu","modelSlug":"glm-5.3-flash","displayName":"GLM-5.3 Flash","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.10901,"shrinkageFactor":0.994403,"score":48.36,"se":6,"samplingSe":1.0013,"residualSe":5.9145,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0035,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":0.2375,"inputPer1mUsd":0.15,"outputPer1mUsd":0.5,"contextWindow":1048576,"releaseDate":"2026-09-28","paretoProb":1,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GLM-5.3-Flash","observedHarness":"Direct API","rawScore":1434.89,"zScore":0.04644,"seModel":"upstream_ci","ciHalfWidth":6.8837,"ciLower":1428.0019,"ciUpper":1441.7692,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"spacexai:grok-4.7:xhigh:direct:native","provider":"spacexai","modelSlug":"grok-4.7","displayName":"Grok 4.7","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.168112,"shrinkageFactor":0.985385,"score":47.48,"se":4.86,"samplingSe":1.618,"residualSe":4.5806,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.044,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-10-18","paretoProb":0.0531,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Grok 4.7 (xHigh)","observedHarness":"Direct API","rawScore":1421.86,"zScore":-0.201923,"seModel":"upstream_ci","ciHalfWidth":9.1852,"ciLower":1412.6728,"ciUpper":1431.0431,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:Grok 4.7 (xHigh reasoning)","observedHarness":"Direct API","rawScore":60.8,"zScore":0.670599,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"openai:gpt-5.6-terra:xhigh:direct:native","provider":"openai","modelSlug":"gpt-5.6-terra","displayName":"GPT-5.6 Terra","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.195322,"shrinkageFactor":0.994082,"score":47.07,"se":6,"samplingSe":1.0296,"residualSe":5.9126,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.068,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":4.5,"inputPer1mUsd":2,"outputPer1mUsd":12,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0032,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GPT-5.6 Terra (xHigh)","observedHarness":"Direct API","rawScore":1430.33,"zScore":-0.040477,"seModel":"upstream_ci","ciHalfWidth":7.0792,"ciLower":1423.2468,"ciUpper":1437.4052,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-6-sol:max:direct:native","provider":"openai","modelSlug":"gpt-6-sol","displayName":"GPT-6 Sol","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.230998,"shrinkageFactor":0.988633,"score":46.54,"se":6.05,"samplingSe":1.427,"residualSe":5.8802,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.06,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0019,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GPT-6 Sol (Max)","observedHarness":"Direct API","rawScore":1428.33,"zScore":-0.078599,"seModel":"upstream_ci","ciHalfWidth":9.8382,"ciLower":1418.4962,"ciUpper":1438.1726,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-5.1:high:direct:native","provider":"openai","modelSlug":"gpt-5.1","displayName":"GPT-5.1","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.240554,"shrinkageFactor":0.992128,"score":46.39,"se":6.02,"samplingSe":1.1875,"residualSe":5.901,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0525,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":3.4375,"inputPer1mUsd":1.25,"outputPer1mUsd":10,"contextWindow":400000,"releaseDate":"2025-11-13","paretoProb":0.0368,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GPT-5.1 (High)","observedHarness":"Direct API","rawScore":1427.9,"zScore":-0.086795,"seModel":"upstream_ci","ciHalfWidth":8.173,"ciLower":1419.7259,"ciUpper":1436.0719,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"spacexai:grok-4.6:high:direct:native","provider":"spacexai","modelSlug":"grok-4.6","displayName":"Grok 4.6","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.488753,"shrinkageFactor":0.997431,"score":42.67,"se":2.24,"samplingSe":0.6783,"residualSe":2.1374,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.044,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-08-25","paretoProb":0,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Grok 4.6 (high)","observedHarness":"Direct API","rawScore":-3.173,"zScore":-1.371078,"seModel":"upstream_ci","ciHalfWidth":0.1565,"ciLower":-3.327,"ciUpper":-3.014,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Grok 4.6 (High)","observedHarness":"Direct API","rawScore":1442.46,"zScore":0.190731,"seModel":"upstream_ci","ciHalfWidth":7.9479,"ciLower":1434.5143,"ciUpper":1450.4101,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-5.6-luna:xhigh:direct:native","provider":"openai","modelSlug":"gpt-5.6-luna","displayName":"GPT-5.6 Luna","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.568876,"shrinkageFactor":0.993184,"score":41.47,"se":6.01,"samplingSe":1.105,"residualSe":5.9073,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0068,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":0.45,"inputPer1mUsd":0.2,"outputPer1mUsd":1.2,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0888,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GPT-5.6 Luna (xHigh)","observedHarness":"Direct API","rawScore":1410.58,"zScore":-0.416929,"seModel":"upstream_ci","ciHalfWidth":7.6009,"ciLower":1402.977,"ciUpper":1418.1789,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-5.2:none:direct:native","provider":"openai","modelSlug":"gpt-5.2","displayName":"GPT-5.2","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.597608,"shrinkageFactor":0.938609,"score":41.04,"se":4.88,"samplingSe":3.3162,"residualSe":3.5804,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0735,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":4.8125,"inputPer1mUsd":1.75,"outputPer1mUsd":14,"contextWindow":400000,"releaseDate":"2025-12-11","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:gpt-5.2","observedHarness":"Direct API","rawScore":14.79,"zScore":0.096765,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:gpt-5.2","observedHarness":"Direct API","rawScore":71.6,"zScore":0.104781,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GPT-5.2","observedHarness":"Direct API","rawScore":1390.32,"zScore":-0.803102,"seModel":"upstream_ci","ciHalfWidth":6.1244,"ciLower":1384.196,"ciUpper":1396.4449,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-6-luna:max:direct:native","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.606553,"shrinkageFactor":0.995611,"score":40.9,"se":5.99,"samplingSe":0.8867,"residualSe":5.9217,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.003,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":1,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GPT-6 Luna (Max)","observedHarness":"Direct API","rawScore":1408.69,"zScore":-0.452954,"seModel":"upstream_ci","ciHalfWidth":6.0922,"ciLower":1402.5938,"ciUpper":1414.7781,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-5.1:none:direct:native","provider":"openai","modelSlug":"gpt-5.1","displayName":"GPT-5.1","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.621038,"shrinkageFactor":0.9926,"score":40.68,"se":6.02,"samplingSe":1.1514,"residualSe":5.9038,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0525,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":3.4375,"inputPer1mUsd":1.25,"outputPer1mUsd":10,"contextWindow":400000,"releaseDate":"2025-11-13","paretoProb":0.0024,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GPT-5.1","observedHarness":"Direct API","rawScore":1407.8,"zScore":-0.469918,"seModel":"upstream_ci","ciHalfWidth":7.9222,"ciLower":1399.8745,"ciUpper":1415.7188,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-5.4-mini:high:direct:native","provider":"openai","modelSlug":"gpt-5.4-mini","displayName":"GPT-5.4 Mini","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.743501,"shrinkageFactor":0.994095,"score":38.85,"se":6,"samplingSe":1.0285,"residualSe":5.9127,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0255,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":1.6875,"inputPer1mUsd":0.75,"outputPer1mUsd":4.5,"contextWindow":400000,"releaseDate":"2026-03-17","paretoProb":0.0148,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GPT-5.4 mini (High)","observedHarness":"Direct API","rawScore":1401.4,"zScore":-0.591908,"seModel":"upstream_ci","ciHalfWidth":7.0712,"ciLower":1394.3336,"ciUpper":1408.4761,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:o3:medium:direct:native","provider":"openai","modelSlug":"o3","displayName":"o3","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.795326,"shrinkageFactor":0.933957,"score":38.07,"se":4.95,"samplingSe":3.4395,"residualSe":3.5627,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.052,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":3.5,"inputPer1mUsd":2,"outputPer1mUsd":8,"contextWindow":200000,"releaseDate":"2025-04-16","paretoProb":0.0012,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:o3","observedHarness":"Direct API","rawScore":14.65,"zScore":0.009225,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:o3","observedHarness":"Direct API","rawScore":62.7,"zScore":-0.554856,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:o3 (2025-04-16)","observedHarness":"Direct API","rawScore":1382.97,"zScore":-0.943199,"seModel":"upstream_ci","ciHalfWidth":7.3262,"ciLower":1375.6477,"ciUpper":1390.3001,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"spacexai:grok-4.5:high:direct:native","provider":"spacexai","modelSlug":"grok-4.5","displayName":"Grok 4.5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.801115,"shrinkageFactor":0.981565,"score":37.98,"se":2.33,"samplingSe":1.8172,"residualSe":1.4603,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.044,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-06-30","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:grok-4.5","observedHarness":"Direct API","rawScore":14.11,"zScore":-0.328427,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Grok 4.5 (high)","observedHarness":"Direct API","rawScore":-5.35,"zScore":-2.395705,"seModel":"upstream_ci","ciHalfWidth":0.0905,"ciLower":-5.443,"ciUpper":-5.262,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Grok 4.5","observedHarness":"Direct API","rawScore":1450.98,"zScore":0.353129,"seModel":"upstream_ci","ciHalfWidth":7.6839,"ciLower":1443.2933,"ciUpper":1458.6612,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-haiku-4.5:none:direct:native","provider":"anthropic","modelSlug":"claude-haiku-4.5","displayName":"Claude Haiku 4.5","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.873071,"shrinkageFactor":0.951193,"score":36.9,"se":5.32,"samplingSe":2.9568,"residualSe":4.4217,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.03,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":200000,"releaseDate":"2025-10-15","paretoProb":0.8988,"benchmarks":[{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:claude-haiku-4.5","observedHarness":"Direct API","rawScore":65,"zScore":-0.384388,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Claude Haiku 4.5 (20251001)","observedHarness":"Direct API","rawScore":1386.62,"zScore":-0.873627,"seModel":"upstream_ci","ciHalfWidth":4.6292,"ciLower":1381.9874,"ciUpper":1391.2458,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-5:high:direct:native","provider":"openai","modelSlug":"gpt-5","displayName":"GPT-5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.969712,"shrinkageFactor":0.951212,"score":35.45,"se":5.32,"samplingSe":2.9563,"residualSe":4.4218,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0525,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":3.4375,"inputPer1mUsd":1.25,"outputPer1mUsd":10,"contextWindow":400000,"releaseDate":"2025-08-07","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:gpt-5-2025-08-07-high-reasoning-high-reasoning","observedHarness":"Direct API","rawScore":73.1,"zScore":0.215956,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GPT-5 (High)","observedHarness":"Direct API","rawScore":1372.46,"zScore":-1.143528,"seModel":"upstream_ci","ciHalfWidth":9.7136,"ciLower":1362.7417,"ciUpper":1382.169,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"spacexai:grok-4.3:low:direct:native","provider":"spacexai","modelSlug":"grok-4.3","displayName":"Grok 4.3","effort":"low","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.990958,"shrinkageFactor":0.998055,"score":35.14,"se":2.22,"samplingSe":0.5903,"residualSe":2.1387,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0225,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":1.5625,"inputPer1mUsd":1.25,"outputPer1mUsd":2.5,"contextWindow":1000000,"releaseDate":"2026-05-01","paretoProb":0.0001,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Grok 4.3","observedHarness":"Direct API","rawScore":-4.516,"zScore":-2.003175,"seModel":"upstream_ci","ciHalfWidth":0.176,"ciLower":-4.703,"ciUpper":-4.351,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Grok 4.3","observedHarness":"Direct API","rawScore":1421.82,"zScore":-0.202685,"seModel":"upstream_ci","ciHalfWidth":6.5099,"ciLower":1415.3129,"ciUpper":1428.3326,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"alibaba:qwen3.7-max:thinking:direct:native","provider":"alibaba","modelSlug":"qwen3.7-max","displayName":"Qwen3.7 Max","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.995183,"shrinkageFactor":0.998923,"score":35.07,"se":6.62,"samplingSe":0.4392,"residualSe":6.6092,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.055,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":3.75,"inputPer1mUsd":2.5,"outputPer1mUsd":7.5,"contextWindow":1000000,"releaseDate":"2026-05-20","paretoProb":0,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Qwen 3.7 Max","observedHarness":"Direct API","rawScore":-2.931,"zScore":-1.257178,"seModel":"upstream_ci","ciHalfWidth":0.122,"ciLower":-3.065,"ciUpper":-2.821,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"}]},{"id":"openai:gpt-5.2:high:direct:native","provider":"openai","modelSlug":"gpt-5.2","displayName":"GPT-5.2","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-1.038723,"shrinkageFactor":0.993479,"score":34.42,"se":6.01,"samplingSe":1.0808,"residualSe":5.909,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0735,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":4.8125,"inputPer1mUsd":1.75,"outputPer1mUsd":14,"contextWindow":400000,"releaseDate":"2025-12-11","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GPT-5.2 (High)","observedHarness":"Direct API","rawScore":1385.78,"zScore":-0.889638,"seModel":"upstream_ci","ciHalfWidth":7.4331,"ciLower":1378.3474,"ciUpper":1393.2137,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-haiku-5.5:high:direct:native","provider":"anthropic","modelSlug":"claude-haiku-5.5","displayName":"Claude Haiku 5.5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-1.525903,"shrinkageFactor":0.986186,"score":27.11,"se":6.07,"samplingSe":1.5731,"residualSe":5.8657,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.003,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1000000,"releaseDate":"2026-10-07","paretoProb":1,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Claude Haiku 5.5 (High)","observedHarness":"Direct API","rawScore":1359.39,"zScore":-1.392654,"seModel":"upstream_ci","ciHalfWidth":10.8591,"ciLower":1348.5271,"ciUpper":1370.2452,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-5.4-nano:high:direct:native","provider":"openai","modelSlug":"gpt-5.4-nano","displayName":"GPT-5.4 Nano","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-1.970431,"shrinkageFactor":0.993757,"score":20.44,"se":6,"samplingSe":1.0575,"residualSe":5.9107,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.007,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":0.4625,"inputPer1mUsd":0.2,"outputPer1mUsd":1.25,"contextWindow":400000,"releaseDate":"2026-03-17","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GPT-5.4 nano (High)","observedHarness":"Direct API","rawScore":1336.61,"zScore":-1.82686,"seModel":"upstream_ci","ciHalfWidth":7.2725,"ciLower":1329.3416,"ciUpper":1343.8865,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]}],"provisionalConfigs":[{"id":"google:gemini-3-pro:high:direct:native","provider":"google","modelSlug":"gemini-3-pro","displayName":"Gemini 3 Pro","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.510896,"shrinkageFactor":0.936129,"score":57.66,"se":4.92,"samplingSe":3.3825,"residualSe":3.571,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-11-18","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:gemini-3-pro-preview","observedHarness":"Direct API","rawScore":14.85,"zScore":0.134282,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:gemini-3-pro-preview","observedHarness":"Direct API","rawScore":68.8,"zScore":-0.102746,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Gemini 3 Pro","observedHarness":"Direct API","rawScore":1484.21,"zScore":0.986521,"seModel":"upstream_ci","ciHalfWidth":8.1698,"ciLower":1476.0436,"ciUpper":1492.3833,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"meta:muse-spark:high:direct:native","provider":"meta","modelSlug":"muse-spark","displayName":"Muse Spark","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.453144,"shrinkageFactor":0.977837,"score":56.8,"se":6.15,"samplingSe":1.9925,"residualSe":5.816,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Muse Spark","observedHarness":"Direct API","rawScore":1464.8,"zScore":0.61655,"seModel":"upstream_ci","ciHalfWidth":13.813,"ciLower":1450.9874,"ciUpper":1478.6133,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"alibaba:qwen3.8-max:xhigh:direct:native","provider":"alibaba","modelSlug":"qwen3.8-max","displayName":"Qwen3.8 Max","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.378483,"shrinkageFactor":0.990559,"score":55.68,"se":1.97,"samplingSe":1.3004,"residualSe":1.4737,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-08-03","paretoProb":0,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Qwen 3.8 Max¶","observedHarness":"Direct API","rawScore":-0.174,"zScore":0.040433,"seModel":"upstream_ci","ciHalfWidth":0.097,"ciLower":-0.269,"ciUpper":-0.075,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Qwen3.8-Max","observedHarness":"Direct API","rawScore":1472.8,"zScore":0.769037,"seModel":"upstream_ci","ciHalfWidth":7.7799,"ciLower":1465.0208,"ciUpper":1480.5806,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","sourceId":"surge","sourceRowId":"surge:complex-constraints:Qwen 3.8 Max (xHigh reasoning)","observedHarness":"Direct API","rawScore":46.3,"zScore":-0.236603,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/complex-constraints"}]},{"id":"openai:gpt-5.5-instant:none:direct:native","provider":"openai","modelSlug":"gpt-5.5-instant","displayName":"GPT-5.5 Instant","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.265945,"shrinkageFactor":0.988868,"score":53.99,"se":6.05,"samplingSe":1.4121,"residualSe":5.8816,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-06-01","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:GPT-5.5 Instant","observedHarness":"Direct API","rawScore":1454.7,"zScore":0.424035,"seModel":"upstream_ci","ciHalfWidth":9.735,"ciLower":1444.9621,"ciUpper":1464.432,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"alibaba:qwen3.8-2.4t-a95b:xhigh:direct:native","provider":"alibaba","modelSlug":"qwen3.8-2.4t-a95b","displayName":"Qwen3.8 2.4T A95B","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.242615,"shrinkageFactor":0.4693,"score":53.64,"se":10.58,"samplingSe":9.7501,"residualSe":4.1107,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-12","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:Qwen/Qwen3.8-2.4T-A95B","observedHarness":"Direct API","rawScore":15.29,"zScore":0.409406,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"}]},{"id":"google:gemini-3.6-flash:medium:direct:native","provider":"google","modelSlug":"gemini-3.6-flash","displayName":"Gemini 3.6 Flash","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"writing","theta":0.021513,"shrinkageFactor":0.446201,"score":50.32,"se":10.7,"samplingSe":9.9601,"residualSe":3.9083,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":0.0225,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":1.5,"inputPer1mUsd":0.75,"outputPer1mUsd":3.75,"contextWindow":1048576,"releaseDate":"2026-05-05","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:gemini-3.6-flash","observedHarness":"Direct API","rawScore":14.51,"zScore":-0.078314,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"}]},{"id":"deepseek:deepseek-v4.1-flash:max:direct:native","provider":"deepseek","modelSlug":"deepseek-v4.1-flash","displayName":"DeepSeek V4.1 Flash","effort":"max","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.010606,"shrinkageFactor":0.983004,"score":49.84,"se":6.1,"samplingSe":1.7449,"residualSe":5.8467,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-10","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:DeepSeek-V4.1-Flash-20260910 (Max)","observedHarness":"Direct API","rawScore":1439.97,"zScore":0.143269,"seModel":"upstream_ci","ciHalfWidth":12.0645,"ciLower":1427.9008,"ciUpper":1452.0299,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"deepseek:deepseek-v4-pro:high:direct:native","provider":"deepseek","modelSlug":"deepseek-v4-pro","displayName":"DeepSeek V4 Pro (0424)","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.014851,"shrinkageFactor":0.993627,"score":49.78,"se":6.01,"samplingSe":1.0685,"residualSe":5.9099,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-04-24","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:DeepSeek-V4-Pro Preview (High)","observedHarness":"Direct API","rawScore":1439.85,"zScore":0.140982,"seModel":"upstream_ci","ciHalfWidth":7.348,"ciLower":1432.4977,"ciUpper":1447.1937,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"deepseek:deepseek-v4.1-flash:high:direct:native","provider":"deepseek","modelSlug":"deepseek-v4.1-flash","displayName":"DeepSeek V4.1 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.029187,"shrinkageFactor":0.92543,"score":49.56,"se":5.44,"samplingSe":3.6548,"residualSe":4.0248,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-10","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:deepseek-v4.1-flash","observedHarness":"Direct API","rawScore":14.05,"zScore":-0.365944,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:deepseek-v4.1-flash","observedHarness":"Direct API","rawScore":74,"zScore":0.28266,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:DeepSeek V4.1 Flash (high)","observedHarness":"Direct API","rawScore":-0.797,"zScore":-0.252788,"seModel":"upstream_ci","ciHalfWidth":0.0905,"ciLower":-0.882,"ciUpper":-0.701,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"}]},{"id":"spacexai:grok-4.1:none:direct:native","provider":"spacexai","modelSlug":"grok-4.1","displayName":"Grok 4.1","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.182094,"shrinkageFactor":0.994713,"score":47.27,"se":6,"samplingSe":0.9732,"residualSe":5.9164,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-11-20","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Grok 4.1","observedHarness":"Direct API","rawScore":1431.04,"zScore":-0.026944,"seModel":"upstream_ci","ciHalfWidth":6.6889,"ciLower":1424.3535,"ciUpper":1437.7313,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"alibaba:qwen3.7-plus:thinking:direct:native","provider":"alibaba","modelSlug":"qwen3.7-plus","displayName":"Qwen3.7 Plus","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.185488,"shrinkageFactor":0.993081,"score":47.22,"se":6.01,"samplingSe":1.1133,"residualSe":5.9067,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-10","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Qwen3.7-Plus","observedHarness":"Direct API","rawScore":1430.83,"zScore":-0.030947,"seModel":"upstream_ci","ciHalfWidth":7.6582,"ciLower":1423.1745,"ciUpper":1438.4908,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"spacexai:grok-4.1:thinking:direct:native","provider":"spacexai","modelSlug":"grok-4.1","displayName":"Grok 4.1","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.276003,"shrinkageFactor":0.994838,"score":45.86,"se":5.99,"samplingSe":0.9616,"residualSe":5.9171,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-11-20","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Grok 4.1 (Thinking)","observedHarness":"Direct API","rawScore":1426.09,"zScore":-0.121295,"seModel":"upstream_ci","ciHalfWidth":6.6088,"ciLower":1419.4779,"ciUpper":1432.6955,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-5.4-nano:none:direct:native","provider":"openai","modelSlug":"gpt-5.4-nano","displayName":"GPT-5.4 Nano","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.325966,"shrinkageFactor":0.415341,"score":45.11,"se":10.86,"samplingSe":10.2338,"residualSe":3.638,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":0.007,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":0.4625,"inputPer1mUsd":0.2,"outputPer1mUsd":1.25,"contextWindow":400000,"releaseDate":"2026-03-17","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:gpt-5.4-nano","observedHarness":"Direct API","rawScore":57.8,"zScore":-0.918027,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"}]},{"id":"google:gemma-4-31b:none:direct:native","provider":"google","modelSlug":"gemma-4-31b","displayName":"Gemma 4 31B","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.40852,"shrinkageFactor":0.959672,"score":43.87,"se":6.31,"samplingSe":2.6878,"residualSe":5.708,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-01","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Gemma 4 31B","observedHarness":"Direct API","rawScore":1417.98,"zScore":-0.275879,"seModel":"upstream_ci","ciHalfWidth":18.8083,"ciLower":1399.1761,"ciUpper":1436.7928,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"alibaba:qwen3.5-397b-a17b:thinking:direct:native","provider":"alibaba","modelSlug":"qwen3.5-397b-a17b","displayName":"Qwen3.5 397B A17B","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.584607,"shrinkageFactor":0.933389,"score":41.23,"se":4.96,"samplingSe":3.4543,"residualSe":3.5605,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-03-20","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:Qwen/Qwen3.5-397B-A17B","observedHarness":"Direct API","rawScore":13.27,"zScore":-0.853664,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:Qwen/Qwen3.5-397B-A17B","observedHarness":"Direct API","rawScore":68.3,"zScore":-0.139804,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Qwen3.5-397B-A17B","observedHarness":"Direct API","rawScore":1406.08,"zScore":-0.502703,"seModel":"upstream_ci","ciHalfWidth":5.5242,"ciLower":1400.5563,"ciUpper":1411.6046,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemma-4-31b:thinking:direct:native","provider":"google","modelSlug":"gemma-4-31b","displayName":"Gemma 4 31B","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.585297,"shrinkageFactor":0.99966,"score":41.22,"se":6.62,"samplingSe":0.2467,"residualSe":6.6141,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-01","paretoProb":0,"benchmarks":[{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Gemma 4 31B Reasoning","observedHarness":"Direct API","rawScore":-2.058,"zScore":-0.846291,"seModel":"upstream_ci","ciHalfWidth":0.0685,"ciLower":-2.124,"ciUpper":-1.987,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"}]},{"id":"deepseek:deepseek-v3.2-exp:none:direct:native","provider":"deepseek","modelSlug":"deepseek-v3.2-exp","displayName":"DeepSeek V3.2-Exp","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.608981,"shrinkageFactor":0.936031,"score":40.87,"se":5.51,"samplingSe":3.3851,"residualSe":4.3512,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-09-29","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:deepseek-ai/DeepSeek-V3.2-Exp","observedHarness":"Direct API","rawScore":60.2,"zScore":-0.740148,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:DeepSeek-V3.2-Exp","observedHarness":"Direct API","rawScore":1408.29,"zScore":-0.460578,"seModel":"upstream_ci","ciHalfWidth":14.5979,"ciLower":1393.6962,"ciUpper":1422.8921,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"deepseek:deepseek-v4-flash:high:direct:native","provider":"deepseek","modelSlug":"deepseek-v4-flash","displayName":"DeepSeek V4 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.632764,"shrinkageFactor":0.993296,"score":40.51,"se":6.01,"samplingSe":1.0958,"residualSe":5.9079,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-31","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:DeepSeek-V4-Flash","observedHarness":"Direct API","rawScore":1407.21,"zScore":-0.481164,"seModel":"upstream_ci","ciHalfWidth":7.5376,"ciLower":1399.6686,"ciUpper":1414.7438,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-3.1-flash-lite-preview:minimal:direct:native","provider":"google","modelSlug":"gemini-3.1-flash-lite-preview","displayName":"Gemini 3.1 Flash-Lite Preview","effort":"minimal","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.678612,"shrinkageFactor":0.94571,"score":39.82,"se":5.39,"samplingSe":3.1185,"residualSe":4.3962,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-03-03","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:gemini-3.1-flash-lite-preview","observedHarness":"Direct API","rawScore":49,"zScore":-1.570253,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Gemini 3.1 Flash-Lite Preview","observedHarness":"Direct API","rawScore":1414.5,"zScore":-0.342211,"seModel":"upstream_ci","ciHalfWidth":6.8675,"ciLower":1407.6362,"ciUpper":1421.3712,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemma-4-26b-a4b:none:direct:native","provider":"google","modelSlug":"gemma-4-26b-a4b","displayName":"Gemma 4 26B A4B","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.742571,"shrinkageFactor":0.961441,"score":38.86,"se":6.29,"samplingSe":2.6281,"residualSe":5.7185,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-01","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Gemma 4 26B A4B","observedHarness":"Direct API","rawScore":1399.81,"zScore":-0.622214,"seModel":"upstream_ci","ciHalfWidth":18.3742,"ciLower":1381.4384,"ciUpper":1418.1868,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"deepseek:deepseek-v3.2:none:direct:native","provider":"deepseek","modelSlug":"deepseek-v3.2","displayName":"DeepSeek V3.2","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.762874,"shrinkageFactor":0.971739,"score":38.56,"se":2.42,"samplingSe":2.25,"residualSe":0.8778,"coverageCount":4,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-12-01","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:deepseek-ai/DeepSeek-V3.2","observedHarness":"Direct API","rawScore":14.34,"zScore":-0.184612,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:deepseek-ai/DeepSeek-V3.2","observedHarness":"Direct API","rawScore":66.8,"zScore":-0.250979,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:DeepSeek V3.2","observedHarness":"Direct API","rawScore":-3.137,"zScore":-1.354134,"seModel":"upstream_ci","ciHalfWidth":0.237,"ciLower":-3.38,"ciUpper":-2.906,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:DeepSeek-V3.2","observedHarness":"Direct API","rawScore":1400.76,"zScore":-0.604107,"seModel":"upstream_ci","ciHalfWidth":7.6705,"ciLower":1393.0943,"ciUpper":1408.4353,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"alibaba:qwen3.8-27b:xhigh:direct:native","provider":"alibaba","modelSlug":"qwen3.8-27b","displayName":"Qwen3.8 27B","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.905346,"shrinkageFactor":0.969216,"score":36.42,"se":2.51,"samplingSe":2.3483,"residualSe":0.8756,"coverageCount":4,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-05","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:Qwen/Qwen3.8-27B","observedHarness":"Direct API","rawScore":12.96,"zScore":-1.047502,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:Qwen/Qwen3.8-27B","observedHarness":"Direct API","rawScore":53.3,"zScore":-1.251552,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:Qwen3.8-27B^","observedHarness":"Direct API","rawScore":-0.956,"zScore":-0.327623,"seModel":"upstream_ci","ciHalfWidth":0.1175,"ciLower":-1.068,"ciUpper":-0.833,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Qwen3.8-27B","observedHarness":"Direct API","rawScore":1360.43,"zScore":-1.37283,"seModel":"upstream_ci","ciHalfWidth":8.0295,"ciLower":1352.4003,"ciUpper":1368.4594,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"deepseek:deepseek-v3.2-exp:thinking:direct:native","provider":"deepseek","modelSlug":"deepseek-v3.2-exp","displayName":"DeepSeek V3.2-Exp","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.924165,"shrinkageFactor":0.964977,"score":36.14,"se":6.26,"samplingSe":2.5047,"residualSe":5.7395,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-09-29","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:DeepSeek-V3.2-Exp (Thinking)","observedHarness":"Direct API","rawScore":1390.12,"zScore":-0.806914,"seModel":"upstream_ci","ciHalfWidth":17.4794,"ciLower":1372.6442,"ciUpper":1407.603,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"deepseek:deepseek-v3.2:thinking:direct:native","provider":"deepseek","modelSlug":"deepseek-v3.2","displayName":"DeepSeek V3.2","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-0.9396,"shrinkageFactor":0.992896,"score":35.91,"se":6.01,"samplingSe":1.1281,"residualSe":5.9056,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-12-01","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:DeepSeek-V3.2 (Thinking)","observedHarness":"Direct API","rawScore":1390.98,"zScore":-0.790522,"seModel":"upstream_ci","ciHalfWidth":7.7608,"ciLower":1383.2208,"ciUpper":1398.7423,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"meta:muse-glimmer:high:direct:native","provider":"meta","modelSlug":"muse-glimmer","displayName":"Muse Glimmer","effort":"high","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-1.125525,"shrinkageFactor":0.916339,"score":33.12,"se":5.76,"samplingSe":3.8712,"residualSe":4.2597,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:meta-models/Muse-Glimmer-30B","observedHarness":"Direct API","rawScore":13.72,"zScore":-0.572287,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Muse Glimmer","observedHarness":"Direct API","rawScore":1368.07,"zScore":-1.227206,"seModel":"upstream_ci","ciHalfWidth":23.1326,"ciLower":1344.939,"ciUpper":1391.2043,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"alibaba:qwen3.5-122b-a10b:thinking:direct:native","provider":"alibaba","modelSlug":"qwen3.5-122b-a10b","displayName":"Qwen3.5 122B A10B","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-1.320234,"shrinkageFactor":0.943248,"score":30.2,"se":5.42,"samplingSe":3.1884,"residualSe":4.3847,"coverageCount":2,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-03-20","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:Qwen/Qwen3.5-122B-A10B","observedHarness":"Direct API","rawScore":54.3,"zScore":-1.177435,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Qwen3.5-122B-A10B","observedHarness":"Direct API","rawScore":1365.16,"zScore":-1.282673,"seModel":"upstream_ci","ciHalfWidth":9.3289,"ciLower":1355.8346,"ciUpper":1374.4925,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-oss-120b:medium:direct:native","provider":"openai","modelSlug":"gpt-oss-120b","displayName":"gpt-oss-120b","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-2.265664,"shrinkageFactor":0.968954,"score":16.02,"se":2.52,"samplingSe":2.3582,"residualSe":0.8753,"coverageCount":4,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-08-05","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:openai/gpt-oss-120b","observedHarness":"Direct API","rawScore":9.57,"zScore":-3,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:openai/gpt-oss-120b","observedHarness":"Direct API","rawScore":35.8,"zScore":-2.548591,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","sourceId":"writing-feeds","sourceRowId":"lechmazur:GPT-OSS-120B","observedHarness":"Direct API","rawScore":-3.29,"zScore":-1.426145,"seModel":"upstream_ci","ciHalfWidth":0.1005,"ciLower":-3.398,"ciUpper":-3.197,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-17","sourceUrl":"https://github.com/lechmazur/writing"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:gpt-oss-120b","observedHarness":"Direct API","rawScore":1278.75,"zScore":-2.92972,"seModel":"upstream_ci","ciHalfWidth":9.9423,"ciLower":1268.8109,"ciUpper":1288.6956,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"meta:llama-4-maverick:none:direct:native","provider":"meta","modelSlug":"llama-4-maverick","displayName":"Llama 4 Maverick","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-2.446296,"shrinkageFactor":0.927794,"score":13.31,"se":5.05,"samplingSe":3.5964,"residualSe":3.5392,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-04-05","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:meta-llama/Llama-4-Maverick-17B-128E-Instruct","observedHarness":"Direct API","rawScore":10.34,"zScore":-2.685742,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:meta-llama/Llama-4-Maverick-17B-128E-Instruct","observedHarness":"Direct API","rawScore":31.9,"zScore":-2.837645,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Llama 4 Maverick 17B 128E Instruct","observedHarness":"Direct API","rawScore":1306.93,"zScore":-2.392585,"seModel":"upstream_ci","ciHalfWidth":8.5997,"ciLower":1298.3308,"ciUpper":1315.5303,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"meta:llama-4-scout:none:direct:native","provider":"meta","modelSlug":"llama-4-scout","displayName":"Llama 4 Scout","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-2.713123,"shrinkageFactor":0.92693,"score":9.3,"se":5.06,"samplingSe":3.6179,"residualSe":3.5359,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-04-05","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:meta-llama/Llama-4-Scout-17B-16E-Instruct","observedHarness":"Direct API","rawScore":9.33,"zScore":-3,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:meta-llama/Llama-4-Scout-17B-16E-Instruct","observedHarness":"Direct API","rawScore":31.1,"zScore":-2.896938,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Llama 4 Scout 17B 16E Instruct","observedHarness":"Direct API","rawScore":1289.08,"zScore":-2.732822,"seModel":"upstream_ci","ciHalfWidth":10.1215,"ciLower":1278.9559,"ciUpper":1299.1989,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-oss-20b:medium:direct:native","provider":"openai","modelSlug":"gpt-oss-20b","displayName":"gpt-oss-20b","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-2.932822,"shrinkageFactor":0.93806,"score":6.01,"se":4.89,"samplingSe":3.331,"residualSe":3.5783,"coverageCount":3,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-08-05","paretoProb":0,"benchmarks":[{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:creative:openai/gpt-oss-20b","observedHarness":"Direct API","rawScore":8.1,"zScore":-3,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing.html"},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","sourceId":"writing-feeds","sourceRowId":"eqbench:longform:openai/gpt-oss-20b","observedHarness":"Direct API","rawScore":21.2,"zScore":-3,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://eqbench.com/creative_writing_longform.html"},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:gpt-oss-20b","observedHarness":"Direct API","rawScore":1241.01,"zScore":-3,"seModel":"upstream_ci","ciHalfWidth":18.3746,"ciLower":1222.6362,"ciUpper":1259.3854,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"meta:llama-3.3-70b:none:direct:native","provider":"meta","modelSlug":"llama-3.3-70b","displayName":"Llama 3.3 70B","effort":"none","harness":"direct","harnessRegime":"native","domainId":"writing","theta":-2.941575,"shrinkageFactor":0.994118,"score":5.88,"se":6,"samplingSe":1.0265,"residualSe":5.9128,"coverageCount":1,"activeBenchmarkCount":5,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"list","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2024-12-06","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","sourceId":"lmarena","sourceRowId":"lmarena:text_style_control:creative_writing:Llama 3.3 70B Instruct","observedHarness":"Direct API","rawScore":1285.4,"zScore":-2.802965,"seModel":"upstream_ci","ciHalfWidth":7.071,"ciLower":1278.3262,"ciUpper":1292.4682,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]}],"frontiers":[{"provider":"anthropic","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"anthropic:claude-haiku-5.5:high:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"anthropic:claude-haiku-4.5:none:direct:native","zStepFromPrev":1.261,"pBetterThanPrev":0.8963,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.8988,"marginalScorePerDollar":362.593,"costMultipleFromPrev":10},{"configId":"anthropic:claude-sonnet-5.5:max:direct:native","zStepFromPrev":2.483,"pBetterThanPrev":0.9935,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.7289,"marginalScorePerDollar":876.333,"costMultipleFromPrev":2},{"configId":"anthropic:claude-opus-5.5:high:direct:native","zStepFromPrev":1.256,"pBetterThanPrev":0.8954,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.7906,"marginalScorePerDollar":198,"costMultipleFromPrev":2},{"configId":"anthropic:claude-opus-5:xhigh:direct:native","zStepFromPrev":0.716,"pBetterThanPrev":0.7631,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.7031,"marginalScorePerDollar":166,"costMultipleFromPrev":1.25},{"configId":"anthropic:claude-fable-5.2:high:direct:native","zStepFromPrev":0.096,"pBetterThanPrev":0.5381,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.466,"marginalScorePerDollar":9.444,"costMultipleFromPrev":1.6}],"pruned":[{"configId":"anthropic:claude-sonnet-5.5:xhigh:direct:native","dominatedById":"anthropic:claude-sonnet-5.5:max:direct:native","scoreDeficit":7.66,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"anthropic:claude-sonnet-5:high:direct:native","dominatedById":"anthropic:claude-sonnet-5.5:max:direct:native","scoreDeficit":12.6,"costRatio":1,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-4.6:none:direct:native","dominatedById":"anthropic:claude-sonnet-5.5:max:direct:native","scoreDeficit":9.1,"costRatio":1.5,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-4.5:none:direct:native","dominatedById":"anthropic:claude-sonnet-5.5:max:direct:native","scoreDeficit":11.04,"costRatio":1.5,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5.5:max:direct:native","dominatedById":"anthropic:claude-opus-5.5:high:direct:native","scoreDeficit":14.21,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"anthropic:claude-opus-5.5:medium:direct:native","dominatedById":"anthropic:claude-opus-5.5:high:direct:native","scoreDeficit":15.49,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"anthropic:claude-opus-5:high:direct:native","dominatedById":"anthropic:claude-opus-5:xhigh:direct:native","scoreDeficit":13.55,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"anthropic:claude-opus-4.6:high:direct:native","dominatedById":"anthropic:claude-opus-5:xhigh:direct:native","scoreDeficit":13.69,"costRatio":1,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.7:high:direct:native","dominatedById":"anthropic:claude-opus-5:xhigh:direct:native","scoreDeficit":15.04,"costRatio":1,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.7:none:direct:native","dominatedById":"anthropic:claude-opus-5:xhigh:direct:native","scoreDeficit":18.65,"costRatio":1,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.8:xhigh:direct:native","dominatedById":"anthropic:claude-opus-5:xhigh:direct:native","scoreDeficit":20.78,"costRatio":1,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.6:none:direct:native","dominatedById":"anthropic:claude-opus-5:xhigh:direct:native","scoreDeficit":20.83,"costRatio":1,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5:max:direct:native","dominatedById":"anthropic:claude-opus-5:xhigh:direct:native","scoreDeficit":20.87,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"anthropic:claude-opus-4.8:none:direct:native","dominatedById":"anthropic:claude-opus-5:xhigh:direct:native","scoreDeficit":22.5,"costRatio":1,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.8:high:direct:native","dominatedById":"anthropic:claude-opus-5:xhigh:direct:native","scoreDeficit":22.92,"costRatio":1,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.5:none:direct:native","dominatedById":"anthropic:claude-opus-5:xhigh:direct:native","scoreDeficit":25.68,"costRatio":1,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.2:max:direct:native","dominatedById":"anthropic:claude-fable-5.2:high:direct:native","scoreDeficit":8.85,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"anthropic:claude-fable-5.1:high:direct:native","dominatedById":"anthropic:claude-fable-5.2:high:direct:native","scoreDeficit":7.37,"costRatio":1.25,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5:high:direct:native","dominatedById":"anthropic:claude-fable-5.2:high:direct:native","scoreDeficit":12.18,"costRatio":1.25,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.1:max:direct:native","dominatedById":"anthropic:claude-fable-5.2:high:direct:native","scoreDeficit":17.48,"costRatio":1.25,"costComparisonNote":null}],"verdicts":{"budgetId":"anthropic:claude-haiku-5.5:high:direct:native","defaultId":"anthropic:claude-opus-5:xhigh:direct:native","maxId":"anthropic:claude-fable-5.2:high:direct:native"}},{"provider":"openai","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"openai:gpt-6-luna:max:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"openai:gpt-5.6-luna:medium:direct:native","zStepFromPrev":1.302,"pBetterThanPrev":0.9036,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.8405,"marginalScorePerDollar":3728.947,"costMultipleFromPrev":2.27},{"configId":"openai:gpt-6.2-sol:max:direct:native","zStepFromPrev":0.783,"pBetterThanPrev":0.7831,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.5096,"marginalScorePerDollar":187.406,"costMultipleFromPrev":8.82},{"configId":"openai:gpt-5.4:medium:direct:native","zStepFromPrev":0.233,"pBetterThanPrev":0.592,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.2604,"marginalScorePerDollar":105.2,"costMultipleFromPrev":1.42},{"configId":"openai:gpt-5.6-sol:high:direct:native","zStepFromPrev":0.256,"pBetterThanPrev":0.6008,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.3149,"marginalScorePerDollar":64.857,"costMultipleFromPrev":1.41},{"configId":"openai:gpt-5.5:xhigh:direct:native","zStepFromPrev":0.125,"pBetterThanPrev":0.5497,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.2817,"marginalScorePerDollar":22.2,"costMultipleFromPrev":1.42},{"configId":"openai:gpt-6.2-astra:high:direct:native","zStepFromPrev":1.695,"pBetterThanPrev":0.9549,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.8628,"marginalScorePerDollar":215,"costMultipleFromPrev":1.41}],"pruned":[{"configId":"openai:gpt-5.6-luna:xhigh:direct:native","dominatedById":"openai:gpt-5.6-luna:medium:direct:native","scoreDeficit":13.6,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"openai:gpt-5.4-nano:high:direct:native","dominatedById":"openai:gpt-5.6-luna:medium:direct:native","scoreDeficit":34.63,"costRatio":1.03,"costComparisonNote":null},{"configId":"openai:gpt-5.4-mini:none:direct:native","dominatedById":"openai:gpt-5.6-luna:medium:direct:native","scoreDeficit":4.4,"costRatio":3.75,"costComparisonNote":null},{"configId":"openai:gpt-5.4-mini:high:direct:native","dominatedById":"openai:gpt-5.6-luna:medium:direct:native","scoreDeficit":16.22,"costRatio":3.75,"costComparisonNote":null},{"configId":"openai:gpt-6.1-terra:max:direct:native","dominatedById":"openai:gpt-5.6-luna:medium:direct:native","scoreDeficit":2.66,"costRatio":4.41,"costComparisonNote":null},{"configId":"openai:o3:medium:direct:native","dominatedById":"openai:gpt-5.6-luna:medium:direct:native","scoreDeficit":17,"costRatio":7.65,"costComparisonNote":null},{"configId":"openai:gpt-5:medium:direct:native","dominatedById":"openai:gpt-5.6-luna:medium:direct:native","scoreDeficit":4.41,"costRatio":7.72,"costComparisonNote":null},{"configId":"openai:gpt-5.1:high:direct:native","dominatedById":"openai:gpt-5.6-luna:medium:direct:native","scoreDeficit":8.68,"costRatio":7.72,"costComparisonNote":null},{"configId":"openai:gpt-5.1:none:direct:native","dominatedById":"openai:gpt-5.6-luna:medium:direct:native","scoreDeficit":14.39,"costRatio":7.72,"costComparisonNote":null},{"configId":"openai:gpt-5:high:direct:native","dominatedById":"openai:gpt-5.6-luna:medium:direct:native","scoreDeficit":19.62,"costRatio":7.72,"costComparisonNote":null},{"configId":"openai:gpt-6-sol:medium:direct:native","dominatedById":"openai:gpt-6.2-sol:max:direct:native","scoreDeficit":5.97,"costRatio":1,"costComparisonNote":null},{"configId":"openai:gpt-6.1-sol:max:direct:native","dominatedById":"openai:gpt-6.2-sol:max:direct:native","scoreDeficit":9.42,"costRatio":1,"costComparisonNote":null},{"configId":"openai:gpt-6-sol:max:direct:native","dominatedById":"openai:gpt-6.2-sol:max:direct:native","scoreDeficit":18.5,"costRatio":1,"costComparisonNote":null},{"configId":"openai:gpt-5.6-terra:medium:direct:native","dominatedById":"openai:gpt-6.2-sol:max:direct:native","scoreDeficit":8.37,"costRatio":1.13,"costComparisonNote":null},{"configId":"openai:gpt-5.6-terra:xhigh:direct:native","dominatedById":"openai:gpt-6.2-sol:max:direct:native","scoreDeficit":17.97,"costRatio":1.13,"costComparisonNote":null},{"configId":"openai:gpt-5.2:medium:direct:native","dominatedById":"openai:gpt-6.2-sol:max:direct:native","scoreDeficit":8.91,"costRatio":1.22,"costComparisonNote":null},{"configId":"openai:gpt-5.2:none:direct:native","dominatedById":"openai:gpt-6.2-sol:max:direct:native","scoreDeficit":24,"costRatio":1.22,"costComparisonNote":null},{"configId":"openai:gpt-5.2:high:direct:native","dominatedById":"openai:gpt-6.2-sol:max:direct:native","scoreDeficit":30.62,"costRatio":1.22,"costComparisonNote":null},{"configId":"openai:gpt-5.4:xhigh:direct:native","dominatedById":"openai:gpt-5.4:medium:direct:native","scoreDeficit":0.04,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"openai:gpt-5.4:none:direct:native","dominatedById":"openai:gpt-5.4:medium:direct:native","scoreDeficit":16.45,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"openai:gpt-5.4:high:direct:native","dominatedById":"openai:gpt-5.4:medium:direct:native","scoreDeficit":17.71,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"openai:gpt-5.6-sol:xhigh:direct:native","dominatedById":"openai:gpt-5.6-sol:high:direct:native","scoreDeficit":5.55,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"openai:gpt-5.6-sol:medium:direct:native","dominatedById":"openai:gpt-5.6-sol:high:direct:native","scoreDeficit":10.92,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"openai:gpt-5.5:medium:direct:native","dominatedById":"openai:gpt-5.5:xhigh:direct:native","scoreDeficit":16.94,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"openai:gpt-5.5:high:direct:native","dominatedById":"openai:gpt-5.5:xhigh:direct:native","scoreDeficit":18.37,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"openai:gpt-6.2-astra:max:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":20.98,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"openai:gpt-6-astra:high:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":8.01,"costRatio":1.25,"costComparisonNote":null},{"configId":"openai:gpt-6-astra:medium:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":27.4,"costRatio":1.25,"costComparisonNote":null},{"configId":"openai:gpt-6-astra:max:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":30.26,"costRatio":1.25,"costComparisonNote":null}],"verdicts":{"budgetId":"openai:gpt-6-luna:max:direct:native","defaultId":"openai:gpt-5.6-luna:medium:direct:native","maxId":"openai:gpt-6.2-astra:high:direct:native"}},{"provider":"google","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"google:gemini-3.5-flash-lite:minimal:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"google:gemini-3-flash:minimal:direct:native","zStepFromPrev":0.398,"pBetterThanPrev":0.6545,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.4612,"marginalScorePerDollar":765,"costMultipleFromPrev":1.31},{"configId":"google:gemini-3.8-flash:high:direct:native","zStepFromPrev":1.295,"pBetterThanPrev":0.9023,"statisticallyTied":false,"imputationSensitive":true,"paretoProb":0.3377,"marginalScorePerDollar":1452.727,"costMultipleFromPrev":1.32},{"configId":"google:gemini-4-hydrogen:xhigh:direct:native","zStepFromPrev":0.902,"pBetterThanPrev":0.8164,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.567,"marginalScorePerDollar":371.556,"costMultipleFromPrev":2},{"configId":"google:gemini-4-carbon:high:direct:native","zStepFromPrev":0.107,"pBetterThanPrev":0.5425,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.3115,"marginalScorePerDollar":66,"costMultipleFromPrev":1.33}],"pruned":[{"configId":"google:gemini-3-flash:high:direct:native","dominatedById":"google:gemini-3-flash:minimal:direct:native","scoreDeficit":1.44,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"google:gemini-3.6-flash:high:direct:native","dominatedById":"google:gemini-3.8-flash:high:direct:native","scoreDeficit":1,"costRatio":1,"costComparisonNote":null},{"configId":"google:gemini-3.7-flash:high:direct:native","dominatedById":"google:gemini-3.8-flash:high:direct:native","scoreDeficit":3.47,"costRatio":1,"costComparisonNote":null},{"configId":"google:gemini-3.8-flash:medium:direct:native","dominatedById":"google:gemini-3.8-flash:high:direct:native","scoreDeficit":4.48,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"google:gemini-4-hydrogen:high:direct:native","dominatedById":"google:gemini-4-hydrogen:xhigh:direct:native","scoreDeficit":4.36,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"google:gemini-3.5-flash:medium:direct:native","dominatedById":"google:gemini-4-hydrogen:xhigh:direct:native","scoreDeficit":9.96,"costRatio":1.13,"costComparisonNote":null},{"configId":"google:gemini-3.5-flash:high:direct:native","dominatedById":"google:gemini-4-hydrogen:xhigh:direct:native","scoreDeficit":18.78,"costRatio":1.13,"costComparisonNote":null},{"configId":"google:gemini-4-argon:high:direct:native","dominatedById":"google:gemini-4-carbon:high:direct:native","scoreDeficit":2.61,"costRatio":1,"costComparisonNote":null},{"configId":"google:gemini-4-argon:medium:direct:native","dominatedById":"google:gemini-4-carbon:high:direct:native","scoreDeficit":12.4,"costRatio":1,"costComparisonNote":null},{"configId":"google:gemini-3.1-pro:high:direct:native","dominatedById":"google:gemini-4-carbon:high:direct:native","scoreDeficit":19.11,"costRatio":1.13,"costComparisonNote":null},{"configId":"google:gemini-4-neon:high:direct:native","dominatedById":"google:gemini-4-carbon:high:direct:native","scoreDeficit":1.62,"costRatio":1.5,"costComparisonNote":null}],"verdicts":{"budgetId":"google:gemini-3.5-flash-lite:minimal:direct:native","defaultId":"google:gemini-4-hydrogen:xhigh:direct:native","maxId":"google:gemini-4-carbon:high:direct:native"}},{"provider":"spacexai","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"spacexai:grok-4.20:none:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.5151,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"spacexai:grok-4.7:high:direct:native","zStepFromPrev":0.849,"pBetterThanPrev":0.802,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.6708,"marginalScorePerDollar":292.558,"costMultipleFromPrev":1.96}],"pruned":[{"configId":"spacexai:grok-4.20:thinking:direct:native","dominatedById":"spacexai:grok-4.20:none:direct:native","scoreDeficit":0.13,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"spacexai:grok-4.3:low:direct:native","dominatedById":"spacexai:grok-4.20:none:direct:native","scoreDeficit":16.32,"costRatio":1,"costComparisonNote":null},{"configId":"spacexai:grok-4.7:xhigh:direct:native","dominatedById":"spacexai:grok-4.7:high:direct:native","scoreDeficit":10.27,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"spacexai:grok-4.6:high:direct:native","dominatedById":"spacexai:grok-4.7:high:direct:native","scoreDeficit":15.08,"costRatio":1,"costComparisonNote":null},{"configId":"spacexai:grok-4.5:high:direct:native","dominatedById":"spacexai:grok-4.7:high:direct:native","scoreDeficit":19.77,"costRatio":1,"costComparisonNote":null}],"verdicts":{"budgetId":"spacexai:grok-4.20:none:direct:native","defaultId":null,"maxId":"spacexai:grok-4.7:high:direct:native"}},{"provider":"meta","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"meta:muse-ember:max:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"meta:muse-spark-1.3:max:direct:native","zStepFromPrev":0.288,"pBetterThanPrev":0.6132,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.3268,"marginalScorePerDollar":111.321,"costMultipleFromPrev":9.83},{"configId":"meta:muse-wonder:high:direct:native","zStepFromPrev":0.932,"pBetterThanPrev":0.8243,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.5558,"marginalScorePerDollar":87.514,"costMultipleFromPrev":4.07}],"pruned":[{"configId":"meta:muse-spark-1.3:high:direct:native","dominatedById":"meta:muse-spark-1.3:max:direct:native","scoreDeficit":1.11,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"meta:muse-spark-1.2:high:direct:native","dominatedById":"meta:muse-spark-1.3:max:direct:native","scoreDeficit":3.82,"costRatio":1,"costComparisonNote":null},{"configId":"meta:muse-spark-1.1:high:direct:native","dominatedById":"meta:muse-spark-1.3:max:direct:native","scoreDeficit":4.52,"costRatio":1,"costComparisonNote":null},{"configId":"meta:muse-spark-1.2:xhigh:direct:native","dominatedById":"meta:muse-spark-1.3:max:direct:native","scoreDeficit":7.88,"costRatio":1,"costComparisonNote":null},{"configId":"meta:muse-wonder:max:direct:native","dominatedById":"meta:muse-wonder:high:direct:native","scoreDeficit":3.85,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"}],"verdicts":{"budgetId":"meta:muse-ember:max:direct:native","defaultId":null,"maxId":"meta:muse-wonder:high:direct:native"}},{"provider":"deepseek","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"deepseek:deepseek-v4-pro-0813:high:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null}],"pruned":[],"verdicts":{"budgetId":"deepseek:deepseek-v4-pro-0813:high:direct:native","defaultId":null,"maxId":"deepseek:deepseek-v4-pro-0813:high:direct:native"}},{"provider":"moonshot","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"moonshot:kimi-k2.6:thinking:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"moonshot:kimi-k3.1-pro:high:direct:native","zStepFromPrev":2.99,"pBetterThanPrev":0.9986,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9574,"marginalScorePerDollar":605.507,"costMultipleFromPrev":2.35}],"pruned":[{"configId":"moonshot:kimi-k3.1-pro:max:direct:native","dominatedById":"moonshot:kimi-k3.1-pro:high:direct:native","scoreDeficit":16.46,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"moonshot:kimi-k3:max:direct:native","dominatedById":"moonshot:kimi-k3.1-pro:high:direct:native","scoreDeficit":11.44,"costRatio":1.5,"costComparisonNote":null}],"verdicts":{"budgetId":"moonshot:kimi-k2.6:thinking:direct:native","defaultId":null,"maxId":"moonshot:kimi-k3.1-pro:high:direct:native"}},{"provider":"alibaba","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"alibaba:qwen4-max:high:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9794,"marginalScorePerDollar":null,"costMultipleFromPrev":null}],"pruned":[{"configId":"alibaba:qwen4-max:xhigh:direct:native","dominatedById":"alibaba:qwen4-max:high:direct:native","scoreDeficit":24.31,"costRatio":null,"costComparisonNote":"same list SKU, effort cost not priced"},{"configId":"alibaba:qwen3.7-max:thinking:direct:native","dominatedById":"alibaba:qwen4-max:high:direct:native","scoreDeficit":45.71,"costRatio":1.67,"costComparisonNote":null}],"verdicts":{"budgetId":"alibaba:qwen4-max:high:direct:native","defaultId":null,"maxId":"alibaba:qwen4-max:high:direct:native"}},{"provider":"zhipu","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"zhipu:glm-5.3-flash:max:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"zhipu:glm-5:thinking:direct:native","zStepFromPrev":0.252,"pBetterThanPrev":0.5995,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.6021,"marginalScorePerDollar":98.446,"costMultipleFromPrev":6.51},{"configId":"zhipu:glm-5.3:max:direct:native","zStepFromPrev":2.279,"pBetterThanPrev":0.9887,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.7686,"marginalScorePerDollar":1372.727,"costMultipleFromPrev":1.39}],"pruned":[{"configId":"zhipu:glm-5.2:thinking:direct:native","dominatedById":"zhipu:glm-5.3:max:direct:native","scoreDeficit":7.53,"costRatio":1,"costComparisonNote":null},{"configId":"zhipu:glm-5.2:max:direct:native","dominatedById":"zhipu:glm-5.3:max:direct:native","scoreDeficit":7.55,"costRatio":1,"costComparisonNote":null},{"configId":"zhipu:glm-5.1:thinking:direct:native","dominatedById":"zhipu:glm-5.3:max:direct:native","scoreDeficit":12.93,"costRatio":1,"costComparisonNote":null}],"verdicts":{"budgetId":"zhipu:glm-5.3-flash:max:direct:native","defaultId":null,"maxId":"zhipu:glm-5.3:max:direct:native"}}]},"design_code":{"domainId":"design_code","epochVersion":"design-code-v2.0.0","costUnit":"usd_per_task","activeBenchmarkIds":["vals_vibe_code","vals_swebench_multimodal","lmarena_webdev","designarena_website","designarena_svg","designarena_3d","designarena_gamedev","aa_code2video"],"candidateBenchmarkIds":[],"benchmarkVersions":{"vals_vibe_code":"1.1","vals_swebench_multimodal":"1","lmarena_webdev":"webdev","designarena_website":"public","designarena_svg":"public","designarena_3d":"public","designarena_gamedev":"public","aa_code2video":"1.0"},"coverageFloor":1,"tauResid":0.3247,"regimeMeta":{"native":{"activeBenchmarkIds":["vals_vibe_code","vals_swebench_multimodal","lmarena_webdev","designarena_website","designarena_svg","designarena_3d","designarena_gamedev","aa_code2video"],"costSource":"measured","measuredCostBenchmarkIds":["designarena_website","designarena_svg","designarena_3d","designarena_gamedev"],"coverageFloor":1,"tauResid":0.3247,"calibrationTauResidSq":0.10545598,"priorMean":-0.034712,"tauModelSq":1.326049,"anchorLabs":9,"withheldBenchmarkIds":[],"linkJointCov":{"benchmarkIds":["aa_code2video","vals_swebench_multimodal"],"covMatrix":[[0.2980476680866576,-0.00024376969870859485,-0.032667654524972584,0.0007556809208108834],[-0.00024376969870859485,2.0211663994503966e-7,0.00001721926600199151,-3.8101364729262293e-7],[-0.032667654524972584,0.00001721926600199151,0.3696951014716373,-0.00816923205919341],[0.0007556809208108834,-3.8101364729262293e-7,-0.00816923205919341,0.00018430528132661248]]},"epochLink":{"previousEpoch":"design-code-v1.0.0","linked":true,"overlapCount":130,"overlapLabs":9,"slope":0.987495,"intercept":0.008378,"pearsonR":0.9756,"reason":"Verified Mean-Sigma link (n=130, labs=9, r=0.976)"}}},"anchorStats":{"vals_vibe_code":{"benchmarkId":"vals_vibe_code","mean":67.496093,"sd":25.161921,"anchorCount":52,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":-0.436723,"reliability":0.9789,"discrimination":0.4998,"redundancyFactor":0.3333,"effectiveWeight":0.1631,"taskFamily":"web_apps","datasetLineageId":"vals-vibe-code-1.1","introducedAt":"2026-07-01","invertScore":false,"popoviciuSe":7.071068,"boundedMax":100,"sampleSize":50},"lmarena_webdev":{"benchmarkId":"lmarena_webdev","mean":1577.854562,"sd":121.209814,"anchorCount":64,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":-0.171996,"reliability":0.9986,"discrimination":0.9375,"redundancyFactor":0.3686,"effectiveWeight":0.3451,"taskFamily":"web_apps","datasetLineageId":"lmarena-webdev","introducedAt":"2026-05-01","invertScore":false},"designarena_website":{"benchmarkId":"designarena_website","mean":1258.236953,"sd":60.720462,"anchorCount":69,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":0.15166,"reliability":0.5,"discrimination":0.3526,"redundancyFactor":0.3063,"effectiveWeight":0.1,"taskFamily":"web_apps","datasetLineageId":"designarena-website","introducedAt":"2026-06-01","invertScore":false},"designarena_svg":{"benchmarkId":"designarena_svg","mean":1210.691534,"sd":82.618436,"anchorCount":61,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":0.285539,"reliability":0.5,"discrimination":0.6595,"redundancyFactor":0.5938,"effectiveWeight":0.1958,"taskFamily":"vector_svg","datasetLineageId":"designarena-svg","introducedAt":"2026-07-01","invertScore":false},"designarena_3d":{"benchmarkId":"designarena_3d","mean":1240.840446,"sd":114.935691,"anchorCount":68,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":0.10254,"reliability":0.5,"discrimination":0.519,"redundancyFactor":0.6119,"effectiveWeight":0.1588,"taskFamily":"spatial_3d","datasetLineageId":"designarena-3d","introducedAt":"2026-07-01","invertScore":false},"designarena_gamedev":{"benchmarkId":"designarena_gamedev","mean":1244.67782,"sd":93.789997,"anchorCount":70,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":0.280332,"reliability":0.5,"discrimination":0.7188,"redundancyFactor":0.6171,"effectiveWeight":0.2218,"taskFamily":"interactive_ui","datasetLineageId":"designarena-gamedev","introducedAt":"2026-07-01","invertScore":false},"vals_swebench_multimodal":{"benchmarkId":"vals_swebench_multimodal","mean":31.420035,"sd":15.042943,"anchorCount":16,"anchorLabs":8,"linkedViaMeanSigma":true,"raschDelta":-0.079003,"reliability":0.8906,"discrimination":0.5982,"redundancyFactor":0.4679,"effectiveWeight":0.2493,"taskFamily":"frontend_visual_debug","datasetLineageId":"swebench-multimodal-1","introducedAt":"2026-08-01","invertScore":false,"popoviciuSe":2.282177,"boundedMax":100,"sampleSize":480,"linkVarIntercept":0.3696951014716373,"linkVarSlope":0.00018430528132661248,"linkCovIS":-0.00816923205919341},"aa_code2video":{"benchmarkId":"aa_code2video","mean":797.936225,"sd":345.59071,"anchorCount":17,"anchorLabs":8,"linkedViaMeanSigma":true,"raschDelta":0.001784,"reliability":0.9959,"discrimination":0.8393,"redundancyFactor":0.5,"effectiveWeight":0.4179,"taskFamily":"motion_video","datasetLineageId":"aa-code2video-1.0","introducedAt":"2026-09-15","invertScore":false,"linkVarIntercept":0.2980476680866576,"linkVarSlope":2.0211663994503966e-7,"linkCovIS":-0.00024376969870859485}},"configs":[{"id":"anthropic:claude-opus-5.5:max:direct:native","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":2.245718,"shrinkageFactor":0.988663,"score":83.69,"se":2.46,"samplingSe":1.8392,"residualSe":1.6326,"coverageCount":3,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.188385,"costLogSd":0.3067,"costMinUsd":0.533963,"costImputed":true,"costEstimated":true,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.9034,"benchmarks":[{"benchmarkId":"aa_code2video","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_code2video:2f3c4dc9-a450-4303-8697-0237257cf08f","observedHarness":"Stirrup","rawScore":1535.57,"zScore":2.134414,"seModel":"upstream_ci","ciHalfWidth":25.535,"ciLower":1510.04,"ciUpper":1561.11,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/code2video-aa"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Claude Opus 5.5 (Max)","observedHarness":"Direct API","rawScore":1819.04,"zScore":1.989818,"seModel":"upstream_ci","ciHalfWidth":10.9094,"ciLower":1808.1346,"ciUpper":1829.9533,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_swebench_multimodal","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:swebench-multimodal:anthropic/claude-opus-5-5","observedHarness":"mini-SWE-agent","rawScore":70.21,"zScore":2.578615,"seModel":"upstream_se","seRaw":2.09,"ciHalfWidth":4.0964,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.626091,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal"}]},{"id":"anthropic:claude-fable-5.2:xhigh:direct:native","provider":"anthropic","modelSlug":"claude-fable-5.2","displayName":"Claude Fable 5.2","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":2.19232,"shrinkageFactor":0.996668,"score":82.88,"se":4.39,"samplingSe":0.997,"residualSe":4.2729,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.073681,"costLogSd":0.3087,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1000000,"releaseDate":"2026-11-09","paretoProb":0.3915,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Claude Fable 5.2 (xHigh)","observedHarness":"Direct API","rawScore":1823.64,"zScore":2.027768,"seModel":"upstream_ci","ciHalfWidth":15.8175,"ciLower":1807.8184,"ciUpper":1839.4534,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-fable-5.2:high:direct:native","provider":"anthropic","modelSlug":"claude-fable-5.2","displayName":"Claude Fable 5.2","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":2.002119,"shrinkageFactor":0.956771,"score":80.03,"se":4.03,"samplingSe":3.5913,"residualSe":1.8352,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.073263,"costFixedFloorUsd":2.71005,"costLogSd":0.2909,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1000000,"releaseDate":"2026-11-09","paretoProb":0.1818,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:claude-fable-5-2","observedHarness":"Direct API","rawScore":1492,"zScore":2.185218,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.589037,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:claude-fable-5-2","observedHarness":"Direct API","rawScore":1539,"zScore":3.138098,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.18782,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:claude-fable-5-2","observedHarness":"Direct API","rawScore":1348,"zScore":1.4783,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.063343,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"vals_swebench_multimodal","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:swebench-multimodal:anthropic/claude-fable-5-2","observedHarness":"mini-SWE-agent","rawScore":62.29,"zScore":2.052123,"seModel":"upstream_se","seRaw":2.214,"ciHalfWidth":4.3394,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.839479,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:anthropic/claude-fable-5-2","observedHarness":"OpenHands","rawScore":93.77,"zScore":1.044193,"seModel":"upstream_se","seRaw":1.152,"ciHalfWidth":2.2579,"ciLower":null,"ciUpper":null,"measuredCostUsd":22.13226,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"anthropic:claude-sonnet-5.5:xhigh:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.77166,"shrinkageFactor":0.998857,"score":76.57,"se":4.32,"samplingSe":0.5839,"residualSe":4.2823,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.612411,"costLogSd":0.3067,"costMinUsd":0.418849,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.6469,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Claude Sonnet 5.5 (xHigh)","observedHarness":"Direct API","rawScore":1772,"zScore":1.60173,"seModel":"upstream_ci","ciHalfWidth":9.2529,"ciLower":1762.7435,"ciUpper":1781.2494,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-sonnet-5.5:max:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.764328,"shrinkageFactor":0.991359,"score":76.46,"se":2.63,"samplingSe":1.6056,"residualSe":2.087,"coverageCount":3,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.612411,"costLogSd":0.3067,"costMinUsd":0.418849,"costImputed":true,"costEstimated":true,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.6565,"benchmarks":[{"benchmarkId":"aa_code2video","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_code2video:b171d979-5ec1-45de-b8b9-db1ee65e77ec","observedHarness":"Stirrup","rawScore":1443.33,"zScore":1.867509,"seModel":"upstream_ci","ciHalfWidth":24.22,"ciLower":1419.11,"ciUpper":1467.55,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/code2video-aa"},{"benchmarkId":"vals_swebench_multimodal","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:swebench-multimodal:anthropic/claude-sonnet-5-5","observedHarness":"mini-SWE-agent","rawScore":58.33,"zScore":1.788876,"seModel":"upstream_se","seRaw":2.253,"ciHalfWidth":4.4159,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.258247,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:anthropic/claude-sonnet-5-5","observedHarness":"OpenHands","rawScore":92.39,"zScore":0.989348,"seModel":"upstream_se","seRaw":1.26,"ciHalfWidth":2.4696,"ciLower":null,"ciUpper":null,"measuredCostUsd":19.729112,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-6.2-astra:medium:direct:native","provider":"openai","modelSlug":"gpt-6.2-astra","displayName":"GPT-6.2 Astra","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.668969,"shrinkageFactor":0.822681,"score":75.03,"se":7.83,"samplingSe":7.2736,"residualSe":2.8906,"coverageCount":2,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.770021,"costFixedFloorUsd":0.479625,"costLogSd":0.2614,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.4835,"benchmarks":[{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-6.2-astra","observedHarness":"Direct API","rawScore":1501,"zScore":2.732937,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.982389,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-6.2-astra","observedHarness":"Direct API","rawScore":1335,"zScore":1.264204,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.93611,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"openai:gpt-6-astra:max:direct:native","provider":"openai","modelSlug":"gpt-6-astra","displayName":"GPT-6 Astra","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.655282,"shrinkageFactor":0.985591,"score":74.83,"se":2.07,"samplingSe":2.0734,"residualSe":0,"coverageCount":8,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.464672,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.1216,"benchmarks":[{"benchmarkId":"aa_code2video","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_code2video:2f339a97-9a0d-499a-9cb5-e0db665bfa25","observedHarness":"Stirrup","rawScore":1426.78,"zScore":1.81962,"seModel":"upstream_ci","ciHalfWidth":24.165,"ciLower":1402.61,"ciUpper":1450.94,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/code2video-aa"},{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gpt-6-astra-max","observedHarness":"Direct API","rawScore":1521,"zScore":2.437533,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.791829,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-6-astra-max","observedHarness":"Direct API","rawScore":1457,"zScore":2.263804,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.111956,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gpt-6-astra-max","observedHarness":"Direct API","rawScore":1406,"zScore":2.363982,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.033031,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-6-astra-max","observedHarness":"Direct API","rawScore":1336,"zScore":1.280673,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.92187,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:GPT-6 Astra (Max)","observedHarness":"Direct API","rawScore":1763.96,"zScore":1.535399,"seModel":"upstream_ci","ciHalfWidth":8.1257,"ciLower":1755.8354,"ciUpper":1772.0869,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_swebench_multimodal","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:swebench-multimodal:openai/gpt-6-astra","observedHarness":"mini-SWE-agent","rawScore":43.33,"zScore":0.791731,"seModel":"upstream_se","seRaw":2.264,"ciHalfWidth":4.4374,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.80573,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:openai/gpt-6-astra","observedHarness":"OpenHands","rawScore":89.59,"zScore":0.878069,"seModel":"upstream_se","seRaw":2.174,"ciHalfWidth":4.261,"ciLower":null,"ciUpper":null,"measuredCostUsd":38.512323,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-6-astra:medium:direct:native","provider":"openai","modelSlug":"gpt-6-astra","displayName":"GPT-6 Astra","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.638904,"shrinkageFactor":0.90813,"score":74.58,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.295343,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.2819,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gpt-6-astra","observedHarness":"Direct API","rawScore":1450,"zScore":1.819796,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.487073,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-6-astra","observedHarness":"Direct API","rawScore":1445,"zScore":2.135859,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.622793,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gpt-6-astra","observedHarness":"Direct API","rawScore":1414,"zScore":2.460812,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.542958,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-6-astra","observedHarness":"Direct API","rawScore":1336,"zScore":1.280673,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.528548,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"google:gemini-4-carbon:high:direct:native","provider":"google","modelSlug":"gemini-4-carbon","displayName":"Gemini 4 Carbon","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.59792,"shrinkageFactor":0.997899,"score":73.97,"se":3.18,"samplingSe":0.7918,"residualSe":3.0813,"coverageCount":2,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.392091,"costLogSd":0.3067,"costMinUsd":0.218447,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-10-12","paretoProb":0.4831,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Gemini 4 Carbon (High)","observedHarness":"Direct API","rawScore":1768.73,"zScore":1.574752,"seModel":"upstream_ci","ciHalfWidth":15.3947,"ciLower":1753.339,"ciUpper":1784.1284,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:google/gemini-4-carbon","observedHarness":"OpenHands","rawScore":89.06,"zScore":0.857006,"seModel":"upstream_se","seRaw":2.294,"ciHalfWidth":4.4962,"ciLower":null,"ciUpper":null,"measuredCostUsd":8.254926,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-6.2-astra:max:direct:native","provider":"openai","modelSlug":"gpt-6.2-astra","displayName":"GPT-6.2 Astra","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.594602,"shrinkageFactor":0.99805,"score":73.92,"se":3.17,"samplingSe":0.7628,"residualSe":3.0818,"coverageCount":2,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.706322,"costLogSd":0.3226,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.1545,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:GPT-6.2 Astra (Max)","observedHarness":"Direct API","rawScore":1766.18,"zScore":1.553714,"seModel":"upstream_ci","ciHalfWidth":15.3059,"ciLower":1750.8786,"ciUpper":1781.4904,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:openai/gpt-6.2-astra","observedHarness":"OpenHands","rawScore":89.9,"zScore":0.890389,"seModel":"upstream_se","seRaw":2.04,"ciHalfWidth":3.9984,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.924977,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"google:gemini-4-neon:high:direct:native","provider":"google","modelSlug":"gemini-4-neon","displayName":"Gemini 4 Neon","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.582703,"shrinkageFactor":0.948907,"score":73.74,"se":4.48,"samplingSe":3.9044,"residualSe":2.2012,"coverageCount":3,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.389618,"costFixedFloorUsd":0.216448,"costLogSd":0.2663,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1048576,"releaseDate":"2026-11-18","paretoProb":0.4475,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gemini-4-neon","observedHarness":"Direct API","rawScore":1489,"zScore":2.159117,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.367091,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gemini-4-neon","observedHarness":"Direct API","rawScore":1395,"zScore":1.602753,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.498701,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Gemini 4 Neon (High)","observedHarness":"Direct API","rawScore":1764.89,"zScore":1.543072,"seModel":"upstream_ci","ciHalfWidth":13.6927,"ciLower":1751.1939,"ciUpper":1778.5793,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-6.2-sol:max:direct:native","provider":"openai","modelSlug":"gpt-6.2-sol","displayName":"GPT-6.2 Sol","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.497779,"shrinkageFactor":0.998103,"score":72.47,"se":3.17,"samplingSe":0.7524,"residualSe":3.0819,"coverageCount":2,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.419422,"costLogSd":0.3715,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.4091,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:GPT-6.2 Sol (Max)","observedHarness":"Direct API","rawScore":1754.04,"zScore":1.453558,"seModel":"upstream_ci","ciHalfWidth":14.5806,"ciLower":1739.4621,"ciUpper":1768.6233,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:openai/gpt-6.2-sol","observedHarness":"OpenHands","rawScore":87.62,"zScore":0.799776,"seModel":"upstream_se","seRaw":2.195,"ciHalfWidth":4.3022,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.228244,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"anthropic:claude-fable-5.1:max:direct:native","provider":"anthropic","modelSlug":"claude-fable-5.1","displayName":"Claude Fable 5.1","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.489607,"shrinkageFactor":0.998578,"score":72.34,"se":1.98,"samplingSe":0.6513,"residualSe":1.8666,"coverageCount":3,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.705005,"costLogSd":0.3067,"costMinUsd":1.166111,"costImputed":true,"costEstimated":true,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-09-12","paretoProb":0.0023,"benchmarks":[{"benchmarkId":"aa_code2video","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_code2video:3e87c73e-a257-495e-9730-367a66229811","observedHarness":"Stirrup","rawScore":1324.83,"zScore":1.524618,"seModel":"upstream_ci","ciHalfWidth":22.39,"ciLower":1302.44,"ciUpper":1347.22,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/code2video-aa"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Claude Fable 5.1 (Max)","observedHarness":"Direct API","rawScore":1741.88,"zScore":1.353236,"seModel":"upstream_ci","ciHalfWidth":7.6034,"ciLower":1734.2726,"ciUpper":1749.4794,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:anthropic/claude-fable-5-1","observedHarness":"OpenHands","rawScore":90.26,"zScore":0.904697,"seModel":"upstream_se","seRaw":1.566,"ciHalfWidth":3.0694,"ciLower":null,"ciUpper":null,"measuredCostUsd":33.367507,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"alibaba:qwen4-max:xhigh:direct:native","provider":"alibaba","modelSlug":"qwen4-max","displayName":"Qwen4 Max","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.467667,"shrinkageFactor":0.997882,"score":72.02,"se":3.18,"samplingSe":0.7949,"residualSe":3.0813,"coverageCount":2,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.156754,"costLogSd":0.5378,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2.25,"inputPer1mUsd":1.5,"outputPer1mUsd":4.5,"contextWindow":1000000,"releaseDate":"2026-10-22","paretoProb":0.9985,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Qwen4-Max (xHigh)","observedHarness":"Direct API","rawScore":1750.9,"zScore":1.427652,"seModel":"upstream_ci","ciHalfWidth":15.5428,"ciLower":1735.3611,"ciUpper":1766.4466,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:alibaba/qwen4-max","observedHarness":"OpenHands","rawScore":86.66,"zScore":0.761623,"seModel":"upstream_se","seRaw":2.273,"ciHalfWidth":4.4551,"ciLower":null,"ciUpper":null,"measuredCostUsd":41.398551,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-6.1-sol:max:direct:native","provider":"openai","modelSlug":"gpt-6.1-sol","displayName":"GPT-6.1 Sol","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.401647,"shrinkageFactor":0.997659,"score":71.02,"se":1.55,"samplingSe":0.8357,"residualSe":1.3106,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.247433,"costLogSd":0.3067,"costMinUsd":0.111176,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.4264,"benchmarks":[{"benchmarkId":"aa_code2video","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_code2video:b25ac058-1e33-4c31-bbc9-a20c3ad0717a","observedHarness":"Stirrup","rawScore":1369.3,"zScore":1.653296,"seModel":"upstream_ci","ciHalfWidth":23.435,"ciLower":1345.86,"ciUpper":1392.73,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/code2video-aa"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:GPT-6.1 Sol (Max)","observedHarness":"Direct API","rawScore":1743.36,"zScore":1.365446,"seModel":"upstream_ci","ciHalfWidth":9.1274,"ciLower":1734.2319,"ciUpper":1752.4867,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_swebench_multimodal","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:swebench-multimodal:openai/gpt-6.1-sol","observedHarness":"mini-SWE-agent","rawScore":43.54,"zScore":0.805691,"seModel":"upstream_se","seRaw":2.265,"ciHalfWidth":4.4394,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.35065,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:openai/gpt-6.1-sol","observedHarness":"OpenHands","rawScore":88.93,"zScore":0.851839,"seModel":"upstream_se","seRaw":1.893,"ciHalfWidth":3.7103,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.234794,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"anthropic:claude-fable-5:max:direct:native","provider":"anthropic","modelSlug":"claude-fable-5","displayName":"Claude Fable 5","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.337808,"shrinkageFactor":0.994789,"score":70.07,"se":4.94,"samplingSe":1.2468,"residualSe":4.7754,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.03613,"costLogSd":0.3067,"costMinUsd":0.708645,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-06-09","paretoProb":0.0451,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:anthropic/claude-fable-5","observedHarness":"OpenHands","rawScore":90.35,"zScore":0.908274,"seModel":"upstream_se","seRaw":2.097,"ciHalfWidth":4.1101,"ciLower":null,"ciUpper":null,"measuredCostUsd":41.714694,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"google:gemini-4-argon:high:direct:native","provider":"google","modelSlug":"gemini-4-argon","displayName":"Gemini 4 Argon","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.332811,"shrinkageFactor":0.996897,"score":69.99,"se":1.63,"samplingSe":0.9621,"residualSe":1.3096,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.305322,"costLogSd":0.3067,"costMinUsd":0.170105,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.2508,"benchmarks":[{"benchmarkId":"aa_code2video","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_code2video:d366dfa1-d940-4788-aa73-d298c45699b3","observedHarness":"Stirrup","rawScore":1259.4,"zScore":1.33529,"seModel":"upstream_ci","ciHalfWidth":22.02,"ciLower":1237.38,"ciUpper":1281.42,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/code2video-aa"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Gemini 4 Argon (High)","observedHarness":"Direct API","rawScore":1682.44,"zScore":0.862846,"seModel":"upstream_ci","ciHalfWidth":8.9667,"ciLower":1673.4729,"ciUpper":1691.4063,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_swebench_multimodal","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:swebench-multimodal:google/gemini-4-argon","observedHarness":"mini-SWE-agent","rawScore":56.04,"zScore":1.636646,"seModel":"upstream_se","seRaw":2.268,"ciHalfWidth":4.4453,"ciLower":null,"ciUpper":null,"measuredCostUsd":11.397153,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:google/gemini-4-argon","observedHarness":"OpenHands","rawScore":91.91,"zScore":0.970272,"seModel":"upstream_se","seRaw":1.899,"ciHalfWidth":3.722,"ciLower":null,"ciUpper":null,"measuredCostUsd":8.10797,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"google:gemini-4-carbon:medium:direct:native","provider":"google","modelSlug":"gemini-4-carbon","displayName":"Gemini 4 Carbon","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.331115,"shrinkageFactor":0.90813,"score":69.97,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.218447,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-10-12","paretoProb":0.5127,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gemini-4-carbon","observedHarness":"Direct API","rawScore":1413,"zScore":1.497877,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.24054,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gemini-4-carbon","observedHarness":"Direct API","rawScore":1410,"zScore":1.762685,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.240715,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gemini-4-carbon","observedHarness":"Direct API","rawScore":1396,"zScore":2.242943,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.155121,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gemini-4-carbon","observedHarness":"Direct API","rawScore":1304,"zScore":0.753668,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.237412,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"openai:gpt-6.2-sol:medium:direct:native","provider":"openai","modelSlug":"gpt-6.2-sol","displayName":"GPT-6.2 Sol","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.268267,"shrinkageFactor":0.726185,"score":69.02,"se":9.65,"samplingSe":9.0386,"residualSe":3.3703,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.189275,"costFixedFloorUsd":0.056852,"costLogSd":0.2963,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.4622,"benchmarks":[{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-6.2-sol","observedHarness":"Direct API","rawScore":1436,"zScore":2.0399,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.227409,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"anthropic:claude-opus-5.5:medium:direct:native","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.258984,"shrinkageFactor":0.90813,"score":68.88,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.533963,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.2788,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:claude-opus-5-5","observedHarness":"Direct API","rawScore":1450,"zScore":1.819796,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.618028,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:claude-opus-5-5","observedHarness":"Direct API","rawScore":1435,"zScore":2.029238,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.720844,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:claude-opus-5-5","observedHarness":"Direct API","rawScore":1315,"zScore":1.262533,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.092659,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:claude-opus-5-5","observedHarness":"Direct API","rawScore":1321,"zScore":1.033639,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.70432,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"google:gemini-4-hydrogen:xhigh:direct:native","provider":"google","modelSlug":"gemini-4-hydrogen","displayName":"Gemini 4 Hydrogen","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.234914,"shrinkageFactor":0.993782,"score":68.52,"se":4.96,"samplingSe":1.3621,"residualSe":4.7705,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.181358,"costLogSd":0.3087,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":1.5,"outputPer1mUsd":7.5,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.6439,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:google/gemini-4-hydrogen","observedHarness":"OpenHands","rawScore":87.78,"zScore":0.806135,"seModel":"upstream_se","seRaw":2.292,"ciHalfWidth":4.4923,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.553735,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"anthropic:claude-sonnet-5.5:high:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.121073,"shrinkageFactor":0.957439,"score":66.82,"se":3.92,"samplingSe":3.5635,"residualSe":1.6383,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.418849,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.4259,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:claude-sonnet-5-5","observedHarness":"Direct API","rawScore":1393,"zScore":1.323867,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.56554,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:claude-sonnet-5-5","observedHarness":"Direct API","rawScore":1425,"zScore":1.922616,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.567507,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:claude-sonnet-5-5","observedHarness":"Direct API","rawScore":1289,"zScore":0.947833,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.112309,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:claude-sonnet-5-5","observedHarness":"Direct API","rawScore":1303,"zScore":0.737199,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.430038,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Claude Sonnet 5.5 (High)","observedHarness":"Direct API","rawScore":1715.49,"zScore":1.135514,"seModel":"upstream_ci","ciHalfWidth":7.9413,"ciLower":1707.5519,"ciUpper":1723.4345,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-opus-5:max:direct:native","provider":"anthropic","modelSlug":"claude-opus-5","displayName":"Claude Opus 5","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.109598,"shrinkageFactor":0.999504,"score":66.64,"se":4.3,"samplingSe":0.3848,"residualSe":4.285,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.958531,"costLogSd":0.3067,"costMinUsd":0.655572,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-07-28","paretoProb":0.0109,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Claude Opus 5 (Max)","observedHarness":"Direct API","rawScore":1691.57,"zScore":0.93817,"seModel":"upstream_ci","ciHalfWidth":6.0964,"ciLower":1685.4741,"ciUpper":1697.667,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-haiku-5.5:max:direct:native","provider":"anthropic","modelSlug":"claude-haiku-5.5","displayName":"Claude Haiku 5.5","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.108537,"shrinkageFactor":0.996596,"score":66.63,"se":2.33,"samplingSe":1.0077,"residualSe":2.098,"coverageCount":3,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.038832,"costLogSd":0.3067,"costMinUsd":0.017448,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1000000,"releaseDate":"2026-10-07","paretoProb":0.9808,"benchmarks":[{"benchmarkId":"aa_code2video","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_code2video:7d064a11-6e78-4030-b535-cdf940615041","observedHarness":"Stirrup","rawScore":1192.35,"zScore":1.141274,"seModel":"upstream_ci","ciHalfWidth":20.91,"ciLower":1171.44,"ciUpper":1213.26,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/code2video-aa"},{"benchmarkId":"vals_swebench_multimodal","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:swebench-multimodal:anthropic/claude-haiku-5-5","observedHarness":"mini-SWE-agent","rawScore":43.96,"zScore":0.833611,"seModel":"upstream_se","seRaw":2.268,"ciHalfWidth":4.4453,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.704848,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:anthropic/claude-haiku-5-5","observedHarness":"OpenHands","rawScore":90.44,"zScore":0.91185,"seModel":"upstream_se","seRaw":1.725,"ciHalfWidth":3.381,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.071856,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"moonshot:kimi-k3.1-pro:max:direct:native","provider":"moonshot","modelSlug":"kimi-k3.1-pro","displayName":"Kimi K3.1 Pro","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.081284,"shrinkageFactor":0.967801,"score":66.22,"se":3.42,"samplingSe":3.0995,"residualSe":1.4445,"coverageCount":6,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.392097,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-10-10","paretoProb":1,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:kimi-k3.1-pro","observedHarness":"Direct API","rawScore":1428,"zScore":1.628385,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.370172,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:kimi-k3.1-pro","observedHarness":"Direct API","rawScore":1382,"zScore":1.464145,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.492345,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:kimi-k3.1-pro","observedHarness":"Direct API","rawScore":1319,"zScore":1.310948,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.165626,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:kimi-k3.1-pro","observedHarness":"Direct API","rawScore":1314,"zScore":0.918357,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.540247,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Kimi K3.1 Pro (Max)","observedHarness":"Direct API","rawScore":1682.95,"zScore":0.867054,"seModel":"upstream_ci","ciHalfWidth":8.9433,"ciLower":1674.0072,"ciUpper":1691.8937,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:kimi/kimi-k3.1-pro","observedHarness":"OpenHands","rawScore":84.89,"zScore":0.691279,"seModel":"upstream_se","seRaw":2.949,"ciHalfWidth":5.78,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.485151,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-6.1-terra:max:direct:native","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.068415,"shrinkageFactor":0.998439,"score":66.03,"se":3.16,"samplingSe":0.6826,"residualSe":3.083,"coverageCount":2,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.103268,"costLogSd":0.3067,"costMinUsd":0.0464,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.5669,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:GPT-6.1 Terra (Max)","observedHarness":"Direct API","rawScore":1685.43,"zScore":0.887514,"seModel":"upstream_ci","ciHalfWidth":9.5069,"ciLower":1675.9243,"ciUpper":1694.9381,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:openai/gpt-6.1-terra","observedHarness":"OpenHands","rawScore":84,"zScore":0.655908,"seModel":"upstream_se","seRaw":2.865,"ciHalfWidth":5.6154,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.996953,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-6.1-sol:medium:direct:native","provider":"openai","modelSlug":"gpt-6.1-sol","displayName":"GPT-6.1 Sol","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.067651,"shrinkageFactor":0.90813,"score":66.01,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.111176,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.4947,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gpt-6.1-sol","observedHarness":"Direct API","rawScore":1414,"zScore":1.506578,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.095479,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-6.1-sol","observedHarness":"Direct API","rawScore":1384,"zScore":1.48547,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.152109,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gpt-6.1-sol","observedHarness":"Direct API","rawScore":1333,"zScore":1.480402,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.041575,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-6.1-sol","observedHarness":"Direct API","rawScore":1312,"zScore":0.885419,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.15554,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"openai:gpt-6-sol:max:direct:native","provider":"openai","modelSlug":"gpt-6-sol","displayName":"GPT-6 Sol","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.055969,"shrinkageFactor":0.998839,"score":65.84,"se":3.14,"samplingSe":0.5886,"residualSe":3.0842,"coverageCount":2,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.230256,"costLogSd":0.3067,"costMinUsd":0.103458,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.106,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:GPT-6 Sol (Max)","observedHarness":"Direct API","rawScore":1674.43,"zScore":0.796763,"seModel":"upstream_ci","ciHalfWidth":7.8506,"ciLower":1666.5788,"ciUpper":1682.28,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:openai/gpt-6-sol","observedHarness":"OpenHands","rawScore":87.82,"zScore":0.807725,"seModel":"upstream_se","seRaw":2.526,"ciHalfWidth":4.951,"ciLower":null,"ciUpper":null,"measuredCostUsd":26.356012,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"anthropic:claude-opus-4.8:max:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.8","displayName":"Claude Opus 4.8","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.029725,"shrinkageFactor":0.98882,"score":65.45,"se":5.09,"samplingSe":1.8264,"residualSe":4.7467,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.866977,"costLogSd":0.3067,"costMinUsd":0.272262,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-06-12","paretoProb":0.0338,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:anthropic/claude-opus-4-8","observedHarness":"OpenHands","rawScore":82.72,"zScore":0.605038,"seModel":"upstream_se","seRaw":3.081,"ciHalfWidth":6.0388,"ciLower":null,"ciUpper":null,"measuredCostUsd":26.879989,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"deepseek:deepseek-v4-pro-0813:max:direct:native","provider":"deepseek","modelSlug":"deepseek-v4-pro-0813","displayName":"DeepSeek V4 Pro (0813)","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.012705,"shrinkageFactor":0.988334,"score":65.19,"se":5.1,"samplingSe":1.8657,"residualSe":4.7444,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.137389,"costLogSd":0.5391,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.98,"inputPer1mUsd":1.32,"outputPer1mUsd":3.96,"contextWindow":1048576,"releaseDate":"2026-08-13","paretoProb":0.5709,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:deepseek/deepseek-v4-pro-0813","observedHarness":"OpenHands","rawScore":82.3,"zScore":0.588346,"seModel":"upstream_se","seRaw":3.148,"ciHalfWidth":6.1701,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.356023,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"google:gemini-4-hydrogen:high:direct:native","provider":"google","modelSlug":"gemini-4-hydrogen","displayName":"Gemini 4 Hydrogen","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.988724,"shrinkageFactor":0.954965,"score":64.83,"se":4.16,"samplingSe":3.6656,"residualSe":1.9716,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.13682,"costFixedFloorUsd":0.120702,"costLogSd":0.2909,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":1.5,"outputPer1mUsd":7.5,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.756,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gemini-4-hydrogen","observedHarness":"Direct API","rawScore":1397,"zScore":1.358669,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.128609,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gemini-4-hydrogen","observedHarness":"Direct API","rawScore":1335,"zScore":0.963026,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.185815,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gemini-4-hydrogen","observedHarness":"Direct API","rawScore":1326,"zScore":1.115984,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.168382,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Gemini 4 Hydrogen (High)","observedHarness":"Direct API","rawScore":1700.63,"zScore":1.012917,"seModel":"upstream_ci","ciHalfWidth":14.247,"ciLower":1686.3796,"ciUpper":1714.8735,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-sonnet-5:max:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-5","displayName":"Claude Sonnet 5","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.975356,"shrinkageFactor":0.98907,"score":64.63,"se":5.08,"samplingSe":1.8059,"residualSe":4.7479,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.342122,"costLogSd":0.3067,"costMinUsd":0.233989,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-07-15","paretoProb":0.3155,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:anthropic/claude-sonnet-5","observedHarness":"OpenHands","rawScore":81.33,"zScore":0.549795,"seModel":"upstream_se","seRaw":3.046,"ciHalfWidth":5.9702,"ciLower":null,"ciUpper":null,"measuredCostUsd":25.38542,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"deepseek:deepseek-v4.1-flash-1017:max:direct:native","provider":"deepseek","modelSlug":"deepseek-v4.1-flash-1017","displayName":"DeepSeek V4.1 Flash (1017)","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.962867,"shrinkageFactor":0.998824,"score":64.44,"se":3.14,"samplingSe":0.5924,"residualSe":3.0842,"coverageCount":2,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.048683,"costLogSd":0.3067,"costMinUsd":0.033296,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.525,"inputPer1mUsd":0.3,"outputPer1mUsd":1.2,"contextWindow":1048576,"releaseDate":"2026-10-17","paretoProb":0.941,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:DeepSeek-V4.1-Flash-20261017 (Max)","observedHarness":"Direct API","rawScore":1658,"zScore":0.661212,"seModel":"upstream_ci","ciHalfWidth":6.351,"ciLower":1651.6443,"ciUpper":1664.3463,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:deepseek/deepseek-v4.1-flash-1017","observedHarness":"OpenHands","rawScore":87.73,"zScore":0.804148,"seModel":"upstream_se","seRaw":2.752,"ciHalfWidth":5.3939,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.579984,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-5.6-sol:max:direct:native","provider":"openai","modelSlug":"gpt-5.6-sol","displayName":"GPT-5.6 Sol","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.937541,"shrinkageFactor":0.983818,"score":64.06,"se":5.21,"samplingSe":2.1973,"residualSe":4.7227,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.876224,"costLogSd":0.2656,"costMinUsd":0.702731,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0169,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:openai/gpt-5.6-sol","observedHarness":"OpenHands","rawScore":80.5,"zScore":0.516809,"seModel":"upstream_se","seRaw":3.716,"ciHalfWidth":7.2834,"ciLower":null,"ciUpper":null,"measuredCostUsd":33.403634,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"meta:muse-wonder:max:direct:native","provider":"meta","modelSlug":"muse-wonder","displayName":"Muse Wonder","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.926467,"shrinkageFactor":0.985598,"score":63.9,"se":2.07,"samplingSe":2.0729,"residualSe":0,"coverageCount":8,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.447948,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.4615,"benchmarks":[{"benchmarkId":"aa_code2video","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_code2video:ff2f3f3d-783f-4e27-8a39-47237501a313","observedHarness":"Stirrup","rawScore":997.01,"zScore":0.576039,"seModel":"upstream_ci","ciHalfWidth":19.86,"ciLower":977.15,"ciUpper":1016.87,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/code2video-aa"},{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:muse-wonder-1.0","observedHarness":"Direct API","rawScore":1420,"zScore":1.558781,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.518941,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:muse-wonder-1.0","observedHarness":"Direct API","rawScore":1394,"zScore":1.592091,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.555243,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:muse-wonder-1.0","observedHarness":"Direct API","rawScore":1285,"zScore":0.899417,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.29005,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:muse-wonder-1.0","observedHarness":"Direct API","rawScore":1374,"zScore":1.906492,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.427559,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Muse Wonder (Max)","observedHarness":"Direct API","rawScore":1647.02,"zScore":0.570626,"seModel":"upstream_ci","ciHalfWidth":8.4966,"ciLower":1638.5241,"ciUpper":1655.5173,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_swebench_multimodal","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:swebench-multimodal:meta/muse-wonder","observedHarness":"mini-SWE-agent","rawScore":45.63,"zScore":0.944627,"seModel":"upstream_se","seRaw":2.276,"ciHalfWidth":4.461,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.697976,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:meta/muse-wonder","observedHarness":"OpenHands","rawScore":83.09,"zScore":0.619742,"seModel":"upstream_se","seRaw":2.49,"ciHalfWidth":4.8804,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.802723,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"spacexai:grok-4.7:xhigh:direct:native","provider":"spacexai","modelSlug":"grok-4.7","displayName":"Grok 4.7","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.876666,"shrinkageFactor":0.998115,"score":63.15,"se":1.51,"samplingSe":0.7499,"residualSe":1.3112,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.505227,"costLogSd":0.3067,"costMinUsd":0.345542,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-10-18","paretoProb":0.8716,"benchmarks":[{"benchmarkId":"aa_code2video","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_code2video:59627802-1546-422c-9a65-65b61c022d36","observedHarness":"Stirrup","rawScore":1126.6,"zScore":0.95102,"seModel":"upstream_ci","ciHalfWidth":20.335,"ciLower":1106.26,"ciUpper":1146.93,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/code2video-aa"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Grok 4.7 (xHigh)","observedHarness":"Direct API","rawScore":1638.92,"zScore":0.503799,"seModel":"upstream_ci","ciHalfWidth":7.5099,"ciLower":1631.409,"ciUpper":1646.4289,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_swebench_multimodal","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:swebench-multimodal:grok/grok-4.7","observedHarness":"mini-SWE-agent","rawScore":42.92,"zScore":0.764476,"seModel":"upstream_se","seRaw":2.262,"ciHalfWidth":4.4335,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.662136,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:grok/grok-4.7","observedHarness":"OpenHands","rawScore":86.17,"zScore":0.742149,"seModel":"upstream_se","seRaw":2.181,"ciHalfWidth":4.2748,"ciLower":null,"ciUpper":null,"measuredCostUsd":15.82706,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"moonshot:kimi-k3:max:direct:native","provider":"moonshot","modelSlug":"kimi-k3","displayName":"Kimi K3","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.821664,"shrinkageFactor":0.98576,"score":62.32,"se":2.06,"samplingSe":2.0613,"residualSe":0,"coverageCount":8,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.433978,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1048576,"releaseDate":"2026-08-14","paretoProb":0.1705,"benchmarks":[{"benchmarkId":"aa_code2video","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_code2video:f7d2fc3e-1f7b-405f-818c-07952a4af78f","observedHarness":"Stirrup","rawScore":1041.34,"zScore":0.704312,"seModel":"upstream_ci","ciHalfWidth":19.765,"ciLower":1021.57,"ciUpper":1061.1,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/code2video-aa"},{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:kimi-k3","observedHarness":"Direct API","rawScore":1352,"zScore":0.967146,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.435662,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:kimi-k3","observedHarness":"Direct API","rawScore":1339,"zScore":1.005674,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.553881,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:kimi-k3","observedHarness":"Direct API","rawScore":1292,"zScore":0.984144,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.254602,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:kimi-k3","observedHarness":"Direct API","rawScore":1334,"zScore":1.247735,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.491766,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Kimi K3 (Max)","observedHarness":"Direct API","rawScore":1659.15,"zScore":0.6707,"seModel":"upstream_ci","ciHalfWidth":5.5961,"ciLower":1653.5561,"ciUpper":1664.7482,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_swebench_multimodal","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:swebench-multimodal:kimi/kimi-k3","observedHarness":"mini-SWE-agent","rawScore":44.17,"zScore":0.847571,"seModel":"upstream_se","seRaw":2.269,"ciHalfWidth":4.4472,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.650005,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:kimi/kimi-k3","observedHarness":"OpenHands","rawScore":84.97,"zScore":0.694458,"seModel":"upstream_se","seRaw":2.746,"ciHalfWidth":5.3822,"ciLower":null,"ciUpper":null,"measuredCostUsd":10.007576,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"meta:muse-spark-1.3:max:direct:native","provider":"meta","modelSlug":"muse-spark-1.3","displayName":"Muse Spark 1.3","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.817287,"shrinkageFactor":0.985721,"score":62.26,"se":2.06,"samplingSe":2.0641,"residualSe":0,"coverageCount":8,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.154882,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.3812,"benchmarks":[{"benchmarkId":"aa_code2video","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_code2video:9999672c-b687-4026-9c15-5cef02ff53bb","observedHarness":"Stirrup","rawScore":1018.78,"zScore":0.639033,"seModel":"upstream_ci","ciHalfWidth":19.675,"ciLower":999.11,"ciUpper":1038.46,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/code2video-aa"},{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:muse-spark-1.3-max","observedHarness":"Direct API","rawScore":1365,"zScore":1.080252,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.169655,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:muse-spark-1.3-max","observedHarness":"Direct API","rawScore":1328,"zScore":0.888391,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.195342,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:muse-spark-1.3-max","observedHarness":"Direct API","rawScore":1306,"zScore":1.153598,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.076405,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:muse-spark-1.3-max","observedHarness":"Direct API","rawScore":1348,"zScore":1.4783,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.178127,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Muse Spark 1.3 (Max)","observedHarness":"Direct API","rawScore":1652.97,"zScore":0.619714,"seModel":"upstream_ci","ciHalfWidth":6.7086,"ciLower":1646.2639,"ciUpper":1659.681,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_swebench_multimodal","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:swebench-multimodal:meta/muse_spark_1_3_max","observedHarness":"mini-SWE-agent","rawScore":43.13,"zScore":0.778436,"seModel":"upstream_se","seRaw":2.263,"ciHalfWidth":4.4355,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.863821,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:meta/muse_spark_1_3_max","observedHarness":"OpenHands","rawScore":85.86,"zScore":0.729829,"seModel":"upstream_se","seRaw":2.508,"ciHalfWidth":4.9157,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.543084,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"google:gemini-4-argon:medium:direct:native","provider":"google","modelSlug":"gemini-4-argon","displayName":"Gemini 4 Argon","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.812556,"shrinkageFactor":0.90813,"score":62.19,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.170105,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.2451,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gemini-4-argon","observedHarness":"Direct API","rawScore":1341,"zScore":0.87144,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.140157,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gemini-4-argon","observedHarness":"Direct API","rawScore":1346,"zScore":1.080309,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.209099,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gemini-4-argon","observedHarness":"Direct API","rawScore":1334,"zScore":1.492505,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.092593,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gemini-4-argon","observedHarness":"Direct API","rawScore":1311,"zScore":0.86895,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.238572,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"openai:gpt-5.6-luna:max:direct:native","provider":"openai","modelSlug":"gpt-5.6-luna","displayName":"GPT-5.6 Luna","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.80737,"shrinkageFactor":0.988906,"score":62.11,"se":5.08,"samplingSe":1.8194,"residualSe":4.7471,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.013685,"costLogSd":0.3067,"costMinUsd":0.006149,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.45,"inputPer1mUsd":0.2,"outputPer1mUsd":1.2,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.8404,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:openai/gpt-5.6-luna","observedHarness":"OpenHands","rawScore":77.06,"zScore":0.380094,"seModel":"upstream_se","seRaw":3.069,"ciHalfWidth":6.0152,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.726426,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"alibaba:qwen3.8-max-0902:xhigh:direct:native","provider":"alibaba","modelSlug":"qwen3.8-max-0902","displayName":"Qwen3.8 Max (0902)","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.791148,"shrinkageFactor":0.997614,"score":61.87,"se":2.05,"samplingSe":0.8438,"residualSe":1.8648,"coverageCount":3,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.210973,"costLogSd":0.5354,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":1000000,"releaseDate":"2026-09-02","paretoProb":0.3433,"benchmarks":[{"benchmarkId":"aa_code2video","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_code2video:b112af07-3bd5-4647-b09f-b23204361bb2","observedHarness":"Stirrup","rawScore":1020.76,"zScore":0.644762,"seModel":"upstream_ci","ciHalfWidth":19.675,"ciLower":1001.09,"ciUpper":1040.44,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/code2video-aa"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Qwen3.8-Max (0902)","observedHarness":"Direct API","rawScore":1673.64,"zScore":0.790245,"seModel":"upstream_ci","ciHalfWidth":5.8916,"ciLower":1667.753,"ciUpper":1679.5362,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:alibaba/qwen3.8-max-internal","observedHarness":"OpenHands","rawScore":77.14,"zScore":0.383274,"seModel":"upstream_se","seRaw":3.721,"ciHalfWidth":7.2932,"ciLower":null,"ciUpper":null,"measuredCostUsd":11.668329,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-5.6-sol:xhigh:direct:native","provider":"openai","modelSlug":"gpt-5.6-sol","displayName":"GPT-5.6 Sol","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.740131,"shrinkageFactor":0.90813,"score":61.1,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.702731,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0119,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gpt-5.6-sol-xhigh","observedHarness":"Direct API","rawScore":1355,"zScore":0.993247,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.77237,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-5.6-sol-xhigh","observedHarness":"Direct API","rawScore":1350,"zScore":1.122957,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.91922,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gpt-5.6-sol-xhigh","observedHarness":"Direct API","rawScore":1297,"zScore":1.044664,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.266119,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-5.6-sol-xhigh","observedHarness":"Direct API","rawScore":1314,"zScore":0.918357,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.853215,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"anthropic:claude-opus-5:high:direct:native","provider":"anthropic","modelSlug":"claude-opus-5","displayName":"Claude Opus 5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.715725,"shrinkageFactor":0.96785,"score":60.74,"se":3.42,"samplingSe":3.0971,"residualSe":1.4446,"coverageCount":6,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.655572,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-07-28","paretoProb":0.001,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:claude-opus-5","observedHarness":"Direct API","rawScore":1309,"zScore":0.593023,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.757281,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:claude-opus-5","observedHarness":"Direct API","rawScore":1316,"zScore":0.760445,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.867536,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:claude-opus-5","observedHarness":"Direct API","rawScore":1298,"zScore":1.056767,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.223172,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:claude-opus-5","observedHarness":"Direct API","rawScore":1299,"zScore":0.671323,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.774299,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Claude Opus 5 (High)","observedHarness":"Direct API","rawScore":1656.51,"zScore":0.64892,"seModel":"upstream_ci","ciHalfWidth":5.2639,"ciLower":1651.2495,"ciUpper":1661.7772,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:anthropic/claude-opus-5","observedHarness":"OpenHands","rawScore":88.4,"zScore":0.830775,"seModel":"upstream_se","seRaw":3.001,"ciHalfWidth":5.882,"ciLower":null,"ciUpper":null,"measuredCostUsd":33.876235,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"anthropic:claude-fable-5.1:high:direct:native","provider":"anthropic","modelSlug":"claude-fable-5.1","displayName":"Claude Fable 5.1","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.708106,"shrinkageFactor":0.90813,"score":60.62,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.166111,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-09-12","paretoProb":0.0013,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:claude-fable-5-1","observedHarness":"Direct API","rawScore":1375,"zScore":1.167258,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.250415,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:claude-fable-5-1","observedHarness":"Direct API","rawScore":1348,"zScore":1.101633,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.587746,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:claude-fable-5-1","observedHarness":"Direct API","rawScore":1287,"zScore":0.923625,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.328039,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:claude-fable-5-1","observedHarness":"Direct API","rawScore":1300,"zScore":0.687792,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.498246,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"openai:gpt-5.6-terra:max:direct:native","provider":"openai","modelSlug":"gpt-5.6-terra","displayName":"GPT-5.6 Terra","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.703128,"shrinkageFactor":0.979392,"score":60.55,"se":5.32,"samplingSe":2.4796,"residualSe":4.7015,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.161013,"costLogSd":0.3067,"costMinUsd":0.072346,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4.5,"inputPer1mUsd":2,"outputPer1mUsd":12,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0861,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:openai/gpt-5.6-terra","observedHarness":"OpenHands","rawScore":74.59,"zScore":0.28193,"seModel":"upstream_se","seRaw":4.203,"ciHalfWidth":8.2379,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.887015,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"meta:muse-spark-1.3:xhigh:direct:native","provider":"meta","modelSlug":"muse-spark-1.3","displayName":"Muse Spark 1.3","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.692735,"shrinkageFactor":0.998694,"score":60.39,"se":3.15,"samplingSe":0.6241,"residualSe":3.0838,"coverageCount":2,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.131624,"costLogSd":0.2656,"costMinUsd":0.106813,"costMaxUsd":0.154882,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.3495,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Muse Spark 1.3 (xHigh)","observedHarness":"Direct API","rawScore":1620.83,"zScore":0.354554,"seModel":"upstream_ci","ciHalfWidth":6.6782,"ciLower":1614.1477,"ciUpper":1627.504,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:meta/muse_spark_1_3","observedHarness":"OpenHands","rawScore":82.86,"zScore":0.610602,"seModel":"upstream_se","seRaw":2.901,"ciHalfWidth":5.686,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.101762,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"zhipu:glm-5.3:max:direct:native","provider":"zhipu","modelSlug":"glm-5.3","displayName":"GLM-5.3","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.658357,"shrinkageFactor":0.997676,"score":59.88,"se":2.04,"samplingSe":0.8327,"residualSe":1.8649,"coverageCount":3,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.152443,"costLogSd":0.538,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":1048576,"releaseDate":"2026-09-18","paretoProb":0.9411,"benchmarks":[{"benchmarkId":"aa_code2video","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_code2video:cd684ea4-b475-4269-b001-d469d06d8a7a","observedHarness":"Stirrup","rawScore":1033.51,"zScore":0.681655,"seModel":"upstream_ci","ciHalfWidth":19.72,"ciLower":1013.79,"ciUpper":1053.23,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/code2video-aa"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:GLM-5.3 (Max)","observedHarness":"Direct API","rawScore":1622.66,"zScore":0.369652,"seModel":"upstream_ci","ciHalfWidth":6.6156,"ciLower":1616.0425,"ciUpper":1629.2737,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:zai/glm-5.3","observedHarness":"OpenHands","rawScore":78.13,"zScore":0.422619,"seModel":"upstream_se","seRaw":3.94,"ciHalfWidth":7.7224,"ciLower":null,"ciUpper":null,"measuredCostUsd":12.450185,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"meta:muse-spark-1.3:high:direct:native","provider":"meta","modelSlug":"muse-spark-1.3","displayName":"Muse Spark 1.3","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.648893,"shrinkageFactor":0.90813,"score":59.73,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.106813,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.475,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:muse-spark-1.3","observedHarness":"Direct API","rawScore":1334,"zScore":0.810536,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.097311,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:muse-spark-1.3","observedHarness":"Direct API","rawScore":1314,"zScore":0.739121,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.128297,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:muse-spark-1.3","observedHarness":"Direct API","rawScore":1288,"zScore":0.935729,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.051945,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:muse-spark-1.3","observedHarness":"Direct API","rawScore":1355,"zScore":1.593582,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.149701,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"meta:muse-ember:max:direct:native","provider":"meta","modelSlug":"muse-ember","displayName":"Muse Ember","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.632347,"shrinkageFactor":0.99731,"score":59.49,"se":3.21,"samplingSe":0.8958,"residualSe":3.0795,"coverageCount":2,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.015773,"costLogSd":0.5815,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.9998,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Muse Ember (Max)","observedHarness":"Direct API","rawScore":1608.29,"zScore":0.251097,"seModel":"upstream_ci","ciHalfWidth":16.9253,"ciLower":1591.3672,"ciUpper":1625.2178,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:meta/muse-ember","observedHarness":"OpenHands","rawScore":83.7,"zScore":0.643985,"seModel":"upstream_se","seRaw":2.756,"ciHalfWidth":5.4018,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.054457,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"spacexai:grok-4.6:high:direct:native","provider":"spacexai","modelSlug":"grok-4.6","displayName":"Grok 4.6","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.591804,"shrinkageFactor":0.997914,"score":58.88,"se":3.18,"samplingSe":0.7888,"residualSe":3.0813,"coverageCount":2,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.165446,"costLogSd":0.5197,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-08-25","paretoProb":0.9563,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Grok 4.6 (High)","observedHarness":"Direct API","rawScore":1617.95,"zScore":0.330794,"seModel":"upstream_ci","ciHalfWidth":6.9296,"ciLower":1611.0237,"ciUpper":1624.8828,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:grok/grok-4.6","observedHarness":"OpenHands","rawScore":76.24,"zScore":0.347506,"seModel":"upstream_se","seRaw":3.824,"ciHalfWidth":7.495,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.875513,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"anthropic:claude-opus-4.7:max:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.7","displayName":"Claude Opus 4.7","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.561557,"shrinkageFactor":0.976387,"score":58.42,"se":5.39,"samplingSe":2.6543,"residualSe":4.687,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.888187,"costLogSd":0.5352,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-04-16","paretoProb":0.0128,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:anthropic/claude-opus-4-7","observedHarness":"OpenHands","rawScore":71,"zScore":0.139254,"seModel":"upstream_se","seRaw":4.506,"ciHalfWidth":8.8318,"ciLower":null,"ciUpper":null,"measuredCostUsd":21.407199,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-6-luna:max:direct:native","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.529102,"shrinkageFactor":0.997507,"score":57.94,"se":1.57,"samplingSe":0.8625,"residualSe":1.3104,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.011023,"costLogSd":0.3067,"costMinUsd":0.004953,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.741,"benchmarks":[{"benchmarkId":"aa_code2video","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_code2video:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","observedHarness":"Stirrup","rawScore":1029.65,"zScore":0.670486,"seModel":"upstream_ci","ciHalfWidth":19.64,"ciLower":1010.01,"ciUpper":1049.29,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/code2video-aa"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:GPT-6 Luna (Max)","observedHarness":"Direct API","rawScore":1578.32,"zScore":0.00384,"seModel":"upstream_ci","ciHalfWidth":5.8507,"ciLower":1572.4709,"ciUpper":1584.1724,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_swebench_multimodal","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:swebench-multimodal:openai/gpt-6-luna","observedHarness":"mini-SWE-agent","rawScore":37.5,"zScore":0.404174,"seModel":"upstream_se","seRaw":2.212,"ciHalfWidth":4.3355,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.03193,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:openai/gpt-6-luna","observedHarness":"OpenHands","rawScore":81.65,"zScore":0.562513,"seModel":"upstream_se","seRaw":3.377,"ciHalfWidth":6.6189,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.345703,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-5.5:xhigh:direct:native","provider":"openai","modelSlug":"gpt-5.5","displayName":"GPT-5.5","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.516764,"shrinkageFactor":0.976089,"score":57.75,"se":5.39,"samplingSe":2.671,"residualSe":4.6856,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.760528,"costLogSd":0.3067,"costMinUsd":0.341719,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":11.25,"inputPer1mUsd":5,"outputPer1mUsd":30,"contextWindow":1050000,"releaseDate":"2026-04-23","paretoProb":0.0013,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:openai/gpt-5.5","observedHarness":"OpenHands","rawScore":69.85,"zScore":0.09355,"seModel":"upstream_se","seRaw":4.535,"ciHalfWidth":8.8886,"ciLower":null,"ciUpper":null,"measuredCostUsd":16.661911,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"anthropic:claude-fable-5:high:direct:native","provider":"anthropic","modelSlug":"claude-fable-5","displayName":"Claude Fable 5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.446155,"shrinkageFactor":0.957478,"score":56.69,"se":3.92,"samplingSe":3.5619,"residualSe":1.6384,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.708645,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-06-09","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:claude-fable-5","observedHarness":"Direct API","rawScore":1284,"zScore":0.37551,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.651505,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:claude-fable-5","observedHarness":"Direct API","rawScore":1308,"zScore":0.675149,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.031183,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:claude-fable-5","observedHarness":"Direct API","rawScore":1287,"zScore":0.923625,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.273115,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:claude-fable-5","observedHarness":"Direct API","rawScore":1283,"zScore":0.40782,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.878776,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Claude Fable 5 (High)","observedHarness":"Direct API","rawScore":1625.88,"zScore":0.396217,"seModel":"upstream_ci","ciHalfWidth":5.9795,"ciLower":1619.9038,"ciUpper":1631.8627,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-3.8-flash:high:direct:native","provider":"google","modelSlug":"gemini-3.8-flash","displayName":"Gemini 3.8 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.439002,"shrinkageFactor":0.996281,"score":56.59,"se":1.68,"samplingSe":1.0534,"residualSe":1.3088,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.117882,"costLogSd":0.3067,"costMinUsd":0.065676,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.5,"inputPer1mUsd":0.75,"outputPer1mUsd":3.75,"contextWindow":1048576,"releaseDate":"2026-08-10","paretoProb":0.3328,"benchmarks":[{"benchmarkId":"aa_code2video","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_code2video:8a999846-4c1d-4ce7-a8b7-1310a7166fd7","observedHarness":"Stirrup","rawScore":899.09,"zScore":0.292698,"seModel":"upstream_ci","ciHalfWidth":20.235,"ciLower":878.86,"ciUpper":919.33,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/code2video-aa"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Gemini 3.8 Flash (High)","observedHarness":"Direct API","rawScore":1584.17,"zScore":0.052103,"seModel":"upstream_ci","ciHalfWidth":6.3861,"ciLower":1577.7823,"ciUpper":1590.5546,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_swebench_multimodal","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:swebench-multimodal:google/gemini-3.8-flash","observedHarness":"mini-SWE-agent","rawScore":40.83,"zScore":0.62554,"seModel":"upstream_se","seRaw":2.246,"ciHalfWidth":4.4022,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.688847,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:google/gemini-3.8-flash","observedHarness":"OpenHands","rawScore":78.65,"zScore":0.443285,"seModel":"upstream_se","seRaw":3.884,"ciHalfWidth":7.6126,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.865141,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-6.1-terra:medium:direct:native","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.438295,"shrinkageFactor":0.90813,"score":56.57,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.0464,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.1845,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gpt-6.1-terra","observedHarness":"Direct API","rawScore":1284,"zScore":0.37551,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.047256,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-6.1-terra","observedHarness":"Direct API","rawScore":1316,"zScore":0.760445,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.055512,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gpt-6.1-terra","observedHarness":"Direct API","rawScore":1299,"zScore":1.068871,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.020347,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-6.1-terra","observedHarness":"Direct API","rawScore":1283,"zScore":0.40782,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.062485,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"openai:gpt-5.4:xhigh:direct:native","provider":"openai","modelSlug":"gpt-5.4","displayName":"GPT-5.4","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.420968,"shrinkageFactor":0.972822,"score":56.31,"se":5.47,"samplingSe":2.8476,"residualSe":4.6699,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.516711,"costLogSd":0.5331,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":5.625,"inputPer1mUsd":2.5,"outputPer1mUsd":15,"contextWindow":1050000,"releaseDate":"2026-03-05","paretoProb":0.0025,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:openai/gpt-5.4-2026-03-05","observedHarness":"OpenHands","rawScore":67.42,"zScore":-0.003024,"seModel":"upstream_se","seRaw":4.843,"ciHalfWidth":9.4923,"ciLower":null,"ciUpper":null,"measuredCostUsd":10.686976,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"zhipu:glm-5.2:max:direct:native","provider":"zhipu","modelSlug":"glm-5.2","displayName":"GLM-5.2","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.376747,"shrinkageFactor":0.996975,"score":55.65,"se":3.22,"samplingSe":0.95,"residualSe":3.0785,"coverageCount":2,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.179332,"costLogSd":0.3067,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":1048576,"releaseDate":"2026-07-02","paretoProb":0.3316,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:GLM-5.2 (Max)","observedHarness":"Direct API","rawScore":1607.51,"zScore":0.244662,"seModel":"upstream_ci","ciHalfWidth":5.8799,"ciLower":1601.6288,"ciUpper":1613.3886,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:zai/glm-5.2","observedHarness":"OpenHands","rawScore":63.96,"zScore":-0.140534,"seModel":"upstream_se","seRaw":4.795,"ciHalfWidth":9.3982,"ciLower":null,"ciUpper":null,"measuredCostUsd":8.488514,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-5.6-sol:medium:direct:native","provider":"openai","modelSlug":"gpt-5.6-sol","displayName":"GPT-5.6 Sol","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.374451,"shrinkageFactor":0.90813,"score":55.62,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.324262,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0015,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gpt-5.6-sol","observedHarness":"Direct API","rawScore":1279,"zScore":0.332008,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.31288,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-5.6-sol","observedHarness":"Direct API","rawScore":1299,"zScore":0.579189,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.376468,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gpt-5.6-sol","observedHarness":"Direct API","rawScore":1287,"zScore":0.923625,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.107843,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-5.6-sol","observedHarness":"Direct API","rawScore":1300,"zScore":0.687792,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.499856,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"openai:gpt-6-sol:medium:direct:native","provider":"openai","modelSlug":"gpt-6-sol","displayName":"GPT-6 Sol","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.352654,"shrinkageFactor":0.90813,"score":55.29,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.103458,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0688,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gpt-6-sol","observedHarness":"Direct API","rawScore":1288,"zScore":0.410313,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.103866,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-6-sol","observedHarness":"Direct API","rawScore":1289,"zScore":0.472568,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.124435,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gpt-6-sol","observedHarness":"Direct API","rawScore":1296,"zScore":1.03256,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.035022,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-6-sol","observedHarness":"Direct API","rawScore":1284,"zScore":0.424289,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.150508,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"google:gemini-3.7-flash:high:direct:native","provider":"google","modelSlug":"gemini-3.7-flash","displayName":"Gemini 3.7 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.31936,"shrinkageFactor":0.967531,"score":54.79,"se":3.43,"samplingSe":3.1125,"residualSe":1.4441,"coverageCount":6,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.065956,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.5,"inputPer1mUsd":0.75,"outputPer1mUsd":3.75,"contextWindow":1048576,"releaseDate":"2026-06-18","paretoProb":0.5783,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gemini-3.7-flash","observedHarness":"Direct API","rawScore":1274,"zScore":0.288505,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.056219,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gemini-3.7-flash","observedHarness":"Direct API","rawScore":1295,"zScore":0.536541,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.076096,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gemini-3.7-flash","observedHarness":"Direct API","rawScore":1271,"zScore":0.729964,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.043556,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gemini-3.7-flash","observedHarness":"Direct API","rawScore":1291,"zScore":0.539572,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.087951,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Gemini 3.7 Flash (High)","observedHarness":"Direct API","rawScore":1588.7,"zScore":0.089477,"seModel":"upstream_ci","ciHalfWidth":6.4642,"ciLower":1582.2391,"ciUpper":1595.1676,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:google/gemini-3.7-flash","observedHarness":"OpenHands","rawScore":70.39,"zScore":0.115011,"seModel":"upstream_se","seRaw":4.837,"ciHalfWidth":9.4805,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.827643,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"anthropic:claude-haiku-5.5:high:direct:native","provider":"anthropic","modelSlug":"claude-haiku-5.5","displayName":"Claude Haiku 5.5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.316303,"shrinkageFactor":0.999201,"score":54.74,"se":4.31,"samplingSe":0.4884,"residualSe":4.2837,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.031317,"costLogSd":0.3067,"costMinUsd":0.017448,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1000000,"releaseDate":"2026-10-07","paretoProb":0.5295,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Claude Haiku 5.5 (High)","observedHarness":"Direct API","rawScore":1595.38,"zScore":0.144588,"seModel":"upstream_ci","ciHalfWidth":7.7382,"ciLower":1587.6399,"ciUpper":1603.1163,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-3.8-flash:medium:direct:native","provider":"google","modelSlug":"gemini-3.8-flash","displayName":"Gemini 3.8 Flash","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.241125,"shrinkageFactor":0.90813,"score":53.62,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.065676,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.5,"inputPer1mUsd":0.75,"outputPer1mUsd":3.75,"contextWindow":1048576,"releaseDate":"2026-08-10","paretoProb":0.9753,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gemini-3.8-flash","observedHarness":"Direct API","rawScore":1262,"zScore":0.184099,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.055824,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gemini-3.8-flash","observedHarness":"Direct API","rawScore":1279,"zScore":0.365947,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.074172,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gemini-3.8-flash","observedHarness":"Direct API","rawScore":1284,"zScore":0.887314,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.047004,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gemini-3.8-flash","observedHarness":"Direct API","rawScore":1287,"zScore":0.473696,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.085704,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"meta:muse-spark-1.2:high:direct:native","provider":"meta","modelSlug":"muse-spark-1.2","displayName":"Muse Spark 1.2","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.220392,"shrinkageFactor":0.90813,"score":53.31,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.088387,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.1664,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:muse-spark-1.2","observedHarness":"Direct API","rawScore":1270,"zScore":0.253703,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.082848,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:muse-spark-1.2","observedHarness":"Direct API","rawScore":1276,"zScore":0.333961,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.107155,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:muse-spark-1.2","observedHarness":"Direct API","rawScore":1264,"zScore":0.645237,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.064483,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:muse-spark-1.2","observedHarness":"Direct API","rawScore":1304,"zScore":0.753668,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.099062,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"zhipu:glm-5.3-flash:max:direct:native","provider":"zhipu","modelSlug":"glm-5.3-flash","displayName":"GLM-5.3 Flash","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.213102,"shrinkageFactor":0.9965,"score":53.2,"se":2.12,"samplingSe":1.0219,"residualSe":1.8627,"coverageCount":3,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.016276,"costLogSd":0.5806,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2375,"inputPer1mUsd":0.15,"outputPer1mUsd":0.5,"contextWindow":1048576,"releaseDate":"2026-09-28","paretoProb":0.9991,"benchmarks":[{"benchmarkId":"aa_code2video","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_code2video:19496b81-9f41-4214-a77a-1df803b3c5ae","observedHarness":"Stirrup","rawScore":974.54,"zScore":0.51102,"seModel":"upstream_ci","ciHalfWidth":19.825,"ciLower":954.71,"ciUpper":994.36,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/code2video-aa"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:GLM-5.3-Flash","observedHarness":"Direct API","rawScore":1610.48,"zScore":0.269165,"seModel":"upstream_ci","ciHalfWidth":6.0465,"ciLower":1604.431,"ciUpper":1616.5239,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:zai/glm-5.3-flash","observedHarness":"OpenHands","rawScore":30.76,"zScore":-1.459988,"seModel":"upstream_se","seRaw":5.277,"ciHalfWidth":10.3429,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.69761,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-5.3-codex:xhigh:direct:native","provider":"openai","modelSlug":"gpt-5.3-codex","displayName":"GPT-5.3 Codex","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.202865,"shrinkageFactor":0.974214,"score":53.04,"se":5.44,"samplingSe":2.7737,"residualSe":4.6766,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.474449,"costLogSd":0.533,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4.8125,"inputPer1mUsd":1.75,"outputPer1mUsd":14,"contextWindow":400000,"releaseDate":"2026-02-15","paretoProb":0.0008,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:openai/gpt-5.3-codex","observedHarness":"OpenHands","rawScore":61.77,"zScore":-0.22757,"seModel":"upstream_se","seRaw":4.714,"ciHalfWidth":9.2394,"ciLower":null,"ciUpper":null,"measuredCostUsd":11.914536,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"deepseek:deepseek-v4-pro-0813:high:direct:native","provider":"deepseek","modelSlug":"deepseek-v4-pro-0813","displayName":"DeepSeek V4 Pro (0813)","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.194863,"shrinkageFactor":0.998789,"score":52.92,"se":4.32,"samplingSe":0.6012,"residualSe":4.282,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.107666,"costLogSd":0.5221,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.98,"inputPer1mUsd":1.32,"outputPer1mUsd":3.96,"contextWindow":1048576,"releaseDate":"2026-08-13","paretoProb":0.0821,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:DeepSeek-V4-Pro (High, 20260813)","observedHarness":"Direct API","rawScore":1580.66,"zScore":0.023145,"seModel":"upstream_ci","ciHalfWidth":9.5272,"ciLower":1571.1363,"ciUpper":1590.1906,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"meta:muse-spark-1.2:xhigh:direct:native","provider":"meta","modelSlug":"muse-spark-1.2","displayName":"Muse Spark 1.2","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.155042,"shrinkageFactor":0.997492,"score":52.33,"se":3.2,"samplingSe":0.865,"residualSe":3.08,"coverageCount":2,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.129233,"costLogSd":0.3067,"costMinUsd":0.088387,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.0159,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Muse Spark 1.2 (xHigh)","observedHarness":"Direct API","rawScore":1533.33,"zScore":-0.367335,"seModel":"upstream_ci","ciHalfWidth":13.8029,"ciLower":1519.5226,"ciUpper":1547.1284,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:meta/muse_spark_1_2","observedHarness":"OpenHands","rawScore":79.1,"zScore":0.461169,"seModel":"upstream_se","seRaw":3.305,"ciHalfWidth":6.4778,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.529642,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"spacexai:grok-4.5:high:direct:native","provider":"spacexai","modelSlug":"grok-4.5","displayName":"Grok 4.5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.133017,"shrinkageFactor":0.997267,"score":52,"se":3.21,"samplingSe":0.9031,"residualSe":3.0793,"coverageCount":2,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.165446,"costLogSd":0.5197,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-06-30","paretoProb":0.5384,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Grok 4.5","observedHarness":"Direct API","rawScore":1552.39,"zScore":-0.210087,"seModel":"upstream_ci","ciHalfWidth":6.7228,"ciLower":1545.6701,"ciUpper":1559.1156,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:grok/grok-4.5","observedHarness":"OpenHands","rawScore":69,"zScore":0.059769,"seModel":"upstream_se","seRaw":4.48,"ciHalfWidth":8.7808,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.102194,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"deepseek:deepseek-v4.1-flash-1017:high:direct:native","provider":"deepseek","modelSlug":"deepseek-v4.1-flash-1017","displayName":"DeepSeek V4.1 Flash (1017)","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.09832,"shrinkageFactor":0.90813,"score":51.47,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.033296,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.525,"inputPer1mUsd":0.3,"outputPer1mUsd":1.2,"contextWindow":1048576,"releaseDate":"2026-10-17","paretoProb":0.9767,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:deepseek-v4-1-flash-1017","observedHarness":"Direct API","rawScore":1330,"zScore":0.775734,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.035476,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:deepseek-v4-1-flash-1017","observedHarness":"Direct API","rawScore":1239,"zScore":-0.060538,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.031117,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:deepseek-v4-1-flash-1017","observedHarness":"Direct API","rawScore":1238,"zScore":0.330537,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.021429,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:deepseek-v4-1-flash-1017","observedHarness":"Direct API","rawScore":1289,"zScore":0.506634,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.045162,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"anthropic:claude-haiku-5.5:medium:direct:native","provider":"anthropic","modelSlug":"claude-haiku-5.5","displayName":"Claude Haiku 5.5","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.082224,"shrinkageFactor":0.90813,"score":51.23,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.017448,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1000000,"releaseDate":"2026-10-07","paretoProb":0.9999,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:claude-haiku-5-5","observedHarness":"Direct API","rawScore":1307,"zScore":0.575622,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.019674,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:claude-haiku-5-5","observedHarness":"Direct API","rawScore":1313,"zScore":0.728459,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.026418,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:claude-haiku-5-5","observedHarness":"Direct API","rawScore":1201,"zScore":-0.117305,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.002857,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:claude-haiku-5-5","observedHarness":"Direct API","rawScore":1248,"zScore":-0.168591,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.020844,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"spacexai:grok-4.7:high:direct:native","provider":"spacexai","modelSlug":"grok-4.7","displayName":"Grok 4.7","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.053093,"shrinkageFactor":0.90813,"score":50.8,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.345542,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-10-18","paretoProb":0.1185,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:grok-4.7","observedHarness":"Direct API","rawScore":1239,"zScore":-0.016013,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.391364,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:grok-4.7","observedHarness":"Direct API","rawScore":1291,"zScore":0.493893,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.464349,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:grok-4.7","observedHarness":"Direct API","rawScore":1259,"zScore":0.584718,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.166718,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:grok-4.7","observedHarness":"Direct API","rawScore":1240,"zScore":-0.300343,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.359739,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"anthropic:claude-opus-4.7:high:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.7","displayName":"Claude Opus 4.7","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.003473,"shrinkageFactor":0.999551,"score":50.05,"se":4.3,"samplingSe":0.3659,"residualSe":4.2852,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.698137,"costLogSd":0.5239,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-04-16","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Claude Opus 4.7 (High)","observedHarness":"Direct API","rawScore":1557.43,"zScore":-0.168506,"seModel":"upstream_ci","ciHalfWidth":5.7962,"ciLower":1551.6292,"ciUpper":1563.2216,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-opus-4.7:none:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.7","displayName":"Claude Opus 4.7","effort":"none","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.012277,"shrinkageFactor":0.999573,"score":49.82,"se":4.3,"samplingSe":0.3569,"residualSe":4.2853,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.300721,"costLogSd":0.5854,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-04-16","paretoProb":0.0005,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Claude Opus 4.7","observedHarness":"Direct API","rawScore":1555.52,"zScore":-0.184264,"seModel":"upstream_ci","ciHalfWidth":5.6542,"ciLower":1549.8643,"ciUpper":1561.1728,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-3.6-flash:high:direct:native","provider":"google","modelSlug":"gemini-3.6-flash","displayName":"Gemini 3.6 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.012151,"shrinkageFactor":0.997395,"score":49.82,"se":3.2,"samplingSe":0.8817,"residualSe":3.0797,"coverageCount":2,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.175254,"costLogSd":0.3087,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.5,"inputPer1mUsd":0.75,"outputPer1mUsd":3.75,"contextWindow":1048576,"releaseDate":"2026-05-05","paretoProb":0.0094,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Gemini 3.6 Flash (High)","observedHarness":"Direct API","rawScore":1537.79,"zScore":-0.330539,"seModel":"upstream_ci","ciHalfWidth":7.5817,"ciLower":1530.2111,"ciUpper":1545.3746,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:google/gemini-3.6-flash","observedHarness":"OpenHands","rawScore":64,"zScore":-0.138944,"seModel":"upstream_se","seRaw":4.29,"ciHalfWidth":8.4084,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.041993,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"anthropic:claude-opus-4.8:high:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.8","displayName":"Claude Opus 4.8","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.012525,"shrinkageFactor":0.999551,"score":49.81,"se":4.3,"samplingSe":0.366,"residualSe":4.2852,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.686426,"costLogSd":0.3067,"costMinUsd":0.272262,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-06-12","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Claude Opus 4.8 (High)","observedHarness":"Direct API","rawScore":1555.49,"zScore":-0.184511,"seModel":"upstream_ci","ciHalfWidth":5.7985,"ciLower":1549.6938,"ciUpper":1561.2908,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-opus-4.6:high:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.6","displayName":"Claude Opus 4.6","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.087735,"shrinkageFactor":0.999593,"score":48.68,"se":4.3,"samplingSe":0.3485,"residualSe":4.2854,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.698137,"costLogSd":0.5239,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-02-05","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Claude Opus 4.6 (High)","observedHarness":"Direct API","rawScore":1546.37,"zScore":-0.259753,"seModel":"upstream_ci","ciHalfWidth":5.5201,"ciLower":1540.8486,"ciUpper":1551.8887,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-opus-4.6:max:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.6","displayName":"Claude Opus 4.6","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.117364,"shrinkageFactor":0.974596,"score":48.24,"se":5.43,"samplingSe":2.7531,"residualSe":4.6785,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.888187,"costLogSd":0.5352,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-02-05","paretoProb":0,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:anthropic/claude-opus-4-6-thinking","observedHarness":"OpenHands","rawScore":53.5,"zScore":-0.556241,"seModel":"upstream_se","seRaw":4.678,"ciHalfWidth":9.1689,"ciLower":null,"ciUpper":null,"measuredCostUsd":8.279062,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-5.2:xhigh:direct:native","provider":"openai","modelSlug":"gpt-5.2","displayName":"GPT-5.2","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.117002,"shrinkageFactor":0.970326,"score":48.24,"se":5.53,"samplingSe":2.9755,"residualSe":4.658,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.474449,"costLogSd":0.533,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4.8125,"inputPer1mUsd":1.75,"outputPer1mUsd":14,"contextWindow":400000,"releaseDate":"2025-12-11","paretoProb":0,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:openai/gpt-5.2-2025-12-11","observedHarness":"OpenHands","rawScore":53.5,"zScore":-0.556241,"seModel":"upstream_se","seRaw":5.067,"ciHalfWidth":9.9313,"ciLower":null,"ciUpper":null,"measuredCostUsd":17.745182,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"meta:muse-spark-1.1:high:direct:native","provider":"meta","modelSlug":"muse-spark-1.1","displayName":"Muse Spark 1.1","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.129922,"shrinkageFactor":0.999212,"score":48.05,"se":4.31,"samplingSe":0.485,"residualSe":4.2838,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.114414,"costLogSd":0.5216,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.0125,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Muse Spark 1.1","observedHarness":"Direct API","rawScore":1541.25,"zScore":-0.301993,"seModel":"upstream_ci","ciHalfWidth":7.684,"ciLower":1533.5695,"ciUpper":1548.9374,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-3.6-flash:medium:direct:native","provider":"google","modelSlug":"gemini-3.6-flash","displayName":"Gemini 3.6 Flash","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.136029,"shrinkageFactor":0.878889,"score":47.96,"se":6.49,"samplingSe":6.0112,"residualSe":2.4335,"coverageCount":3,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.100016,"costFixedFloorUsd":0.088207,"costLogSd":0.2909,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.5,"inputPer1mUsd":0.75,"outputPer1mUsd":3.75,"contextWindow":1048576,"releaseDate":"2026-05-05","paretoProb":0.091,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gemini-3.6-flash","observedHarness":"Direct API","rawScore":1237,"zScore":-0.033414,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.097217,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gemini-3.6-flash","observedHarness":"Direct API","rawScore":1236,"zScore":-0.092524,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.120269,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gemini-3.6-flash","observedHarness":"Direct API","rawScore":1287,"zScore":0.473696,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.135341,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"anthropic:claude-opus-4.6:thinking:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.6","displayName":"Claude Opus 4.6","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.155519,"shrinkageFactor":0.90813,"score":47.67,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.565459,"costLogSd":0.571,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-02-05","paretoProb":0.0003,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:claude-opus-4-6-thinking","observedHarness":"Direct API","rawScore":1238,"zScore":-0.024713,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:claude-opus-4-6-thinking","observedHarness":"Direct API","rawScore":1250,"zScore":0.056746,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:claude-opus-4-6-thinking","observedHarness":"Direct API","rawScore":1212,"zScore":0.015837,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:claude-opus-4-6-thinking","observedHarness":"Direct API","rawScore":1273,"zScore":0.243131,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"anthropic:claude-opus-4.6:none:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.6","displayName":"Claude Opus 4.6","effort":"none","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.1927,"shrinkageFactor":0.957487,"score":47.11,"se":3.92,"samplingSe":3.5615,"residualSe":1.6384,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.300721,"costLogSd":0.5854,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-02-05","paretoProb":0.0002,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:claude-opus-4-6","observedHarness":"Direct API","rawScore":1241,"zScore":0.001388,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:claude-opus-4-6","observedHarness":"Direct API","rawScore":1244,"zScore":-0.007227,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:claude-opus-4-6","observedHarness":"Direct API","rawScore":1199,"zScore":-0.141512,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:claude-opus-4-6","observedHarness":"Direct API","rawScore":1278,"zScore":0.325476,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Claude Opus 4.6","observedHarness":"Direct API","rawScore":1536.66,"zScore":-0.339862,"seModel":"upstream_ci","ciHalfWidth":5.3928,"ciLower":1531.2711,"ciUpper":1542.0567,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-sonnet-4.6:max:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-4.6","displayName":"Claude Sonnet 4.6","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.195662,"shrinkageFactor":0.974944,"score":47.07,"se":5.42,"samplingSe":2.7342,"residualSe":4.6801,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.522356,"costLogSd":0.5331,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1000000,"releaseDate":"2026-02-17","paretoProb":0.0002,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:anthropic/claude-sonnet-4-6","observedHarness":"OpenHands","rawScore":51.48,"zScore":-0.636521,"seModel":"upstream_se","seRaw":4.645,"ciHalfWidth":9.1042,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.907892,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"zhipu:glm-5.2:thinking:direct:native","provider":"zhipu","modelSlug":"glm-5.2","displayName":"GLM-5.2","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.239869,"shrinkageFactor":0.90813,"score":46.4,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.106788,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":1048576,"releaseDate":"2026-07-02","paretoProb":0.0915,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:glm-5.2","observedHarness":"Direct API","rawScore":1252,"zScore":0.097094,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.142618,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:glm-5.2","observedHarness":"Direct API","rawScore":1239,"zScore":-0.060538,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.136749,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:glm-5.2","observedHarness":"Direct API","rawScore":1189,"zScore":-0.262551,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.057761,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:glm-5.2","observedHarness":"Direct API","rawScore":1272,"zScore":0.226662,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.090022,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"anthropic:claude-sonnet-5:high:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-5","displayName":"Claude Sonnet 5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.250501,"shrinkageFactor":0.95748,"score":46.24,"se":3.92,"samplingSe":3.5618,"residualSe":1.6384,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.233989,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-07-15","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:claude-sonnet-5","observedHarness":"Direct API","rawScore":1224,"zScore":-0.146521,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.2436,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:claude-sonnet-5","observedHarness":"Direct API","rawScore":1256,"zScore":0.120718,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.370711,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:claude-sonnet-5","observedHarness":"Direct API","rawScore":1178,"zScore":-0.395693,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.078974,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:claude-sonnet-5","observedHarness":"Direct API","rawScore":1263,"zScore":0.078442,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.242672,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Claude Sonnet 5 (High)","observedHarness":"Direct API","rawScore":1539.44,"zScore":-0.316926,"seModel":"upstream_ci","ciHalfWidth":5.8027,"ciLower":1533.6376,"ciUpper":1545.243,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-6-luna:medium:direct:native","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.261967,"shrinkageFactor":0.90813,"score":46.07,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.004953,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.9999,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gpt-6-luna","observedHarness":"Direct API","rawScore":1181,"zScore":-0.520643,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.005141,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-6-luna","observedHarness":"Direct API","rawScore":1236,"zScore":-0.092524,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.005812,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gpt-6-luna","observedHarness":"Direct API","rawScore":1232,"zScore":0.257914,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.001876,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-6-luna","observedHarness":"Direct API","rawScore":1264,"zScore":0.094911,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.006982,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"openai:gpt-5.5:medium:direct:native","provider":"openai","modelSlug":"gpt-5.5","displayName":"GPT-5.5","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.289751,"shrinkageFactor":0.90813,"score":45.65,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.341719,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":11.25,"inputPer1mUsd":5,"outputPer1mUsd":30,"contextWindow":1050000,"releaseDate":"2026-04-23","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gpt-5.5","observedHarness":"Direct API","rawScore":1162,"zScore":-0.685953,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.360207,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-5.5","observedHarness":"Direct API","rawScore":1265,"zScore":0.216677,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.417792,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gpt-5.5","observedHarness":"Direct API","rawScore":1220,"zScore":0.112668,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.125741,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-5.5","observedHarness":"Direct API","rawScore":1243,"zScore":-0.250936,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.463135,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"alibaba:qwen3.7-max:thinking:direct:native","provider":"alibaba","modelSlug":"qwen3.7-max","displayName":"Qwen3.7 Max","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.29077,"shrinkageFactor":0.937526,"score":45.64,"se":4.7,"samplingSe":4.3174,"residualSe":1.8669,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.104638,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3.75,"inputPer1mUsd":2.5,"outputPer1mUsd":7.5,"contextWindow":1000000,"releaseDate":"2026-05-20","paretoProb":0.6005,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:qwen3.7-max","observedHarness":"Direct API","rawScore":1231,"zScore":-0.085617,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.089464,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:qwen3.7-max","observedHarness":"Direct API","rawScore":1235,"zScore":-0.103186,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.122576,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:qwen3.7-max","observedHarness":"Direct API","rawScore":1201,"zScore":-0.117305,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.055523,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:qwen3.7-max","observedHarness":"Direct API","rawScore":1263,"zScore":0.078442,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.150991,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:alibaba/qwen3.7-max","observedHarness":"OpenHands","rawScore":47.67,"zScore":-0.78794,"seModel":"upstream_se","seRaw":4.63,"ciHalfWidth":9.0748,"ciLower":null,"ciUpper":null,"measuredCostUsd":11.400322,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"google:gemini-3.5-flash:high:direct:native","provider":"google","modelSlug":"gemini-3.5-flash","displayName":"Gemini 3.5 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.292844,"shrinkageFactor":0.967541,"score":45.61,"se":3.43,"samplingSe":3.112,"residualSe":1.4441,"coverageCount":6,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.153971,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3.375,"inputPer1mUsd":1.5,"outputPer1mUsd":9,"contextWindow":1048576,"releaseDate":"2026-04-12","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gemini-3.5-flash","observedHarness":"Direct API","rawScore":1212,"zScore":-0.250927,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.137383,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gemini-3.5-flash","observedHarness":"Direct API","rawScore":1236,"zScore":-0.092524,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.168374,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gemini-3.5-flash","observedHarness":"Direct API","rawScore":1241,"zScore":0.366849,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.14027,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gemini-3.5-flash","observedHarness":"Direct API","rawScore":1255,"zScore":-0.053309,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.169857,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Gemini 3.5 Flash (High)","observedHarness":"Direct API","rawScore":1499.75,"zScore":-0.644375,"seModel":"upstream_ci","ciHalfWidth":7.323,"ciLower":1492.429,"ciUpper":1507.075,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:google/gemini-3.5-flash","observedHarness":"OpenHands","rawScore":48.68,"zScore":-0.7478,"seModel":"upstream_se","seRaw":4.727,"ciHalfWidth":9.2649,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.540385,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-5.4-mini:xhigh:direct:native","provider":"openai","modelSlug":"gpt-5.4-mini","displayName":"GPT-5.4 Mini","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.328304,"shrinkageFactor":0.963917,"score":45.08,"se":5.67,"samplingSe":3.2811,"residualSe":4.6272,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.148674,"costLogSd":0.5383,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.6875,"inputPer1mUsd":0.75,"outputPer1mUsd":4.5,"contextWindow":400000,"releaseDate":"2026-03-17","paretoProb":0.0009,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:openai/gpt-5.4-mini-2026-03-17","observedHarness":"OpenHands","rawScore":47.97,"zScore":-0.776018,"seModel":"upstream_se","seRaw":5.606,"ciHalfWidth":10.9878,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.191299,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"anthropic:claude-sonnet-4.6:none:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-4.6","displayName":"Claude Sonnet 4.6","effort":"none","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.361905,"shrinkageFactor":0.95749,"score":44.57,"se":3.92,"samplingSe":3.5614,"residualSe":1.6384,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.176487,"costLogSd":0.5798,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1000000,"releaseDate":"2026-02-17","paretoProb":0.001,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:claude-sonnet-4-6","observedHarness":"Direct API","rawScore":1201,"zScore":-0.346633,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:claude-sonnet-4-6","observedHarness":"Direct API","rawScore":1231,"zScore":-0.145835,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:claude-sonnet-4-6","observedHarness":"Direct API","rawScore":1181,"zScore":-0.359381,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:claude-sonnet-4-6","observedHarness":"Direct API","rawScore":1272,"zScore":0.226662,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Claude Sonnet 4.6","observedHarness":"Direct API","rawScore":1521.92,"zScore":-0.461469,"seModel":"upstream_ci","ciHalfWidth":5.1945,"ciLower":1516.7292,"ciUpper":1527.1182,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-opus-4.8:none:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.8","displayName":"Claude Opus 4.8","effort":"none","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.385316,"shrinkageFactor":0.957482,"score":44.22,"se":3.92,"samplingSe":3.5617,"residualSe":1.6384,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.272262,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-06-12","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:claude-opus-4-8","observedHarness":"Direct API","rawScore":1189,"zScore":-0.451039,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.240308,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:claude-opus-4-8","observedHarness":"Direct API","rawScore":1233,"zScore":-0.12451,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.385206,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:claude-opus-4-8","observedHarness":"Direct API","rawScore":1176,"zScore":-0.419901,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.096332,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:claude-opus-4-8","observedHarness":"Direct API","rawScore":1246,"zScore":-0.201529,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.367202,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Claude Opus 4.8","observedHarness":"Direct API","rawScore":1536.51,"zScore":-0.341099,"seModel":"upstream_ci","ciHalfWidth":5.7303,"ciLower":1530.783,"ciUpper":1542.2437,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"moonshot:kimi-k2.6:thinking:direct:native","provider":"moonshot","modelSlug":"kimi-k2.6","displayName":"Kimi K2.6","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.471472,"shrinkageFactor":0.967504,"score":42.93,"se":3.43,"samplingSe":3.1138,"residualSe":1.444,"coverageCount":6,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.06285,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.7125,"inputPer1mUsd":0.95,"outputPer1mUsd":4,"contextWindow":262144,"releaseDate":"2026-04-10","paretoProb":0.6718,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:kimi-k2.6","observedHarness":"Direct API","rawScore":1234,"zScore":-0.059515,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.06452,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:kimi-k2.6","observedHarness":"Direct API","rawScore":1222,"zScore":-0.241794,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.071084,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:kimi-k2.6","observedHarness":"Direct API","rawScore":1166,"zScore":-0.540939,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.022708,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:kimi-k2.6","observedHarness":"Direct API","rawScore":1258,"zScore":-0.003902,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.093086,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Kimi K2.6","observedHarness":"Direct API","rawScore":1509.14,"zScore":-0.566906,"seModel":"upstream_ci","ciHalfWidth":7.2788,"ciLower":1501.8596,"ciUpper":1516.4172,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:kimi/kimi-k2.6","observedHarness":"OpenHands","rawScore":37.89,"zScore":-1.176623,"seModel":"upstream_se","seRaw":4.912,"ciHalfWidth":9.6275,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.925351,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"google:gemini-3.5-flash:medium:direct:native","provider":"google","modelSlug":"gemini-3.5-flash","displayName":"Gemini 3.5 Flash","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.551847,"shrinkageFactor":0.9994,"score":41.72,"se":4.31,"samplingSe":0.4231,"residualSe":4.2846,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.092626,"costLogSd":0.3067,"costMaxUsd":0.153971,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3.375,"inputPer1mUsd":1.5,"outputPer1mUsd":9,"contextWindow":1048576,"releaseDate":"2026-04-12","paretoProb":0.115,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Gemini 3.5 Flash (Medium)","observedHarness":"Direct API","rawScore":1490.08,"zScore":-0.724154,"seModel":"upstream_ci","ciHalfWidth":6.7028,"ciLower":1483.3801,"ciUpper":1496.7856,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"zhipu:glm-5.1:thinking:direct:native","provider":"zhipu","modelSlug":"glm-5.1","displayName":"GLM-5.1","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.592061,"shrinkageFactor":0.997178,"score":41.12,"se":3.21,"samplingSe":0.9175,"residualSe":3.0791,"coverageCount":2,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.096163,"costLogSd":0.5574,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":204800,"releaseDate":"2026-04-15","paretoProb":0.0128,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:GLM-5.1","observedHarness":"Direct API","rawScore":1508.07,"zScore":-0.575734,"seModel":"upstream_ci","ciHalfWidth":6.8177,"ciLower":1501.2482,"ciUpper":1514.8837,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:zai/glm-5.1","observedHarness":"OpenHands","rawScore":31.46,"zScore":-1.432168,"seModel":"upstream_se","seRaw":4.553,"ciHalfWidth":8.9239,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.894911,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"moonshot:kimi-k2.7-code:max:direct:native","provider":"moonshot","modelSlug":"kimi-k2.7-code","displayName":"Kimi K2.7 Code","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.613686,"shrinkageFactor":0.967405,"score":40.79,"se":3.44,"samplingSe":3.1185,"residualSe":1.4439,"coverageCount":6,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.057139,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.7125,"inputPer1mUsd":0.95,"outputPer1mUsd":4,"contextWindow":262144,"releaseDate":"2026-06-15","paretoProb":1,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:kimi-k2.7-code","observedHarness":"Direct API","rawScore":1207,"zScore":-0.294429,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.052254,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:kimi-k2.7-code","observedHarness":"Direct API","rawScore":1193,"zScore":-0.550995,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.071041,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:kimi-k2.7-code","observedHarness":"Direct API","rawScore":1157,"zScore":-0.649874,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.032679,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:kimi-k2.7-code","observedHarness":"Direct API","rawScore":1252,"zScore":-0.102716,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.072583,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Kimi K2.7 Code","observedHarness":"Direct API","rawScore":1474.89,"zScore":-0.849474,"seModel":"upstream_ci","ciHalfWidth":9.4603,"ciLower":1465.4343,"ciUpper":1484.355,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:kimi/kimi-k2.7-code","observedHarness":"OpenHands","rawScore":47.21,"zScore":-0.806222,"seModel":"upstream_se","seRaw":5.191,"ciHalfWidth":10.1744,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.835806,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"google:gemini-3.1-pro:high:direct:native","provider":"google","modelSlug":"gemini-3.1-pro","displayName":"Gemini 3.1 Pro","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.628958,"shrinkageFactor":0.967644,"score":40.57,"se":3.43,"samplingSe":3.1071,"residualSe":1.4442,"coverageCount":6,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.171496,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4.5,"inputPer1mUsd":2,"outputPer1mUsd":12,"contextWindow":1048576,"releaseDate":"2026-02-19","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gemini-3.1-pro-preview","observedHarness":"Direct API","rawScore":1201,"zScore":-0.346633,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.133164,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gemini-3.1-pro-preview","observedHarness":"Direct API","rawScore":1173,"zScore":-0.764237,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.150842,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gemini-3.1-pro-preview","observedHarness":"Direct API","rawScore":1270,"zScore":0.71786,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.251654,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gemini-3.1-pro-preview","observedHarness":"Direct API","rawScore":1221,"zScore":-0.613252,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.150324,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Gemini 3.1 Pro Preview","observedHarness":"Direct API","rawScore":1446.34,"zScore":-1.085016,"seModel":"upstream_ci","ciHalfWidth":5.1016,"ciLower":1441.2395,"ciUpper":1451.4426,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:google/gemini-3.1-pro-preview","observedHarness":"OpenHands","rawScore":32.03,"zScore":-1.409515,"seModel":"upstream_se","seRaw":4.342,"ciHalfWidth":8.5103,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.825202,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-5.6-terra:medium:direct:native","provider":"openai","modelSlug":"gpt-5.6-terra","displayName":"GPT-5.6 Terra","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.641085,"shrinkageFactor":0.90813,"score":40.38,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.072346,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4.5,"inputPer1mUsd":2,"outputPer1mUsd":12,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0001,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gpt-5.6-terra","observedHarness":"Direct API","rawScore":1097,"zScore":-1.251486,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.070281,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-5.6-terra","observedHarness":"Direct API","rawScore":1201,"zScore":-0.465698,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.080712,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gpt-5.6-terra","observedHarness":"Direct API","rawScore":1203,"zScore":-0.093097,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.030365,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-5.6-terra","observedHarness":"Direct API","rawScore":1255,"zScore":-0.053309,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.108026,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"anthropic:claude-opus-4.5:none:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.5","displayName":"Claude Opus 4.5","effort":"none","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.691195,"shrinkageFactor":0.957454,"score":39.63,"se":3.92,"samplingSe":3.5629,"residualSe":1.6384,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.300721,"costLogSd":0.5854,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":200000,"releaseDate":"2025-11-24","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:claude-opus-4-5","observedHarness":"Direct API","rawScore":1180,"zScore":-0.529343,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:claude-opus-4-5","observedHarness":"Direct API","rawScore":1201,"zScore":-0.465698,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:claude-opus-4-5","observedHarness":"Direct API","rawScore":1165,"zScore":-0.553043,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:claude-opus-4-5","observedHarness":"Direct API","rawScore":1234,"zScore":-0.399156,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Claude Opus 4.5 (20251101)","observedHarness":"Direct API","rawScore":1468.93,"zScore":-0.898645,"seModel":"upstream_ci","ciHalfWidth":7.2753,"ciLower":1461.6547,"ciUpper":1476.2054,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-3.5-flash-lite:high:direct:native","provider":"google","modelSlug":"gemini-3.5-flash-lite","displayName":"Gemini 3.5 Flash-Lite","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.750655,"shrinkageFactor":0.975133,"score":38.74,"se":5.42,"samplingSe":2.7238,"residualSe":4.681,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.062576,"costLogSd":0.5276,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.85,"inputPer1mUsd":0.3,"outputPer1mUsd":2.5,"contextWindow":1048576,"releaseDate":"2026-04-12","paretoProb":0.5807,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:google/gemini-3.5-flash-lite","observedHarness":"OpenHands","rawScore":37.16,"zScore":-1.205635,"seModel":"upstream_se","seRaw":4.627,"ciHalfWidth":9.0689,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.833111,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"zhipu:glm-5:thinking:direct:native","provider":"zhipu","modelSlug":"glm-5","displayName":"GLM-5","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.851086,"shrinkageFactor":0.967654,"score":37.23,"se":3.43,"samplingSe":3.1065,"residualSe":1.4443,"coverageCount":6,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.069011,"costLogSd":0.5578,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.55,"inputPer1mUsd":1,"outputPer1mUsd":3.2,"contextWindow":204800,"releaseDate":"2026-02-01","paretoProb":0.0326,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:glm-5","observedHarness":"Direct API","rawScore":1196,"zScore":-0.390135,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:glm-5","observedHarness":"Direct API","rawScore":1206,"zScore":-0.412387,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:glm-5","observedHarness":"Direct API","rawScore":1153,"zScore":-0.698289,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:glm-5","observedHarness":"Direct API","rawScore":1235,"zScore":-0.382687,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:GLM-5","observedHarness":"Direct API","rawScore":1434.88,"zScore":-1.179563,"seModel":"upstream_ci","ciHalfWidth":8.2035,"ciLower":1426.6724,"ciUpper":1443.0794,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:zai/glm-5-thinking","observedHarness":"OpenHands","rawScore":23.36,"zScore":-1.754083,"seModel":"upstream_se","seRaw":4.034,"ciHalfWidth":7.9066,"ciLower":null,"ciUpper":null,"measuredCostUsd":40.270258,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-5.6-luna:medium:direct:native","provider":"openai","modelSlug":"gpt-5.6-luna","displayName":"GPT-5.6 Luna","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.862828,"shrinkageFactor":0.90813,"score":37.06,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.006149,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.45,"inputPer1mUsd":0.2,"outputPer1mUsd":1.2,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.1222,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gpt-5.6-luna","observedHarness":"Direct API","rawScore":1086,"zScore":-1.347192,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.005698,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-5.6-luna","observedHarness":"Direct API","rawScore":1172,"zScore":-0.774899,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.006499,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gpt-5.6-luna","observedHarness":"Direct API","rawScore":1177,"zScore":-0.407797,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.00326,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-5.6-luna","observedHarness":"Direct API","rawScore":1243,"zScore":-0.250936,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.009139,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"openai:gpt-5.2:none:direct:native","provider":"openai","modelSlug":"gpt-5.2","displayName":"GPT-5.2","effort":"none","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.131354,"shrinkageFactor":0.993158,"score":33.03,"se":4.49,"samplingSe":1.4288,"residualSe":4.2578,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.160239,"costLogSd":0.5791,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4.8125,"inputPer1mUsd":1.75,"outputPer1mUsd":14,"contextWindow":400000,"releaseDate":"2025-12-11","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:GPT-5.2","observedHarness":"Direct API","rawScore":1418.96,"zScore":-1.310905,"seModel":"upstream_ci","ciHalfWidth":22.7076,"ciLower":1396.2484,"ciUpper":1441.6636,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-3-flash:high:direct:native","provider":"google","modelSlug":"gemini-3-flash","displayName":"Gemini 3 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.139879,"shrinkageFactor":0.967885,"score":32.9,"se":3.54,"samplingSe":3.0955,"residualSe":1.7087,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.076676,"costLogSd":0.5252,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.125,"inputPer1mUsd":0.5,"outputPer1mUsd":3,"contextWindow":1048576,"releaseDate":"2025-12-15","paretoProb":0.2802,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gemini-3-flash-preview","observedHarness":"Direct API","rawScore":1150,"zScore":-0.790359,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gemini-3-flash-preview","observedHarness":"Direct API","rawScore":1138,"zScore":-1.137411,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gemini-3-flash-preview","observedHarness":"Direct API","rawScore":1183,"zScore":-1.239071,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Gemini 3 Flash","observedHarness":"Direct API","rawScore":1440.15,"zScore":-1.136084,"seModel":"upstream_ci","ciHalfWidth":8.6765,"ciLower":1431.4698,"ciUpper":1448.8227,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:google/gemini-3-flash-preview","observedHarness":"OpenHands","rawScore":20.2,"zScore":-1.879669,"seModel":"upstream_se","seRaw":3.946,"ciHalfWidth":7.7342,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.942067,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-5.4-nano:high:direct:native","provider":"openai","modelSlug":"gpt-5.4-nano","displayName":"GPT-5.4 Nano","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.173529,"shrinkageFactor":0.970235,"score":32.4,"se":5.53,"samplingSe":2.98,"residualSe":4.6575,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.031083,"costLogSd":0.5386,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.4625,"inputPer1mUsd":0.2,"outputPer1mUsd":1.25,"contextWindow":400000,"releaseDate":"2026-03-17","paretoProb":0.0016,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:openai/gpt-5.4-nano-2026-03-17","observedHarness":"OpenHands","rawScore":26.1,"zScore":-1.645188,"seModel":"upstream_se","seRaw":5.075,"ciHalfWidth":9.947,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.277298,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"spacexai:grok-4.20:thinking:direct:native","provider":"spacexai","modelSlug":"grok-4.20","displayName":"Grok 4.20","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.246694,"shrinkageFactor":0.967941,"score":31.3,"se":3.41,"samplingSe":3.0928,"residualSe":1.4447,"coverageCount":6,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.059332,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.5625,"inputPer1mUsd":1.25,"outputPer1mUsd":2.5,"contextWindow":2000000,"releaseDate":"2026-03-09","paretoProb":0.3784,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:grok-4-20-beta-reasoning","observedHarness":"Direct API","rawScore":1160,"zScore":-0.703354,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.050258,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:grok-4-20-beta-reasoning","observedHarness":"Direct API","rawScore":1164,"zScore":-0.860196,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.072285,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:grok-4-20-beta-reasoning","observedHarness":"Direct API","rawScore":1148,"zScore":-0.758808,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.022802,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:grok-4-20-beta-reasoning","observedHarness":"Direct API","rawScore":1216,"zScore":-0.695597,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.091985,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Grok 4.20 Beta (Reasoning, 0309)","observedHarness":"Direct API","rawScore":1376.38,"zScore":-1.662197,"seModel":"upstream_ci","ciHalfWidth":6.1224,"ciLower":1370.2599,"ciUpper":1382.5047,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:grok/grok-4.20-0309-reasoning","observedHarness":"OpenHands","rawScore":4.06,"zScore":-2.521115,"seModel":"upstream_se","seRaw":2.058,"ciHalfWidth":4.0337,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.771885,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-5.4:low:direct:native","provider":"openai","modelSlug":"gpt-5.4","displayName":"GPT-5.4","effort":"low","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.282091,"shrinkageFactor":0.90813,"score":30.77,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.174572,"costLogSd":0.5798,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":5.625,"inputPer1mUsd":2.5,"outputPer1mUsd":15,"contextWindow":1050000,"releaseDate":"2026-03-05","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gpt-5.4-low","observedHarness":"Direct API","rawScore":1032,"zScore":-1.81702,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-5.4-low","observedHarness":"Direct API","rawScore":1163,"zScore":-0.870859,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gpt-5.4-low","observedHarness":"Direct API","rawScore":1126,"zScore":-1.025092,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-5.4-low","observedHarness":"Direct API","rawScore":1185,"zScore":-1.206133,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"openai:gpt-5.4-mini:high:direct:native","provider":"openai","modelSlug":"gpt-5.4-mini","displayName":"GPT-5.4 Mini","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.30961,"shrinkageFactor":0.999422,"score":30.36,"se":4.3,"samplingSe":0.4152,"residualSe":4.2847,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.116525,"costLogSd":0.5215,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.6875,"inputPer1mUsd":0.75,"outputPer1mUsd":4.5,"contextWindow":400000,"releaseDate":"2026-03-17","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:GPT-5.4 mini (High)","observedHarness":"Direct API","rawScore":1398.18,"zScore":-1.482343,"seModel":"upstream_ci","ciHalfWidth":6.5784,"ciLower":1391.5991,"ciUpper":1404.7559,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-5:high:direct:native","provider":"openai","modelSlug":"gpt-5","displayName":"GPT-5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.311162,"shrinkageFactor":0.937914,"score":30.33,"se":4.69,"samplingSe":4.304,"residualSe":1.8676,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.262714,"costLogSd":0.5189,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3.4375,"inputPer1mUsd":1.25,"outputPer1mUsd":10,"contextWindow":400000,"releaseDate":"2025-08-07","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gpt-5-high","observedHarness":"Direct API","rawScore":993,"zScore":-2.15634,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-5-high","observedHarness":"Direct API","rawScore":1157,"zScore":-0.934831,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gpt-5-high","observedHarness":"Direct API","rawScore":1173,"zScore":-0.456212,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-5-high","observedHarness":"Direct API","rawScore":1169,"zScore":-1.469636,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:openai/gpt-5-2025-08-07","observedHarness":"OpenHands","rawScore":20.09,"zScore":-1.884041,"seModel":"upstream_se","seRaw":3.409,"ciHalfWidth":6.6816,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.525167,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"anthropic:claude-sonnet-4.5:thinking:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-4.5","displayName":"Claude Sonnet 4.5","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.337561,"shrinkageFactor":0.937824,"score":29.94,"se":4.69,"samplingSe":4.3071,"residualSe":1.8675,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.331638,"costLogSd":0.5642,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1000000,"releaseDate":"2025-09-29","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:claude-sonnet-4-5-thinking","observedHarness":"Direct API","rawScore":1119,"zScore":-1.060075,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:claude-sonnet-4-5-thinking","observedHarness":"Direct API","rawScore":1135,"zScore":-1.169398,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:claude-sonnet-4-5-thinking","observedHarness":"Direct API","rawScore":1099,"zScore":-1.351896,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:claude-sonnet-4-5-thinking","observedHarness":"Direct API","rawScore":1177,"zScore":-1.337884,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:anthropic/claude-sonnet-4-5-20250929-thinking","observedHarness":"OpenHands","rawScore":22.62,"zScore":-1.783492,"seModel":"upstream_se","seRaw":3.728,"ciHalfWidth":7.3069,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.656984,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-5.1:high:direct:native","provider":"openai","modelSlug":"gpt-5.1","displayName":"GPT-5.1","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.339252,"shrinkageFactor":0.937658,"score":29.91,"se":4.7,"samplingSe":4.3128,"residualSe":1.8671,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.262714,"costLogSd":0.5189,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3.4375,"inputPer1mUsd":1.25,"outputPer1mUsd":10,"contextWindow":400000,"releaseDate":"2025-11-13","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gpt-5.1-high","observedHarness":"Direct API","rawScore":1028,"zScore":-1.851822,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-5.1-high","observedHarness":"Direct API","rawScore":1150,"zScore":-1.009466,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gpt-5.1-high","observedHarness":"Direct API","rawScore":1133,"zScore":-0.940366,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-5.1-high","observedHarness":"Direct API","rawScore":1174,"zScore":-1.387291,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:openai/gpt-5.1-2025-11-13","observedHarness":"OpenHands","rawScore":24.61,"zScore":-1.704405,"seModel":"upstream_se","seRaw":4.254,"ciHalfWidth":8.3378,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.574926,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-5.4:none:direct:native","provider":"openai","modelSlug":"gpt-5.4","displayName":"GPT-5.4","effort":"none","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.361684,"shrinkageFactor":0.957392,"score":29.57,"se":3.92,"samplingSe":3.5655,"residualSe":1.6383,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.174572,"costLogSd":0.5798,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":5.625,"inputPer1mUsd":2.5,"outputPer1mUsd":15,"contextWindow":1050000,"releaseDate":"2026-03-05","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gpt-5.4-none","observedHarness":"Direct API","rawScore":1014,"zScore":-1.973629,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-5.4-none","observedHarness":"Direct API","rawScore":1166,"zScore":-0.838872,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gpt-5.4-none","observedHarness":"Direct API","rawScore":1111,"zScore":-1.20665,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-5.4-none","observedHarness":"Direct API","rawScore":1186,"zScore":-1.189664,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:GPT-5.4","observedHarness":"Direct API","rawScore":1399.99,"zScore":-1.467411,"seModel":"upstream_ci","ciHalfWidth":9.858,"ciLower":1390.135,"ciUpper":1409.851,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-sonnet-4.5:none:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-4.5","displayName":"Claude Sonnet 4.5","effort":"none","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.379451,"shrinkageFactor":0.957452,"score":29.31,"se":3.92,"samplingSe":3.563,"residualSe":1.6384,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.176487,"costLogSd":0.5798,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1000000,"releaseDate":"2025-09-29","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:claude-sonnet-4-5","observedHarness":"Direct API","rawScore":1124,"zScore":-1.016572,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:claude-sonnet-4-5","observedHarness":"Direct API","rawScore":1138,"zScore":-1.137411,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:claude-sonnet-4-5","observedHarness":"Direct API","rawScore":1092,"zScore":-1.436623,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:claude-sonnet-4-5","observedHarness":"Direct API","rawScore":1176,"zScore":-1.354353,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Claude Sonnet 4.5 (20250929)","observedHarness":"Direct API","rawScore":1384.84,"zScore":-1.5924,"seModel":"upstream_ci","ciHalfWidth":7.357,"ciLower":1377.4788,"ciUpper":1392.1928,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"spacexai:grok-4.20:none:direct:native","provider":"spacexai","modelSlug":"grok-4.20","displayName":"Grok 4.20","effort":"none","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.395532,"shrinkageFactor":0.90813,"score":29.07,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.056345,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.5625,"inputPer1mUsd":1.25,"outputPer1mUsd":2.5,"contextWindow":2000000,"releaseDate":"2026-03-09","paretoProb":0.4017,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:grok-4-20-beta-non-reasoning","observedHarness":"Direct API","rawScore":1119,"zScore":-1.060075,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.048324,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:grok-4-20-beta-non-reasoning","observedHarness":"Direct API","rawScore":1147,"zScore":-1.041452,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.067466,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:grok-4-20-beta-non-reasoning","observedHarness":"Direct API","rawScore":1041,"zScore":-2.053918,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.021114,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:grok-4-20-beta-non-reasoning","observedHarness":"Direct API","rawScore":1206,"zScore":-0.860286,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.088477,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"anthropic:claude-opus-4.5:high:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.5","displayName":"Claude Opus 4.5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.409516,"shrinkageFactor":0.988253,"score":28.86,"se":5.1,"samplingSe":1.8721,"residualSe":4.744,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.698137,"costLogSd":0.5239,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":200000,"releaseDate":"2025-11-24","paretoProb":0,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:anthropic/claude-opus-4-5-20251101-thinking","observedHarness":"OpenHands","rawScore":20.63,"zScore":-1.86258,"seModel":"upstream_se","seRaw":3.159,"ciHalfWidth":6.1916,"ciLower":null,"ciUpper":null,"measuredCostUsd":32.873283,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-5:medium:direct:native","provider":"openai","modelSlug":"gpt-5","displayName":"GPT-5","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.410028,"shrinkageFactor":0.957187,"score":28.85,"se":3.93,"samplingSe":3.574,"residualSe":1.6379,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.156971,"costLogSd":0.5167,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3.4375,"inputPer1mUsd":1.25,"outputPer1mUsd":10,"contextWindow":400000,"releaseDate":"2025-08-07","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gpt-5","observedHarness":"Direct API","rawScore":1022,"zScore":-1.904025,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-5","observedHarness":"Direct API","rawScore":1127,"zScore":-1.254695,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gpt-5","observedHarness":"Direct API","rawScore":1112,"zScore":-1.194546,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-5","observedHarness":"Direct API","rawScore":1172,"zScore":-1.420229,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:GPT-5 (Medium)","observedHarness":"Direct API","rawScore":1417.54,"zScore":-1.32262,"seModel":"upstream_ci","ciHalfWidth":15.6118,"ciLower":1401.9305,"ciUpper":1433.154,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-3-flash:minimal:direct:native","provider":"google","modelSlug":"gemini-3-flash","displayName":"Gemini 3 Flash","effort":"minimal","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.423022,"shrinkageFactor":0.999627,"score":28.65,"se":4.3,"samplingSe":0.3335,"residualSe":4.2856,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.032856,"costLogSd":0.5776,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.125,"inputPer1mUsd":0.5,"outputPer1mUsd":3,"contextWindow":1048576,"releaseDate":"2025-12-15","paretoProb":0.7646,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Gemini 3 Flash (Minimal)","observedHarness":"Direct API","rawScore":1384.46,"zScore":-1.595536,"seModel":"upstream_ci","ciHalfWidth":5.2822,"ciLower":1379.177,"ciUpper":1389.7414,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"spacexai:grok-4.3:high:direct:native","provider":"spacexai","modelSlug":"grok-4.3","displayName":"Grok 4.3","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.444509,"shrinkageFactor":0.979007,"score":28.33,"se":5.32,"samplingSe":2.5027,"residualSe":4.6996,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.027151,"costLogSd":0.3067,"costMinUsd":0.010769,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.5625,"inputPer1mUsd":1.25,"outputPer1mUsd":2.5,"contextWindow":1000000,"releaseDate":"2026-05-01","paretoProb":0.5708,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:grok/grok-4.3","observedHarness":"OpenHands","rawScore":19.4,"zScore":-1.911463,"seModel":"upstream_se","seRaw":4.243,"ciHalfWidth":8.3163,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.28472,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"openai:gpt-5.1:medium:direct:native","provider":"openai","modelSlug":"gpt-5.1","displayName":"GPT-5.1","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.481675,"shrinkageFactor":0.957339,"score":27.77,"se":3.93,"samplingSe":3.5677,"residualSe":1.6382,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.156971,"costLogSd":0.5167,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3.4375,"inputPer1mUsd":1.25,"outputPer1mUsd":10,"contextWindow":400000,"releaseDate":"2025-11-13","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gpt-5.1-medium","observedHarness":"Direct API","rawScore":1005,"zScore":-2.051934,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-5.1-medium","observedHarness":"Direct API","rawScore":1131,"zScore":-1.212046,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gpt-5.1-medium","observedHarness":"Direct API","rawScore":1115,"zScore":-1.158235,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-5.1-medium","observedHarness":"Direct API","rawScore":1165,"zScore":-1.535511,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:GPT-5.1 (Medium)","observedHarness":"Direct API","rawScore":1397.25,"zScore":-1.490016,"seModel":"upstream_ci","ciHalfWidth":11.6034,"ciLower":1385.6455,"ciUpper":1408.8522,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"spacexai:grok-4.3:low:direct:native","provider":"spacexai","modelSlug":"grok-4.3","displayName":"Grok 4.3","effort":"low","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.534105,"shrinkageFactor":0.957472,"score":26.99,"se":3.92,"samplingSe":3.5621,"residualSe":1.6384,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.010769,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.5625,"inputPer1mUsd":1.25,"outputPer1mUsd":2.5,"contextWindow":1000000,"releaseDate":"2026-05-01","paretoProb":1,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:grok-4.3","observedHarness":"Direct API","rawScore":1091,"zScore":-1.303689,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.008812,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:grok-4.3","observedHarness":"Direct API","rawScore":1143,"zScore":-1.084101,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.013188,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:grok-4.3","observedHarness":"Direct API","rawScore":1066,"zScore":-1.751323,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.00369,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:grok-4.3","observedHarness":"Direct API","rawScore":1181,"zScore":-1.272009,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.017386,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Grok 4.3","observedHarness":"Direct API","rawScore":1357.51,"zScore":-1.817877,"seModel":"upstream_ci","ciHalfWidth":6.2825,"ciLower":1351.2266,"ciUpper":1363.7916,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-5.1:low:direct:native","provider":"openai","modelSlug":"gpt-5.1","displayName":"GPT-5.1","effort":"low","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.58561,"shrinkageFactor":0.90813,"score":26.22,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.112903,"costLogSd":0.577,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3.4375,"inputPer1mUsd":1.25,"outputPer1mUsd":10,"contextWindow":400000,"releaseDate":"2025-11-13","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gpt-5.1-low","observedHarness":"Direct API","rawScore":1009,"zScore":-2.017132,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-5.1-low","observedHarness":"Direct API","rawScore":1123,"zScore":-1.297343,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gpt-5.1-low","observedHarness":"Direct API","rawScore":1102,"zScore":-1.315585,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-5.1-low","observedHarness":"Direct API","rawScore":1159,"zScore":-1.634325,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"openai:gpt-5.3-codex:medium:direct:native","provider":"openai","modelSlug":"gpt-5.3-codex","displayName":"GPT-5.3 Codex","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.601829,"shrinkageFactor":0.90813,"score":25.97,"se":5.63,"samplingSe":5.2355,"residualSe":2.0731,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.222721,"costLogSd":0.5186,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4.8125,"inputPer1mUsd":1.75,"outputPer1mUsd":14,"contextWindow":400000,"releaseDate":"2026-02-15","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gpt-5.3-codex","observedHarness":"Direct API","rawScore":975,"zScore":-2.312949,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-5.3-codex","observedHarness":"Direct API","rawScore":1133,"zScore":-1.190722,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gpt-5.3-codex","observedHarness":"Direct API","rawScore":1113,"zScore":-1.182442,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-5.3-codex","observedHarness":"Direct API","rawScore":1150,"zScore":-1.782545,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"openai:gpt-5.1:none:direct:native","provider":"openai","modelSlug":"gpt-5.1","displayName":"GPT-5.1","effort":"none","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.749568,"shrinkageFactor":0.957425,"score":23.76,"se":3.92,"samplingSe":3.5641,"residualSe":1.6383,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.112903,"costLogSd":0.577,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3.4375,"inputPer1mUsd":1.25,"outputPer1mUsd":10,"contextWindow":400000,"releaseDate":"2025-11-13","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gpt-5.1-none","observedHarness":"Direct API","rawScore":1017,"zScore":-1.947528,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-5.1-none","observedHarness":"Direct API","rawScore":1104,"zScore":-1.499923,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gpt-5.1-none","observedHarness":"Direct API","rawScore":1088,"zScore":-1.485038,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-5.1-none","observedHarness":"Direct API","rawScore":1154,"zScore":-1.716669,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:GPT-5.1","observedHarness":"Direct API","rawScore":1342.35,"zScore":-1.94295,"seModel":"upstream_ci","ciHalfWidth":8.5659,"ciLower":1333.7808,"ciUpper":1350.9127,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-haiku-4.5:none:direct:native","provider":"anthropic","modelSlug":"claude-haiku-4.5","displayName":"Claude Haiku 4.5","effort":"none","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-2.07389,"shrinkageFactor":0.957493,"score":18.89,"se":3.92,"samplingSe":3.5613,"residualSe":1.6384,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.056354,"costLogSd":0.5758,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":200000,"releaseDate":"2025-10-15","paretoProb":0.0399,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:claude-haiku-4-5","observedHarness":"Direct API","rawScore":1039,"zScore":-1.756116,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:claude-haiku-4-5","observedHarness":"Direct API","rawScore":1069,"zScore":-1.873098,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:claude-haiku-4-5","observedHarness":"Direct API","rawScore":1010,"zScore":-2.429137,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:claude-haiku-4-5","observedHarness":"Direct API","rawScore":1109,"zScore":-2.45777,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Claude Haiku 4.5 (20251001)","observedHarness":"Direct API","rawScore":1330.22,"zScore":-2.043024,"seModel":"upstream_ci","ciHalfWidth":5.0002,"ciLower":1325.2187,"ciUpper":1335.2191,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]}],"provisionalConfigs":[{"id":"meta:muse-wonder-full:max:direct:native","provider":"meta","modelSlug":"muse-wonder-full","displayName":"Muse Wonder Full","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":1.243719,"shrinkageFactor":0.996598,"score":68.66,"se":2.46,"samplingSe":1.0076,"residualSe":2.2416,"coverageCount":3,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Muse Wonder Full (Max)","observedHarness":"Direct API","rawScore":1722.63,"zScore":1.19442,"seModel":"upstream_ci","ciHalfWidth":14.4207,"ciLower":1708.2095,"ciUpper":1737.0509,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_swebench_multimodal","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:swebench-multimodal:meta/muse-wonder-full","observedHarness":"mini-SWE-agent","rawScore":47.29,"zScore":1.054977,"seModel":"upstream_se","seRaw":2.281,"ciHalfWidth":4.4708,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.860894,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:meta/muse-wonder-full","observedHarness":"OpenHands","rawScore":86,"zScore":0.735393,"seModel":"upstream_se","seRaw":1.933,"ciHalfWidth":3.7887,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.293448,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"meta:muse-spark-open:xhigh:direct:native","provider":"meta","modelSlug":"muse-spark-open","displayName":"Muse Spark Open","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.771674,"shrinkageFactor":0.987064,"score":61.58,"se":5.13,"samplingSe":1.9646,"residualSe":4.7383,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:meta/muse-spark-open","observedHarness":"OpenHands","rawScore":76.19,"zScore":0.345518,"seModel":"upstream_se","seRaw":3.317,"ciHalfWidth":6.5013,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.655609,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"alibaba:qwen3.8-max:xhigh:direct:native","provider":"alibaba","modelSlug":"qwen3.8-max","displayName":"Qwen3.8 Max","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.760974,"shrinkageFactor":0.995512,"score":61.41,"se":2.52,"samplingSe":1.1571,"residualSe":2.2391,"coverageCount":3,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-08-03","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Qwen3.8-Max","observedHarness":"Direct API","rawScore":1670.36,"zScore":0.763184,"seModel":"upstream_ci","ciHalfWidth":12.1218,"ciLower":1658.2419,"ciUpper":1682.4855,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_swebench_multimodal","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:swebench-multimodal:alibaba/qwen3.8-max","observedHarness":"mini-SWE-agent","rawScore":42.5,"zScore":0.736556,"seModel":"upstream_se","seRaw":2.259,"ciHalfWidth":4.4276,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.994423,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:alibaba/qwen3.8-max","observedHarness":"OpenHands","rawScore":64.7,"zScore":-0.111124,"seModel":"upstream_se","seRaw":5.362,"ciHalfWidth":10.5095,"ciLower":null,"ciUpper":null,"measuredCostUsd":8.238852,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"deepseek:deepseek-v4.1-flash:max:direct:native","provider":"deepseek","modelSlug":"deepseek-v4.1-flash","displayName":"DeepSeek V4.1 Flash","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.756735,"shrinkageFactor":0.995299,"score":61.35,"se":3.03,"samplingSe":1.1843,"residualSe":2.7893,"coverageCount":2,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-10","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:DeepSeek-V4.1-Flash-20260910 (Max)","observedHarness":"Direct API","rawScore":1621.31,"zScore":0.358514,"seModel":"upstream_ci","ciHalfWidth":11.137,"ciLower":1610.1743,"ciUpper":1632.4483,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_swebench_multimodal","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:swebench-multimodal:deepseek/deepseek-v4.1-flash","observedHarness":"mini-SWE-agent","rawScore":46.46,"zScore":0.999802,"seModel":"upstream_se","seRaw":2.279,"ciHalfWidth":4.4668,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.225505,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/swebench-multimodal"}]},{"id":"alibaba:qwen3.8-flash-next:xhigh:direct:native","provider":"alibaba","modelSlug":"qwen3.8-flash-next","displayName":"Qwen3.8 Flash Next","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.56652,"shrinkageFactor":0.999379,"score":58.5,"se":4.31,"samplingSe":0.4305,"residualSe":4.2845,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-15","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Qwen3.8-Flash-Next","observedHarness":"Direct API","rawScore":1625.72,"zScore":0.394897,"seModel":"upstream_ci","ciHalfWidth":6.8208,"ciLower":1618.8955,"ciUpper":1632.537,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"deepseek:deepseek-v4.1-flash:high:direct:native","provider":"deepseek","modelSlug":"deepseek-v4.1-flash","displayName":"DeepSeek V4.1 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.434362,"shrinkageFactor":0.938043,"score":56.52,"se":4.69,"samplingSe":4.2995,"residualSe":1.8679,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-10","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:deepseek-v4-1-flash","observedHarness":"Direct API","rawScore":1325,"zScore":0.732232,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.050126,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:deepseek-v4-1-flash","observedHarness":"Direct API","rawScore":1308,"zScore":0.675149,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.05299,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:deepseek-v4-1-flash","observedHarness":"Direct API","rawScore":1225,"zScore":0.173187,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.023268,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:deepseek-v4-1-flash","observedHarness":"Direct API","rawScore":1293,"zScore":0.57251,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.050123,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:deepseek/deepseek-v4.1-flash","observedHarness":"OpenHands","rawScore":84.74,"zScore":0.685318,"seModel":"upstream_se","seRaw":2.891,"ciHalfWidth":5.6664,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.407445,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"deepseek:deepseek-v4-flash:high:direct:native","provider":"deepseek","modelSlug":"deepseek-v4-flash","displayName":"DeepSeek V4 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.372378,"shrinkageFactor":0.998005,"score":55.59,"se":3.18,"samplingSe":0.7715,"residualSe":3.0816,"coverageCount":2,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-31","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:DeepSeek-V4-Flash (High)","observedHarness":"Direct API","rawScore":1582.11,"zScore":0.035108,"seModel":"upstream_ci","ciHalfWidth":9.4106,"ciLower":1572.7041,"ciUpper":1591.5252,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:deepseek/deepseek-v4-flash-0731","observedHarness":"OpenHands","rawScore":74.74,"zScore":0.287892,"seModel":"upstream_se","seRaw":3.442,"ciHalfWidth":6.7463,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.905592,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"alibaba:qwen3.8-27b:xhigh:direct:native","provider":"alibaba","modelSlug":"qwen3.8-27b","displayName":"Qwen3.8 27B","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":0.353426,"shrinkageFactor":0.996283,"score":55.3,"se":2.14,"samplingSe":1.0531,"residualSe":1.8623,"coverageCount":3,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-05","paretoProb":0,"benchmarks":[{"benchmarkId":"aa_code2video","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_code2video:b01dee41-c62b-48ed-8d16-984adc405e5c","observedHarness":"Stirrup","rawScore":949.04,"zScore":0.437233,"seModel":"upstream_ci","ciHalfWidth":20.08,"ciLower":928.96,"ciUpper":969.12,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/code2video-aa"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Qwen3.8-27B","observedHarness":"Direct API","rawScore":1589.53,"zScore":0.096324,"seModel":"upstream_ci","ciHalfWidth":5.8962,"ciLower":1583.6368,"ciUpper":1595.4293,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:alibaba/qwen3.8-27b","observedHarness":"OpenHands","rawScore":64.85,"zScore":-0.105163,"seModel":"upstream_se","seRaw":4.825,"ciHalfWidth":9.457,"ciLower":null,"ciUpper":null,"measuredCostUsd":23.456372,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"meta:muse-spark-open:high:direct:native","provider":"meta","modelSlug":"muse-spark-open","displayName":"Muse Spark Open","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.108828,"shrinkageFactor":0.878889,"score":48.37,"se":6.49,"samplingSe":6.0112,"residualSe":2.4335,"coverageCount":3,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:muse-spark-open","observedHarness":"Direct API","rawScore":1222,"zScore":-0.163922,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.064511,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:muse-spark-open","observedHarness":"Direct API","rawScore":1242,"zScore":-0.028551,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.07499,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:muse-spark-open","observedHarness":"Direct API","rawScore":1300,"zScore":0.687792,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.081175,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"meta:muse-spark-open:max:direct:native","provider":"meta","modelSlug":"muse-spark-open","displayName":"Muse Spark Open","effort":"max","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.244191,"shrinkageFactor":0.997306,"score":46.34,"se":4.37,"samplingSe":0.8966,"residualSe":4.2756,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Muse Spark Open (Max)","observedHarness":"Direct API","rawScore":1527.34,"zScore":-0.416753,"seModel":"upstream_ci","ciHalfWidth":14.2191,"ciLower":1513.1232,"ciUpper":1541.5614,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"alibaba:qwen3.7-max-0517:thinking:direct:native","provider":"alibaba","modelSlug":"qwen3.7-max-0517","displayName":"Qwen3.7 Max (0517)","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.340647,"shrinkageFactor":0.999266,"score":44.89,"se":4.31,"samplingSe":0.4679,"residualSe":4.284,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-05-17","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Qwen3.7-Max (20260517)","observedHarness":"Direct API","rawScore":1515.69,"zScore":-0.512867,"seModel":"upstream_ci","ciHalfWidth":7.4136,"ciLower":1508.2727,"ciUpper":1523.1,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"alibaba:qwen3.7-plus:thinking:direct:native","provider":"alibaba","modelSlug":"qwen3.7-plus","displayName":"Qwen3.7 Plus","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.379479,"shrinkageFactor":0.937531,"score":44.31,"se":4.7,"samplingSe":4.3172,"residualSe":1.8669,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":0.039163,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-10","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:qwen3.7-plus","observedHarness":"Direct API","rawScore":1205,"zScore":-0.31183,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.029296,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:qwen3.7-plus","observedHarness":"Direct API","rawScore":1232,"zScore":-0.135172,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.045819,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:qwen3.7-plus","observedHarness":"Direct API","rawScore":1194,"zScore":-0.202032,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.02202,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:qwen3.7-plus","observedHarness":"Direct API","rawScore":1256,"zScore":-0.03684,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.059518,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:alibaba/qwen3.7-plus","observedHarness":"OpenHands","rawScore":46.39,"zScore":-0.838811,"seModel":"upstream_se","seRaw":4.614,"ciHalfWidth":9.0434,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.082348,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"deepseek:deepseek-v4-pro:high:direct:native","provider":"deepseek","modelSlug":"deepseek-v4-pro","displayName":"DeepSeek V4 Pro (0424)","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-0.768245,"shrinkageFactor":0.999459,"score":38.48,"se":4.3,"samplingSe":0.4017,"residualSe":4.2848,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-04-24","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:DeepSeek-V4-Pro Preview (High)","observedHarness":"Direct API","rawScore":1463.84,"zScore":-0.940638,"seModel":"upstream_ci","ciHalfWidth":6.3642,"ciLower":1457.48,"ciUpper":1470.2083,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-3-pro:high:direct:native","provider":"google","modelSlug":"gemini-3-pro","displayName":"Gemini 3 Pro","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.185698,"shrinkageFactor":0.998407,"score":32.21,"se":3.16,"samplingSe":0.6894,"residualSe":3.0829,"coverageCount":2,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-11-18","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Gemini 3 Pro","observedHarness":"Direct API","rawScore":1441.13,"zScore":-1.127999,"seModel":"upstream_ci","ciHalfWidth":8.5087,"ciLower":1432.6222,"ciUpper":1449.6397,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:google/gemini-3-pro-preview","observedHarness":"OpenHands","rawScore":14.3,"zScore":-2.114151,"seModel":"upstream_se","seRaw":3.061,"ciHalfWidth":5.9996,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.417592,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"alibaba:qwen3.5-397b-a17b:thinking:direct:native","provider":"alibaba","modelSlug":"qwen3.5-397b-a17b","displayName":"Qwen3.5 397B A17B","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.31746,"shrinkageFactor":0.95749,"score":30.24,"se":3.92,"samplingSe":3.5614,"residualSe":1.6384,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-03-20","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:qwen3.5-397b-a17b","observedHarness":"Direct API","rawScore":1128,"zScore":-0.98177,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:qwen3.5-397b-a17b","observedHarness":"Direct API","rawScore":1112,"zScore":-1.414627,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:qwen3.5-397b-a17b","observedHarness":"Direct API","rawScore":1124,"zScore":-1.0493,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:qwen3.5-397b-a17b","observedHarness":"Direct API","rawScore":1178,"zScore":-1.321415,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Qwen3.5-397B-A17B","observedHarness":"Direct API","rawScore":1399.95,"zScore":-1.467741,"seModel":"upstream_ci","ciHalfWidth":5.1724,"ciLower":1394.7772,"ciUpper":1405.1221,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"meta:muse-spark:high:direct:native","provider":"meta","modelSlug":"muse-spark","displayName":"Muse Spark","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.437724,"shrinkageFactor":0.981609,"score":28.43,"se":5.26,"samplingSe":2.3424,"residualSe":4.7121,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:meta/muse_spark","observedHarness":"OpenHands","rawScore":19.67,"zScore":-1.900733,"seModel":"upstream_se","seRaw":3.966,"ciHalfWidth":7.7734,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.579705,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"google:gemma-4-31b:none:direct:native","provider":"google","modelSlug":"gemma-4-31b","displayName":"Gemma 4 31B","effort":"none","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.540138,"shrinkageFactor":0.999523,"score":26.9,"se":4.3,"samplingSe":0.3774,"residualSe":4.2851,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-01","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Gemma 4 31B","observedHarness":"Direct API","rawScore":1370.24,"zScore":-1.712853,"seModel":"upstream_ci","ciHalfWidth":5.9786,"ciLower":1364.2658,"ciUpper":1376.223,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"deepseek:deepseek-v3.2:thinking:direct:native","provider":"deepseek","modelSlug":"deepseek-v3.2","displayName":"DeepSeek V3.2","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.603122,"shrinkageFactor":0.998882,"score":25.95,"se":4.32,"samplingSe":0.5776,"residualSe":4.2824,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-12-01","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:DeepSeek-V3.2 (Thinking)","observedHarness":"Direct API","rawScore":1362.48,"zScore":-1.776874,"seModel":"upstream_ci","ciHalfWidth":9.1536,"ciLower":1353.3306,"ciUpper":1371.6379,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"alibaba:qwen3.5-122b-a10b:thinking:direct:native","provider":"alibaba","modelSlug":"qwen3.5-122b-a10b","displayName":"Qwen3.5 122B A10B","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.622349,"shrinkageFactor":0.999209,"score":25.66,"se":4.31,"samplingSe":0.4858,"residualSe":4.2838,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-03-20","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Qwen3.5-122B-A10B","observedHarness":"Direct API","rawScore":1360.21,"zScore":-1.795602,"seModel":"upstream_ci","ciHalfWidth":7.6966,"ciLower":1352.5156,"ciUpper":1367.9089,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemma-4-26b-a4b:none:direct:native","provider":"google","modelSlug":"gemma-4-26b-a4b","displayName":"Gemma 4 26B A4B","effort":"none","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.638943,"shrinkageFactor":0.996153,"score":25.42,"se":4.4,"samplingSe":1.0714,"residualSe":4.2707,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-01","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Gemma 4 26B A4B","observedHarness":"Direct API","rawScore":1357.6,"zScore":-1.817135,"seModel":"upstream_ci","ciHalfWidth":17.0009,"ciLower":1340.5959,"ciUpper":1374.5977,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"meta:muse-glimmer:high:direct:native","provider":"meta","modelSlug":"muse-glimmer","displayName":"Muse Glimmer","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.669353,"shrinkageFactor":0.95487,"score":24.96,"se":4.17,"samplingSe":3.6695,"residualSe":1.9714,"coverageCount":4,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:muse-glimmer-30b","observedHarness":"Direct API","rawScore":1056,"zScore":-1.608208,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.007494,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:muse-glimmer-30b","observedHarness":"Direct API","rawScore":1086,"zScore":-1.691842,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.01035,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:muse-glimmer-30b","observedHarness":"Direct API","rawScore":1175,"zScore":-1.370822,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.00987,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Muse Glimmer","observedHarness":"Direct API","rawScore":1353,"zScore":-1.855085,"seModel":"upstream_ci","ciHalfWidth":15.7446,"ciLower":1337.2593,"ciUpper":1368.7485,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"deepseek:deepseek-v3.2:none:direct:native","provider":"deepseek","modelSlug":"deepseek-v3.2","displayName":"DeepSeek V3.2","effort":"none","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-1.906458,"shrinkageFactor":0.999139,"score":21.4,"se":4.31,"samplingSe":0.5069,"residualSe":4.2835,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-12-01","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:DeepSeek-V3.2","observedHarness":"Direct API","rawScore":1325.73,"zScore":-2.080067,"seModel":"upstream_ci","ciHalfWidth":8.0322,"ciLower":1317.6971,"ciUpper":1333.7614,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-3.1-flash-lite-preview:high:direct:native","provider":"google","modelSlug":"gemini-3.1-flash-lite-preview","displayName":"Gemini 3.1 Flash-Lite Preview","effort":"high","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-2.240645,"shrinkageFactor":0.997693,"score":16.39,"se":4.86,"samplingSe":0.8297,"residualSe":4.7893,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-03-03","paretoProb":0,"benchmarks":[{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:vibe-code:google/gemini-3.1-flash-lite-preview","observedHarness":"OpenHands","rawScore":0,"zScore":-2.68247,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.51,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code"}]},{"id":"meta:llama-4-maverick:none:direct:native","provider":"meta","modelSlug":"llama-4-maverick","displayName":"Llama 4 Maverick","effort":"none","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-2.298192,"shrinkageFactor":0.726185,"score":15.53,"se":9.73,"samplingSe":9.0386,"residualSe":3.6062,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-04-05","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:llama-4-maverick","observedHarness":"Direct API","rawScore":858,"zScore":-3,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"meta:llama-4-scout:none:direct:native","provider":"meta","modelSlug":"llama-4-scout","displayName":"Llama 4 Scout","effort":"none","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-2.298192,"shrinkageFactor":0.726185,"score":15.53,"se":9.73,"samplingSe":9.0386,"residualSe":3.6062,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-04-05","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:llama-4-scout","observedHarness":"Direct API","rawScore":737,"zScore":-3,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"openai:gpt-oss-20b:medium:direct:native","provider":"openai","modelSlug":"gpt-oss-20b","displayName":"gpt-oss-20b","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-2.298192,"shrinkageFactor":0.726185,"score":15.53,"se":9.73,"samplingSe":9.0386,"residualSe":3.6062,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-08-05","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-oss-20b","observedHarness":"Direct API","rawScore":840,"zScore":-3,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"deepseek:deepseek-v3.2-exp:none:direct:native","provider":"deepseek","modelSlug":"deepseek-v3.2-exp","displayName":"DeepSeek V3.2-Exp","effort":"none","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-2.350655,"shrinkageFactor":0.997532,"score":14.74,"se":4.36,"samplingSe":0.8581,"residualSe":4.2766,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-09-29","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:DeepSeek-V3.2-Exp","observedHarness":"Direct API","rawScore":1271.39,"zScore":-2.528381,"seModel":"upstream_ci","ciHalfWidth":13.6073,"ciLower":1257.7823,"ciUpper":1284.9969,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-3.1-flash-lite-preview:minimal:direct:native","provider":"google","modelSlug":"gemini-3.1-flash-lite-preview","displayName":"Gemini 3.1 Flash-Lite Preview","effort":"minimal","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-2.444159,"shrinkageFactor":0.957461,"score":13.34,"se":3.92,"samplingSe":3.5626,"residualSe":1.6384,"coverageCount":5,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-03-03","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_3d","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:3d:gemini-3.1-flash-lite-preview","observedHarness":"Direct API","rawScore":1014,"zScore":-1.973629,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gemini-3.1-flash-lite-preview","observedHarness":"Direct API","rawScore":1002,"zScore":-2.58746,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:svg:gemini-3.1-flash-lite-preview","observedHarness":"Direct API","rawScore":1037,"zScore":-2.102334,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gemini-3.1-flash-lite-preview","observedHarness":"Direct API","rawScore":1070,"zScore":-3,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Gemini 3.1 Flash-Lite Preview","observedHarness":"Direct API","rawScore":1255.8,"zScore":-2.657001,"seModel":"upstream_ci","ciHalfWidth":6.8936,"ciLower":1248.902,"ciUpper":1262.6893,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-oss-120b:medium:direct:native","provider":"openai","modelSlug":"gpt-oss-120b","displayName":"gpt-oss-120b","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-2.667741,"shrinkageFactor":0.822681,"score":9.98,"se":7.83,"samplingSe":7.2736,"residualSe":2.8906,"coverageCount":2,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-08-05","paretoProb":0,"benchmarks":[{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:gamedev:gpt-oss-120b","observedHarness":"Direct API","rawScore":964,"zScore":-2.99262,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"},{"benchmarkId":"designarena_website","benchmarkVersion":"public","sourceId":"designarena","sourceRowId":"designarena:website:gpt-oss-120b","observedHarness":"Direct API","rawScore":955,"zScore":-3,"seModel":"unreported","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://designarena.ai/leaderboard"}]},{"id":"spacexai:grok-4.1:thinking:direct:native","provider":"spacexai","modelSlug":"grok-4.1","displayName":"Grok 4.1","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"design_code","theta":-2.804087,"shrinkageFactor":0.996692,"score":7.94,"se":4.39,"samplingSe":0.9935,"residualSe":4.273,"coverageCount":1,"activeBenchmarkCount":8,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-11-20","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","sourceId":"lmarena","sourceRowId":"lmarena:webdev:overall:Grok 4.1 (Thinking)","observedHarness":"Direct API","rawScore":1216.01,"zScore":-2.985274,"seModel":"upstream_ci","ciHalfWidth":24.9721,"ciLower":1191.0368,"ciUpper":1240.9809,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-20","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]}],"frontiers":[{"provider":"anthropic","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"anthropic:claude-haiku-5.5:medium:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9999,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"anthropic:claude-haiku-5.5:high:direct:native","zStepFromPrev":0.481,"pBetterThanPrev":0.6847,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.5295,"marginalScorePerDollar":253.082,"costMultipleFromPrev":1.79},{"configId":"anthropic:claude-haiku-5.5:max:direct:native","zStepFromPrev":2.209,"pBetterThanPrev":0.9864,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9808,"marginalScorePerDollar":1582.169,"costMultipleFromPrev":1.24},{"configId":"anthropic:claude-sonnet-5.5:high:direct:native","zStepFromPrev":0.041,"pBetterThanPrev":0.5163,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.4259,"marginalScorePerDollar":0.5,"costMultipleFromPrev":10.79},{"configId":"anthropic:claude-opus-5.5:medium:direct:native","zStepFromPrev":0.302,"pBetterThanPrev":0.6185,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.2788,"marginalScorePerDollar":17.895,"costMultipleFromPrev":1.27},{"configId":"anthropic:claude-sonnet-5.5:xhigh:direct:native","zStepFromPrev":1.077,"pBetterThanPrev":0.8593,"statisticallyTied":false,"imputationSensitive":true,"paretoProb":0.6469,"marginalScorePerDollar":98.027,"costMultipleFromPrev":1.15},{"configId":"anthropic:claude-opus-5.5:max:direct:native","zStepFromPrev":1.447,"pBetterThanPrev":0.9261,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9034,"marginalScorePerDollar":12.362,"costMultipleFromPrev":1.94}],"pruned":[{"configId":"anthropic:claude-haiku-4.5:none:direct:native","dominatedById":"anthropic:claude-haiku-5.5:max:direct:native","scoreDeficit":47.74,"costRatio":1.45,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-4.6:none:direct:native","dominatedById":"anthropic:claude-haiku-5.5:max:direct:native","scoreDeficit":22.06,"costRatio":4.54,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-4.5:none:direct:native","dominatedById":"anthropic:claude-haiku-5.5:max:direct:native","scoreDeficit":37.32,"costRatio":4.54,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5:high:direct:native","dominatedById":"anthropic:claude-haiku-5.5:max:direct:native","scoreDeficit":20.39,"costRatio":6.03,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.8:none:direct:native","dominatedById":"anthropic:claude-haiku-5.5:max:direct:native","scoreDeficit":22.41,"costRatio":7.01,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.7:none:direct:native","dominatedById":"anthropic:claude-haiku-5.5:max:direct:native","scoreDeficit":16.81,"costRatio":7.74,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.6:none:direct:native","dominatedById":"anthropic:claude-haiku-5.5:max:direct:native","scoreDeficit":19.52,"costRatio":7.74,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.5:none:direct:native","dominatedById":"anthropic:claude-haiku-5.5:max:direct:native","scoreDeficit":27,"costRatio":7.74,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-4.5:thinking:direct:native","dominatedById":"anthropic:claude-haiku-5.5:max:direct:native","scoreDeficit":36.69,"costRatio":8.54,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5:max:direct:native","dominatedById":"anthropic:claude-haiku-5.5:max:direct:native","scoreDeficit":2,"costRatio":8.81,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-4.6:max:direct:native","dominatedById":"anthropic:claude-sonnet-5.5:high:direct:native","scoreDeficit":19.75,"costRatio":1.25,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.6:thinking:direct:native","dominatedById":"anthropic:claude-opus-5.5:medium:direct:native","scoreDeficit":21.21,"costRatio":1.06,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5.5:max:direct:native","dominatedById":"anthropic:claude-sonnet-5.5:xhigh:direct:native","scoreDeficit":0.11,"costRatio":1,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5:high:direct:native","dominatedById":"anthropic:claude-sonnet-5.5:xhigh:direct:native","scoreDeficit":15.83,"costRatio":1.07,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.8:high:direct:native","dominatedById":"anthropic:claude-sonnet-5.5:xhigh:direct:native","scoreDeficit":26.76,"costRatio":1.12,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.7:high:direct:native","dominatedById":"anthropic:claude-sonnet-5.5:xhigh:direct:native","scoreDeficit":26.52,"costRatio":1.14,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.6:high:direct:native","dominatedById":"anthropic:claude-sonnet-5.5:xhigh:direct:native","scoreDeficit":27.89,"costRatio":1.14,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.5:high:direct:native","dominatedById":"anthropic:claude-sonnet-5.5:xhigh:direct:native","scoreDeficit":47.71,"costRatio":1.14,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5:high:direct:native","dominatedById":"anthropic:claude-sonnet-5.5:xhigh:direct:native","scoreDeficit":19.88,"costRatio":1.16,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.8:max:direct:native","dominatedById":"anthropic:claude-sonnet-5.5:xhigh:direct:native","scoreDeficit":11.12,"costRatio":1.42,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.7:max:direct:native","dominatedById":"anthropic:claude-sonnet-5.5:xhigh:direct:native","scoreDeficit":18.15,"costRatio":1.45,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.6:max:direct:native","dominatedById":"anthropic:claude-sonnet-5.5:xhigh:direct:native","scoreDeficit":28.33,"costRatio":1.45,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5:max:direct:native","dominatedById":"anthropic:claude-sonnet-5.5:xhigh:direct:native","scoreDeficit":9.93,"costRatio":1.57,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5:max:direct:native","dominatedById":"anthropic:claude-sonnet-5.5:xhigh:direct:native","scoreDeficit":6.5,"costRatio":1.69,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.1:high:direct:native","dominatedById":"anthropic:claude-sonnet-5.5:xhigh:direct:native","scoreDeficit":15.95,"costRatio":1.9,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.1:max:direct:native","dominatedById":"anthropic:claude-opus-5.5:max:direct:native","scoreDeficit":11.35,"costRatio":1.43,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.2:high:direct:native","dominatedById":"anthropic:claude-opus-5.5:max:direct:native","scoreDeficit":3.66,"costRatio":2.59,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.2:xhigh:direct:native","dominatedById":"anthropic:claude-opus-5.5:max:direct:native","scoreDeficit":0.81,"costRatio":3.43,"costComparisonNote":null}],"verdicts":{"budgetId":"anthropic:claude-haiku-5.5:medium:direct:native","defaultId":"anthropic:claude-haiku-5.5:max:direct:native","maxId":"anthropic:claude-opus-5.5:max:direct:native"}},{"provider":"openai","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"openai:gpt-6-luna:medium:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9999,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"openai:gpt-6-luna:max:direct:native","zStepFromPrev":1.954,"pBetterThanPrev":0.9747,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.741,"marginalScorePerDollar":1955.519,"costMultipleFromPrev":2.23},{"configId":"openai:gpt-5.6-luna:max:direct:native","zStepFromPrev":0.785,"pBetterThanPrev":0.7838,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.8404,"marginalScorePerDollar":1566.491,"costMultipleFromPrev":1.24},{"configId":"openai:gpt-6.1-terra:max:direct:native","zStepFromPrev":0.665,"pBetterThanPrev":0.7469,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.5669,"marginalScorePerDollar":43.758,"costMultipleFromPrev":7.55},{"configId":"openai:gpt-6.2-sol:medium:direct:native","zStepFromPrev":0.294,"pBetterThanPrev":0.6156,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.4622,"marginalScorePerDollar":34.765,"costMultipleFromPrev":1.83},{"configId":"openai:gpt-6.1-sol:max:direct:native","zStepFromPrev":0.204,"pBetterThanPrev":0.5809,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.4264,"marginalScorePerDollar":34.389,"costMultipleFromPrev":1.31},{"configId":"openai:gpt-6.2-sol:max:direct:native","zStepFromPrev":0.415,"pBetterThanPrev":0.661,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.4091,"marginalScorePerDollar":8.431,"costMultipleFromPrev":1.7},{"configId":"openai:gpt-6.2-astra:medium:direct:native","zStepFromPrev":0.302,"pBetterThanPrev":0.6188,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.4835,"marginalScorePerDollar":7.302,"costMultipleFromPrev":1.84}],"pruned":[{"configId":"openai:gpt-5.6-luna:medium:direct:native","dominatedById":"openai:gpt-6-luna:medium:direct:native","scoreDeficit":9.01,"costRatio":1.24,"costComparisonNote":null},{"configId":"openai:gpt-5.4-nano:high:direct:native","dominatedById":"openai:gpt-5.6-luna:max:direct:native","scoreDeficit":29.71,"costRatio":2.27,"costComparisonNote":null},{"configId":"openai:gpt-6.1-terra:medium:direct:native","dominatedById":"openai:gpt-5.6-luna:max:direct:native","scoreDeficit":5.54,"costRatio":3.39,"costComparisonNote":null},{"configId":"openai:gpt-5.6-terra:medium:direct:native","dominatedById":"openai:gpt-5.6-luna:max:direct:native","scoreDeficit":21.73,"costRatio":5.29,"costComparisonNote":null},{"configId":"openai:gpt-6-sol:medium:direct:native","dominatedById":"openai:gpt-6.1-terra:max:direct:native","scoreDeficit":10.74,"costRatio":1,"costComparisonNote":null},{"configId":"openai:gpt-6.1-sol:medium:direct:native","dominatedById":"openai:gpt-6.1-terra:max:direct:native","scoreDeficit":0.02,"costRatio":1.08,"costComparisonNote":null},{"configId":"openai:gpt-5.1:low:direct:native","dominatedById":"openai:gpt-6.1-terra:max:direct:native","scoreDeficit":39.81,"costRatio":1.09,"costComparisonNote":null},{"configId":"openai:gpt-5.1:none:direct:native","dominatedById":"openai:gpt-6.1-terra:max:direct:native","scoreDeficit":42.27,"costRatio":1.09,"costComparisonNote":null},{"configId":"openai:gpt-5.4-mini:high:direct:native","dominatedById":"openai:gpt-6.1-terra:max:direct:native","scoreDeficit":35.67,"costRatio":1.13,"costComparisonNote":null},{"configId":"openai:gpt-5.4-mini:xhigh:direct:native","dominatedById":"openai:gpt-6.1-terra:max:direct:native","scoreDeficit":20.95,"costRatio":1.44,"costComparisonNote":null},{"configId":"openai:gpt-5:medium:direct:native","dominatedById":"openai:gpt-6.1-terra:max:direct:native","scoreDeficit":37.18,"costRatio":1.52,"costComparisonNote":null},{"configId":"openai:gpt-5.1:medium:direct:native","dominatedById":"openai:gpt-6.1-terra:max:direct:native","scoreDeficit":38.26,"costRatio":1.52,"costComparisonNote":null},{"configId":"openai:gpt-5.2:none:direct:native","dominatedById":"openai:gpt-6.1-terra:max:direct:native","scoreDeficit":33,"costRatio":1.55,"costComparisonNote":null},{"configId":"openai:gpt-5.6-terra:max:direct:native","dominatedById":"openai:gpt-6.1-terra:max:direct:native","scoreDeficit":5.48,"costRatio":1.56,"costComparisonNote":null},{"configId":"openai:gpt-5.4:low:direct:native","dominatedById":"openai:gpt-6.1-terra:max:direct:native","scoreDeficit":35.26,"costRatio":1.69,"costComparisonNote":null},{"configId":"openai:gpt-5.4:none:direct:native","dominatedById":"openai:gpt-6.1-terra:max:direct:native","scoreDeficit":36.46,"costRatio":1.69,"costComparisonNote":null},{"configId":"openai:gpt-5.3-codex:medium:direct:native","dominatedById":"openai:gpt-6.2-sol:medium:direct:native","scoreDeficit":43.05,"costRatio":1.18,"costComparisonNote":null},{"configId":"openai:gpt-6-sol:max:direct:native","dominatedById":"openai:gpt-6.2-sol:medium:direct:native","scoreDeficit":3.18,"costRatio":1.22,"costComparisonNote":null},{"configId":"openai:gpt-5:high:direct:native","dominatedById":"openai:gpt-6.1-sol:max:direct:native","scoreDeficit":40.69,"costRatio":1.06,"costComparisonNote":null},{"configId":"openai:gpt-5.1:high:direct:native","dominatedById":"openai:gpt-6.1-sol:max:direct:native","scoreDeficit":41.11,"costRatio":1.06,"costComparisonNote":null},{"configId":"openai:gpt-5.6-sol:medium:direct:native","dominatedById":"openai:gpt-6.1-sol:max:direct:native","scoreDeficit":15.4,"costRatio":1.31,"costComparisonNote":null},{"configId":"openai:gpt-5.5:medium:direct:native","dominatedById":"openai:gpt-6.1-sol:max:direct:native","scoreDeficit":25.37,"costRatio":1.38,"costComparisonNote":null},{"configId":"openai:gpt-5.3-codex:xhigh:direct:native","dominatedById":"openai:gpt-6.2-sol:max:direct:native","scoreDeficit":19.43,"costRatio":1.13,"costComparisonNote":null},{"configId":"openai:gpt-5.2:xhigh:direct:native","dominatedById":"openai:gpt-6.2-sol:max:direct:native","scoreDeficit":24.23,"costRatio":1.13,"costComparisonNote":null},{"configId":"openai:gpt-5.4:xhigh:direct:native","dominatedById":"openai:gpt-6.2-sol:max:direct:native","scoreDeficit":16.16,"costRatio":1.23,"costComparisonNote":null},{"configId":"openai:gpt-5.6-sol:xhigh:direct:native","dominatedById":"openai:gpt-6.2-sol:max:direct:native","scoreDeficit":11.37,"costRatio":1.68,"costComparisonNote":null},{"configId":"openai:gpt-5.5:xhigh:direct:native","dominatedById":"openai:gpt-6.2-sol:max:direct:native","scoreDeficit":14.72,"costRatio":1.81,"costComparisonNote":null},{"configId":"openai:gpt-5.6-sol:max:direct:native","dominatedById":"openai:gpt-6.2-astra:medium:direct:native","scoreDeficit":10.97,"costRatio":1.14,"costComparisonNote":null},{"configId":"openai:gpt-6-astra:medium:direct:native","dominatedById":"openai:gpt-6.2-astra:medium:direct:native","scoreDeficit":0.45,"costRatio":1.68,"costComparisonNote":null},{"configId":"openai:gpt-6.2-astra:max:direct:native","dominatedById":"openai:gpt-6.2-astra:medium:direct:native","scoreDeficit":1.11,"costRatio":2.22,"costComparisonNote":null},{"configId":"openai:gpt-6-astra:max:direct:native","dominatedById":"openai:gpt-6.2-astra:medium:direct:native","scoreDeficit":0.2,"costRatio":3.2,"costComparisonNote":null}],"verdicts":{"budgetId":"openai:gpt-6-luna:medium:direct:native","defaultId":"openai:gpt-5.6-luna:max:direct:native","maxId":"openai:gpt-6.2-astra:medium:direct:native"}},{"provider":"google","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"google:gemini-3-flash:minimal:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.7646,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"google:gemini-3.5-flash-lite:high:direct:native","zStepFromPrev":1.45,"pBetterThanPrev":0.9265,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.5807,"marginalScorePerDollar":339.502,"costMultipleFromPrev":1.9},{"configId":"google:gemini-3.8-flash:medium:direct:native","zStepFromPrev":1.902,"pBetterThanPrev":0.9714,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9753,"marginalScorePerDollar":4800,"costMultipleFromPrev":1.05},{"configId":"google:gemini-3.7-flash:high:direct:native","zStepFromPrev":0.178,"pBetterThanPrev":0.5707,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.5783,"marginalScorePerDollar":4178.571,"costMultipleFromPrev":1},{"configId":"google:gemini-3.8-flash:high:direct:native","zStepFromPrev":0.468,"pBetterThanPrev":0.68,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.3328,"marginalScorePerDollar":34.665,"costMultipleFromPrev":1.79},{"configId":"google:gemini-4-hydrogen:high:direct:native","zStepFromPrev":1.828,"pBetterThanPrev":0.9662,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.756,"marginalScorePerDollar":435.104,"costMultipleFromPrev":1.16},{"configId":"google:gemini-4-hydrogen:xhigh:direct:native","zStepFromPrev":0.559,"pBetterThanPrev":0.7119,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.6439,"marginalScorePerDollar":82.851,"costMultipleFromPrev":1.33},{"configId":"google:gemini-4-carbon:medium:direct:native","zStepFromPrev":0.193,"pBetterThanPrev":0.5765,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.5127,"marginalScorePerDollar":39.095,"costMultipleFromPrev":1.2},{"configId":"google:gemini-4-argon:high:direct:native","zStepFromPrev":0.003,"pBetterThanPrev":0.5014,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.2508,"marginalScorePerDollar":0.23,"costMultipleFromPrev":1.4},{"configId":"google:gemini-4-neon:high:direct:native","zStepFromPrev":0.784,"pBetterThanPrev":0.7835,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.4475,"marginalScorePerDollar":44.486,"costMultipleFromPrev":1.28},{"configId":"google:gemini-4-carbon:high:direct:native","zStepFromPrev":0.042,"pBetterThanPrev":0.5169,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.4831,"marginalScorePerDollar":93.004,"costMultipleFromPrev":1.01}],"pruned":[{"configId":"google:gemini-3-flash:high:direct:native","dominatedById":"google:gemini-3.7-flash:high:direct:native","scoreDeficit":21.89,"costRatio":1.16,"costComparisonNote":null},{"configId":"google:gemini-3.5-flash:medium:direct:native","dominatedById":"google:gemini-3.7-flash:high:direct:native","scoreDeficit":13.07,"costRatio":1.4,"costComparisonNote":null},{"configId":"google:gemini-3.6-flash:medium:direct:native","dominatedById":"google:gemini-3.7-flash:high:direct:native","scoreDeficit":6.83,"costRatio":1.52,"costComparisonNote":null},{"configId":"google:gemini-3.5-flash:high:direct:native","dominatedById":"google:gemini-4-hydrogen:high:direct:native","scoreDeficit":19.22,"costRatio":1.13,"costComparisonNote":null},{"configId":"google:gemini-4-argon:medium:direct:native","dominatedById":"google:gemini-4-hydrogen:high:direct:native","scoreDeficit":2.64,"costRatio":1.24,"costComparisonNote":null},{"configId":"google:gemini-3.1-pro:high:direct:native","dominatedById":"google:gemini-4-hydrogen:high:direct:native","scoreDeficit":24.26,"costRatio":1.25,"costComparisonNote":null},{"configId":"google:gemini-3.6-flash:high:direct:native","dominatedById":"google:gemini-4-hydrogen:high:direct:native","scoreDeficit":15.01,"costRatio":1.28,"costComparisonNote":null}],"verdicts":{"budgetId":"google:gemini-3-flash:minimal:direct:native","defaultId":"google:gemini-4-hydrogen:high:direct:native","maxId":"google:gemini-4-carbon:high:direct:native"}},{"provider":"spacexai","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"spacexai:grok-4.3:low:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"spacexai:grok-4.3:high:direct:native","zStepFromPrev":0.199,"pBetterThanPrev":0.5789,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.5708,"marginalScorePerDollar":81.797,"costMultipleFromPrev":2.52},{"configId":"spacexai:grok-4.20:none:direct:native","zStepFromPrev":0.095,"pBetterThanPrev":0.538,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.4017,"marginalScorePerDollar":25.348,"costMultipleFromPrev":2.08},{"configId":"spacexai:grok-4.20:thinking:direct:native","zStepFromPrev":0.345,"pBetterThanPrev":0.6351,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.3784,"marginalScorePerDollar":746.568,"costMultipleFromPrev":1.05},{"configId":"spacexai:grok-4.6:high:direct:native","zStepFromPrev":5.982,"pBetterThanPrev":1,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9563,"marginalScorePerDollar":259.909,"costMultipleFromPrev":2.79},{"configId":"spacexai:grok-4.7:xhigh:direct:native","zStepFromPrev":1.227,"pBetterThanPrev":0.8901,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.8716,"marginalScorePerDollar":12.567,"costMultipleFromPrev":3.05}],"pruned":[{"configId":"spacexai:grok-4.5:high:direct:native","dominatedById":"spacexai:grok-4.6:high:direct:native","scoreDeficit":6.88,"costRatio":1,"costComparisonNote":null},{"configId":"spacexai:grok-4.7:high:direct:native","dominatedById":"spacexai:grok-4.6:high:direct:native","scoreDeficit":8.08,"costRatio":2.09,"costComparisonNote":null}],"verdicts":{"budgetId":"spacexai:grok-4.3:low:direct:native","defaultId":"spacexai:grok-4.6:high:direct:native","maxId":"spacexai:grok-4.7:xhigh:direct:native"}},{"provider":"meta","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"meta:muse-ember:max:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9998,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"meta:muse-spark-1.3:high:direct:native","zStepFromPrev":0.037,"pBetterThanPrev":0.5147,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.475,"marginalScorePerDollar":2.636,"costMultipleFromPrev":6.77},{"configId":"meta:muse-spark-1.3:xhigh:direct:native","zStepFromPrev":0.1,"pBetterThanPrev":0.5397,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.3495,"marginalScorePerDollar":26.601,"costMultipleFromPrev":1.23},{"configId":"meta:muse-spark-1.3:max:direct:native","zStepFromPrev":0.497,"pBetterThanPrev":0.6904,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.3812,"marginalScorePerDollar":80.402,"costMultipleFromPrev":1.18},{"configId":"meta:muse-wonder:max:direct:native","zStepFromPrev":0.573,"pBetterThanPrev":0.7166,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.4615,"marginalScorePerDollar":5.596,"costMultipleFromPrev":2.89}],"pruned":[{"configId":"meta:muse-spark-1.2:high:direct:native","dominatedById":"meta:muse-ember:max:direct:native","scoreDeficit":6.18,"costRatio":5.6,"costComparisonNote":null},{"configId":"meta:muse-spark-1.1:high:direct:native","dominatedById":"meta:muse-spark-1.3:high:direct:native","scoreDeficit":11.68,"costRatio":1.07,"costComparisonNote":null},{"configId":"meta:muse-spark-1.2:xhigh:direct:native","dominatedById":"meta:muse-spark-1.3:high:direct:native","scoreDeficit":7.4,"costRatio":1.21,"costComparisonNote":null}],"verdicts":{"budgetId":"meta:muse-ember:max:direct:native","defaultId":null,"maxId":"meta:muse-wonder:max:direct:native"}},{"provider":"deepseek","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"deepseek:deepseek-v4.1-flash-1017:high:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9767,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"deepseek:deepseek-v4.1-flash-1017:max:direct:native","zStepFromPrev":1.959,"pBetterThanPrev":0.9749,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.941,"marginalScorePerDollar":842.919,"costMultipleFromPrev":1.46},{"configId":"deepseek:deepseek-v4-pro-0813:max:direct:native","zStepFromPrev":0.127,"pBetterThanPrev":0.5506,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.5709,"marginalScorePerDollar":8.455,"costMultipleFromPrev":2.82}],"pruned":[{"configId":"deepseek:deepseek-v4-pro-0813:high:direct:native","dominatedById":"deepseek:deepseek-v4.1-flash-1017:max:direct:native","scoreDeficit":11.52,"costRatio":2.21,"costComparisonNote":null}],"verdicts":{"budgetId":"deepseek:deepseek-v4.1-flash-1017:high:direct:native","defaultId":"deepseek:deepseek-v4.1-flash-1017:max:direct:native","maxId":"deepseek:deepseek-v4-pro-0813:max:direct:native"}},{"provider":"moonshot","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"moonshot:kimi-k2.7-code:max:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"moonshot:kimi-k2.6:thinking:direct:native","zStepFromPrev":0.445,"pBetterThanPrev":0.6717,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.6718,"marginalScorePerDollar":374.715,"costMultipleFromPrev":1.1},{"configId":"moonshot:kimi-k3.1-pro:max:direct:native","zStepFromPrev":4.85,"pBetterThanPrev":1,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":70.737,"costMultipleFromPrev":6.24}],"pruned":[{"configId":"moonshot:kimi-k3:max:direct:native","dominatedById":"moonshot:kimi-k3.1-pro:max:direct:native","scoreDeficit":3.9,"costRatio":1.11,"costComparisonNote":null}],"verdicts":{"budgetId":"moonshot:kimi-k2.7-code:max:direct:native","defaultId":null,"maxId":"moonshot:kimi-k3.1-pro:max:direct:native"}},{"provider":"alibaba","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"alibaba:qwen3.7-max:thinking:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.6005,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"alibaba:qwen4-max:xhigh:direct:native","zStepFromPrev":4.632,"pBetterThanPrev":1,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9985,"marginalScorePerDollar":506.179,"costMultipleFromPrev":1.5}],"pruned":[{"configId":"alibaba:qwen3.8-max-0902:xhigh:direct:native","dominatedById":"alibaba:qwen4-max:xhigh:direct:native","scoreDeficit":10.15,"costRatio":1.35,"costComparisonNote":null}],"verdicts":{"budgetId":"alibaba:qwen3.7-max:thinking:direct:native","defaultId":null,"maxId":"alibaba:qwen4-max:xhigh:direct:native"}},{"provider":"zhipu","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"zhipu:glm-5.3-flash:max:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9991,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"zhipu:glm-5.3:max:direct:native","zStepFromPrev":2.546,"pBetterThanPrev":0.9945,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9411,"marginalScorePerDollar":49.057,"costMultipleFromPrev":9.37}],"pruned":[{"configId":"zhipu:glm-5:thinking:direct:native","dominatedById":"zhipu:glm-5.3-flash:max:direct:native","scoreDeficit":15.97,"costRatio":4.24,"costComparisonNote":null},{"configId":"zhipu:glm-5.1:thinking:direct:native","dominatedById":"zhipu:glm-5.3-flash:max:direct:native","scoreDeficit":12.08,"costRatio":5.91,"costComparisonNote":null},{"configId":"zhipu:glm-5.2:thinking:direct:native","dominatedById":"zhipu:glm-5.3-flash:max:direct:native","scoreDeficit":6.8,"costRatio":6.56,"costComparisonNote":null},{"configId":"zhipu:glm-5.2:max:direct:native","dominatedById":"zhipu:glm-5.3:max:direct:native","scoreDeficit":4.23,"costRatio":1.18,"costComparisonNote":null}],"verdicts":{"budgetId":"zhipu:glm-5.3-flash:max:direct:native","defaultId":null,"maxId":"zhipu:glm-5.3:max:direct:native"}}]},"research":{"domainId":"research","epochVersion":"research-v2.0.0","costUnit":"usd_per_task","activeBenchmarkIds":["mlcr_aa","aa_omniscience","gdp_pdf","hle_diamond","critpt_v2_hard","terminalbench_science_aa","aa_analyst_agent","aa_mka","aa_vials","aa_briefcase","gdpval_aa","epoch_simpleqa_verified","epoch_apex_agents","lmarena_factuality","vals_legal_research","vals_finance_agent","vals_emb","vals_tax_agent","vals_mystery_mechanism","vals_medcode","surge_gdp_xlsx","surge_dayjob_finance","surge_dayjob_law","surge_dayjob_healthcare","surge_riemann","scale_prbench_finance","scale_prbench_legal"],"candidateBenchmarkIds":[],"benchmarkVersions":{"mlcr_aa":"1.0","aa_omniscience":"1.1","gdp_pdf":"1.0","hle_diamond":"1.0","critpt_v2_hard":"2.0","terminalbench_science_aa":"0.1","aa_analyst_agent":"1.0","aa_mka":"1.0","aa_vials":"1.0","aa_briefcase":"1.1","gdpval_aa":"2.1","epoch_simpleqa_verified":"2026.11","epoch_apex_agents":"2026.11","lmarena_factuality":"text_factuality","vals_legal_research":"1","vals_finance_agent":"2.1","vals_emb":"1.1","vals_tax_agent":"1","vals_mystery_mechanism":"1","vals_medcode":"1","surge_gdp_xlsx":"1.0","surge_dayjob_finance":"1.0","surge_dayjob_law":"1.0","surge_dayjob_healthcare":"1.0","surge_riemann":"1.0","scale_prbench_finance":"1.0","scale_prbench_legal":"1.0"},"coverageFloor":1,"tauResid":0.6356,"regimeMeta":{"native":{"activeBenchmarkIds":["mlcr_aa","aa_omniscience","gdp_pdf","hle_diamond","critpt_v2_hard","terminalbench_science_aa","aa_analyst_agent","aa_mka","aa_vials","aa_briefcase","gdpval_aa","epoch_simpleqa_verified","epoch_apex_agents","lmarena_factuality","vals_legal_research","vals_finance_agent","vals_emb","vals_tax_agent","vals_mystery_mechanism","vals_medcode","surge_gdp_xlsx","surge_dayjob_finance","surge_dayjob_law","surge_dayjob_healthcare","surge_riemann","scale_prbench_finance","scale_prbench_legal"],"costSource":"suite","measuredCostBenchmarkIds":[],"coverageFloor":1,"tauResid":0.6356,"calibrationTauResidSq":0.40398805,"priorMean":-0.207431,"tauModelSq":0.897739,"anchorLabs":9,"withheldBenchmarkIds":[],"linkJointCov":{"benchmarkIds":["aa_analyst_agent","aa_briefcase","aa_mka","aa_vials","gdpval_aa","scale_prbench_finance","surge_dayjob_law"],"covMatrix":[[0.12473246398277571,-0.0023085253837375696,-0.004683507034969239,-0.0000019366319607495946,-0.013971534101392735,0.00026714866889074134,-0.07691818309946906,0.0013341931410764361,-0.06755401825360138,0.00003616512288848048,-0.004347927821556098,-0.00003385634429188864,0.011968980197663808,-0.0006900251218285052],[-0.0023085253837375696,0.00005363913433773498,0.0006391165234194082,-3.8972470258568446e-7,0.0002722843731096518,-0.00000517941051689445,0.0017543058616659164,-0.000029920160514597963,0.0013155532812910778,-7.282324702945506e-7,-0.00036492079414031126,0.000009589635708611119,-0.0002544979564118707,0.00001076413269146671],[-0.004683507034969239,0.0006391165234194082,1.8328266563872857,-0.001150403726648012,0.04226595669694538,-0.0012924687913892976,0.16699476507791505,-0.0034405924133161614,0.6485359056122606,-0.00040001320966801223,-0.009962539572361486,-3.410765908333974e-7,0.01018719727819796,-0.003195679142874292],[-0.0000019366319607495946,-3.8972470258568446e-7,-0.001150403726648012,7.278881232622068e-7,-0.0000349639919640389,0.000001008065168217377,-0.00012191345679244882,0.000002447277360357208,-0.0004285516385149247,2.6766016994141765e-7,0.000003161380142724552,1.183226216964899e-8,-0.000005661067354360755,0.00000212571179954581],[-0.013971534101392735,0.0002722843731096518,0.04226595669694538,-0.0000349639919640389,0.08074949489669192,-0.0018265245936352021,0.06863673053438875,-0.0013457963986443517,0.04769747572588744,-0.0000401664411612485,-0.0072379882592986736,0.00011345493680729404,0.015537594286670196,-0.001256097740437878],[0.00026714866889074134,-0.00000517941051689445,-0.0012924687913892976,0.000001008065168217377,-0.0018265245936352021,0.00004260820874244952,-0.001487512733431903,0.000030258378960592552,-0.0010238673523933757,8.358741523550062e-7,0.00003394229810262816,-0.0000019131793988826238,-0.0003126912733098673,0.00003001771797336586],[-0.07691818309946906,0.0017543058616659164,0.16699476507791505,-0.00012191345679244882,0.06863673053438875,-0.001487512733431903,0.24530447714423886,-0.004276387970722542,0.26918808868885447,-0.00017905303951103093,-0.0236921683157732,0.00039963499981641103,0.005331122575537903,-0.0008802566321783289],[0.0013341931410764361,-0.000029920160514597963,-0.0034405924133161614,0.000002447277360357208,-0.0013457963986443517,0.000030258378960592552,-0.004276387970722542,0.00007693349144075212,-0.005982225681092268,0.000003913048044495156,0.000399862285123191,-0.000008114736732398546,-0.00011202726912252334,0.000019867551525670536],[-0.06755401825360138,0.0013155532812910778,0.6485359056122606,-0.0004285516385149247,0.04769747572588744,-0.0010238673523933757,0.26918808868885447,-0.005982225681092268,6.143502424333781,-0.0038721738530352852,-0.0810380232489631,0.0008136248021998548,0.010723422187513557,-0.003422181317037612],[0.00003616512288848048,-7.282324702945506e-7,-0.00040001320966801223,2.6766016994141765e-7,-0.0000401664411612485,8.358741523550062e-7,-0.00017905303951103093,0.000003913048044495156,-0.0038721738530352852,0.0000024499169505096503,0.00004628257755673268,-4.6811371147666455e-7,-0.000009463712113716135,0.000002320819451896923],[-0.004347927821556098,-0.00036492079414031126,-0.009962539572361486,0.000003161380142724552,-0.0072379882592986736,0.00003394229810262816,-0.0236921683157732,0.000399862285123191,-0.0810380232489631,0.00004628257755673268,1.7304114395207546,-0.033219248447585524,-0.010838570055417928,0.0002591865621525238],[-0.00003385634429188864,0.000009589635708611119,-3.410765908333974e-7,1.183226216964899e-8,0.00011345493680729404,-0.0000019131793988826238,0.00039963499981641103,-0.000008114736732398546,0.0008136248021998548,-4.6811371147666455e-7,-0.033219248447585524,0.0006584854600180772,0.00014124072955366498,-0.000004507434070243686],[0.011968980197663808,-0.0002544979564118707,0.01018719727819796,-0.000005661067354360755,0.015537594286670196,-0.0003126912733098673,0.005331122575537903,-0.00011202726912252334,0.010723422187513557,-0.000009463712113716135,-0.010838570055417928,0.00014124072955366498,0.02303740442073309,-0.0007100946619638656],[-0.0006900251218285052,0.00001076413269146671,-0.003195679142874292,0.00000212571179954581,-0.001256097740437878,0.00003001771797336586,-0.0008802566321783289,0.000019867551525670536,-0.003422181317037612,0.000002320819451896923,0.0002591865621525238,-0.000004507434070243686,-0.0007100946619638656,0.00005033865964980318]]},"epochLink":{"previousEpoch":"research-v1.0.0","linked":true,"overlapCount":205,"overlapLabs":9,"slope":0.96704,"intercept":0.221817,"pearsonR":0.9257,"reason":"Verified Mean-Sigma link (n=205, labs=9, r=0.926)"}}},"anchorStats":{"mlcr_aa":{"benchmarkId":"mlcr_aa","mean":21.002422,"sd":8.943806,"anchorCount":72,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":-0.045615,"reliability":0.6794,"discrimination":0.3944,"redundancyFactor":0.5366,"effectiveWeight":0.1438,"taskFamily":"long_context_retrieval","datasetLineageId":"aa-mlcr-1.0","introducedAt":"2026-07-01","invertScore":false,"popoviciuSe":6.454972,"boundedMax":100,"sampleSize":60},"aa_omniscience":{"benchmarkId":"aa_omniscience","mean":41.054398,"sd":14.009239,"anchorCount":86,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":0.125405,"reliability":0.9963,"discrimination":0.9167,"redundancyFactor":0.2265,"effectiveWeight":0.2069,"taskFamily":"knowledge_synthesis","datasetLineageId":"aa-omniscience-1.1","introducedAt":"2026-07-01","invertScore":false,"popoviciuSe":0.912871,"boundedMax":100,"sampleSize":3000},"gdp_pdf":{"benchmarkId":"gdp_pdf","mean":19.089206,"sd":8.473006,"anchorCount":92,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":0.102501,"reliability":0.8265,"discrimination":0.1704,"redundancyFactor":0.476,"effectiveWeight":0.1,"taskFamily":"document_synthesis","datasetLineageId":"surge-gdp-pdf-1.0","introducedAt":"2026-08-01","invertScore":false,"popoviciuSe":5,"boundedMax":100,"sampleSize":100},"hle_diamond":{"benchmarkId":"hle_diamond","mean":28.545927,"sd":18.34366,"anchorCount":92,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":0.264958,"reliability":0.995,"discrimination":0.9375,"redundancyFactor":0.3906,"effectiveWeight":0.3644,"taskFamily":"frontier_science","datasetLineageId":"hle-diamond-1.0","introducedAt":"2026-07-15","invertScore":false,"popoviciuSe":1.581139,"boundedMax":100,"sampleSize":1000},"critpt_v2_hard":{"benchmarkId":"critpt_v2_hard","mean":38.060778,"sd":19.388952,"anchorCount":84,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":0.106121,"reliability":0.8666,"discrimination":0.6034,"redundancyFactor":0.386,"effectiveWeight":0.2018,"taskFamily":"physical_science","datasetLineageId":"critpt-v2-hard","introducedAt":"2026-08-15","invertScore":false,"popoviciuSe":7.905694,"boundedMax":100,"sampleSize":40},"terminalbench_science_aa":{"benchmarkId":"terminalbench_science_aa","mean":21.461106,"sd":21.795726,"anchorCount":85,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":0.205829,"reliability":0.9626,"discrimination":0.7708,"redundancyFactor":0.4005,"effectiveWeight":0.2972,"taskFamily":"scientific_workflows","datasetLineageId":"terminal-bench-science-0.1","introducedAt":"2026-09-01","invertScore":false,"popoviciuSe":5.976143,"boundedMax":100,"sampleSize":70},"epoch_simpleqa_verified":{"benchmarkId":"epoch_simpleqa_verified","mean":51.909366,"sd":14.056684,"anchorCount":52,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":0.08874,"reliability":0.9887,"discrimination":0.875,"redundancyFactor":0.2902,"effectiveWeight":0.2511,"taskFamily":"knowledge_synthesis","datasetLineageId":"epoch-simpleqa-verified","introducedAt":"2025-09-09","invertScore":false,"popoviciuSe":1.581139,"boundedMax":100,"sampleSize":1000},"epoch_apex_agents":{"benchmarkId":"epoch_apex_agents","mean":51.239087,"sd":16.630941,"anchorCount":28,"anchorLabs":8,"linkedViaMeanSigma":false,"raschDelta":0.011567,"reliability":0.9205,"discrimination":0.6764,"redundancyFactor":0.1687,"effectiveWeight":0.105,"taskFamily":"knowledge_synthesis","datasetLineageId":"mercor-apex-agents","introducedAt":"2026-01-21","invertScore":false,"popoviciuSe":8.703883,"boundedMax":100,"sampleSize":33},"lmarena_factuality":{"benchmarkId":"lmarena_factuality","mean":1461.935293,"sd":21.262964,"anchorCount":54,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":0.161395,"reliability":0.9886,"discrimination":0.8542,"redundancyFactor":0.2799,"effectiveWeight":0.2364,"taskFamily":"knowledge_synthesis","datasetLineageId":"lmarena-text-factuality","introducedAt":"2026-08-01","invertScore":false},"vals_legal_research":{"benchmarkId":"vals_legal_research","mean":38.78941,"sd":11.273299,"anchorCount":53,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":-0.054945,"reliability":0.9296,"discrimination":0.305,"redundancyFactor":0.5598,"effectiveWeight":0.1587,"taskFamily":"legal_analysis","datasetLineageId":"vals-legal-research-1","introducedAt":"2026-07-15","invertScore":false,"popoviciuSe":3.466876,"boundedMax":100,"sampleSize":208},"vals_finance_agent":{"benchmarkId":"vals_finance_agent","mean":62.898333,"sd":3.490811,"anchorCount":26,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":-0.42399,"reliability":0.6524,"discrimination":0.3987,"redundancyFactor":0.2795,"effectiveWeight":0.1,"taskFamily":"finance_analysis","datasetLineageId":"vals-finance-agent-2.1","introducedAt":"2026-08-15","invertScore":false,"popoviciuSe":2.359646,"boundedMax":100,"sampleSize":449},"vals_emb":{"benchmarkId":"vals_emb","mean":71.036593,"sd":3.644371,"anchorCount":30,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":-0.289071,"reliability":0.5445,"discrimination":0.3518,"redundancyFactor":0.2202,"effectiveWeight":0.1,"taskFamily":"finance_analysis","datasetLineageId":"vals-emb-1.1","introducedAt":"2026-08-15","invertScore":false,"popoviciuSe":4.950738,"boundedMax":100,"sampleSize":102},"vals_tax_agent":{"benchmarkId":"vals_tax_agent","mean":30.087507,"sd":11.344919,"anchorCount":55,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":-0.149226,"reliability":0.9333,"discrimination":0.518,"redundancyFactor":0.5855,"effectiveWeight":0.2831,"taskFamily":"tax_accounting","datasetLineageId":"vals-tax-agent-1","introducedAt":"2026-09-01","invertScore":false,"popoviciuSe":3.599079,"boundedMax":100,"sampleSize":193},"vals_mystery_mechanism":{"benchmarkId":"vals_mystery_mechanism","mean":31.014615,"sd":9.472427,"anchorCount":24,"anchorLabs":8,"linkedViaMeanSigma":false,"raschDelta":-0.00499,"reliability":0.8983,"discrimination":0.7292,"redundancyFactor":0.196,"effectiveWeight":0.1284,"taskFamily":"biomedical_science","datasetLineageId":"vals-mystery-mechanism-1","introducedAt":"2026-09-01","invertScore":false,"popoviciuSe":3.35578,"boundedMax":100,"sampleSize":222},"vals_medcode":{"benchmarkId":"vals_medcode","mean":45.756221,"sd":6.846699,"anchorCount":44,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":0.488677,"reliability":0.8969,"discrimination":0.7917,"redundancyFactor":0.2753,"effectiveWeight":0.1955,"taskFamily":"biomedical_science","datasetLineageId":"vals-medcode-1","introducedAt":"2026-08-15","invertScore":false,"popoviciuSe":3.892495,"boundedMax":100,"sampleSize":165},"surge_gdp_xlsx":{"benchmarkId":"surge_gdp_xlsx","mean":21.284444,"sd":7.667158,"anchorCount":28,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":-0.255373,"reliability":0.6011,"discrimination":0.4371,"redundancyFactor":0.3046,"effectiveWeight":0.1,"taskFamily":"data_analysis","datasetLineageId":"surge-gdp-xlsx","introducedAt":"2026-09-30","invertScore":false,"popoviciuSe":5.976143,"boundedMax":100,"sampleSize":70},"surge_dayjob_finance":{"benchmarkId":"surge_dayjob_finance","mean":18.134832,"sd":13.110197,"anchorCount":38,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":-0.162464,"reliability":0.9121,"discrimination":0.7083,"redundancyFactor":0.2276,"effectiveWeight":0.147,"taskFamily":"finance_analysis","datasetLineageId":"surge-dayjob-finance","introducedAt":"2026-09-23","invertScore":false,"popoviciuSe":5.59017,"boundedMax":100,"sampleSize":80},"surge_dayjob_healthcare":{"benchmarkId":"surge_dayjob_healthcare","mean":14.110935,"sd":13.052419,"anchorCount":38,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":-0.169441,"reliability":0.8798,"discrimination":0.7022,"redundancyFactor":0.237,"effectiveWeight":0.1464,"taskFamily":"biomedical_science","datasetLineageId":"surge-dayjob-healthcare","introducedAt":"2026-09-23","invertScore":false,"popoviciuSe":7.071068,"boundedMax":100,"sampleSize":50},"surge_riemann":{"benchmarkId":"surge_riemann","mean":55.844444,"sd":19.238128,"anchorCount":23,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":-0.252562,"reliability":0.7702,"discrimination":0.5661,"redundancyFactor":0.3261,"effectiveWeight":0.1422,"taskFamily":"frontier_math","datasetLineageId":"surge-riemann-bench","introducedAt":"2026-03-24","invertScore":false,"popoviciuSe":10,"boundedMax":100,"sampleSize":25},"scale_prbench_legal":{"benchmarkId":"scale_prbench_legal","mean":49.632448,"sd":5.129951,"anchorCount":22,"anchorLabs":5,"linkedViaMeanSigma":false,"raschDelta":-0.444345,"reliability":0.9899,"discrimination":0.8929,"redundancyFactor":0.2907,"effectiveWeight":0.2569,"taskFamily":"legal_analysis","datasetLineageId":"scale-prbench-legal","introducedAt":"2025-11-13","invertScore":false,"popoviciuSe":2.236068,"boundedMax":100,"sampleSize":500},"aa_analyst_agent":{"benchmarkId":"aa_analyst_agent","mean":41.632308,"sd":13.735494,"anchorCount":23,"anchorLabs":8,"linkedViaMeanSigma":true,"raschDelta":-0.050174,"reliability":0.806,"discrimination":0.6171,"redundancyFactor":0.212,"effectiveWeight":0.1054,"taskFamily":"data_analysis","datasetLineageId":"aa-analyst-agent-1.0","introducedAt":"2026-08-10","invertScore":false,"popoviciuSe":5.59017,"boundedMax":100,"sampleSize":80,"linkVarIntercept":0.12473246398277571,"linkVarSlope":0.00005363913433773498,"linkCovIS":-0.0023085253837375696},"aa_mka":{"benchmarkId":"aa_mka","mean":40.678356,"sd":8.685815,"anchorCount":18,"anchorLabs":8,"linkedViaMeanSigma":true,"raschDelta":-0.014195,"reliability":0.6936,"discrimination":0.5706,"redundancyFactor":0.2048,"effectiveWeight":0.1,"taskFamily":"multimodal_knowledge","datasetLineageId":"aa-mka-1.0","introducedAt":"2026-11-10","invertScore":false,"popoviciuSe":3.535534,"boundedMax":100,"sampleSize":200,"linkVarIntercept":0.08074949489669192,"linkVarSlope":0.00004260820874244952,"linkCovIS":-0.0018265245936352021},"aa_vials":{"benchmarkId":"aa_vials","mean":43.018385,"sd":15.130523,"anchorCount":15,"anchorLabs":8,"linkedViaMeanSigma":true,"raschDelta":0.02728,"reliability":0.8712,"discrimination":0.6786,"redundancyFactor":0.1204,"effectiveWeight":0.1,"taskFamily":"biomedical_science","datasetLineageId":"aa-vials-1.0","introducedAt":"2026-10-01","invertScore":false,"popoviciuSe":3.940552,"boundedMax":100,"sampleSize":161,"linkVarIntercept":0.24530447714423886,"linkVarSlope":0.00007693349144075212,"linkCovIS":-0.004276387970722542},"aa_briefcase":{"benchmarkId":"aa_briefcase","mean":1547.91028,"sd":150.932598,"anchorCount":20,"anchorLabs":9,"linkedViaMeanSigma":true,"raschDelta":-0.184071,"reliability":0.998,"discrimination":0.9375,"redundancyFactor":0.1256,"effectiveWeight":0.1175,"taskFamily":"knowledge_synthesis","datasetLineageId":"aa-briefcase-1.1","introducedAt":"2026-06-18","invertScore":false,"linkVarIntercept":1.8328266563872857,"linkVarSlope":7.278881232622068e-7,"linkCovIS":-0.001150403726648012},"gdpval_aa":{"benchmarkId":"gdpval_aa","mean":1625.651194,"sd":157.136873,"anchorCount":18,"anchorLabs":8,"linkedViaMeanSigma":true,"raschDelta":-0.000552,"reliability":0.9916,"discrimination":0.8661,"redundancyFactor":0.1535,"effectiveWeight":0.1318,"taskFamily":"knowledge_synthesis","datasetLineageId":"gdpval-aa-2.1","introducedAt":"2026-06-15","invertScore":false,"linkVarIntercept":6.143502424333781,"linkVarSlope":0.0000024499169505096503,"linkCovIS":-0.0038721738530352852},"surge_dayjob_law":{"benchmarkId":"surge_dayjob_law","mean":5.923632,"sd":17.044621,"anchorCount":21,"anchorLabs":8,"linkedViaMeanSigma":true,"raschDelta":-0.082557,"reliability":0.8803,"discrimination":0.6514,"redundancyFactor":0.2508,"effectiveWeight":0.1438,"taskFamily":"legal_analysis","datasetLineageId":"surge-dayjob-law","introducedAt":"2026-11-18","invertScore":false,"popoviciuSe":5.933908,"boundedMax":100,"sampleSize":71,"linkVarIntercept":0.02303740442073309,"linkVarSlope":0.00005033865964980318,"linkCovIS":-0.0007100946619638656},"scale_prbench_finance":{"benchmarkId":"scale_prbench_finance","mean":46.870045,"sd":7.531501,"anchorCount":28,"anchorLabs":6,"linkedViaMeanSigma":true,"raschDelta":0.098297,"reliability":0.9925,"discrimination":0.95,"redundancyFactor":0.2028,"effectiveWeight":0.1912,"taskFamily":"finance_analysis","datasetLineageId":"scale-prbench-finance","introducedAt":"2025-11-13","invertScore":false,"popoviciuSe":2.041241,"boundedMax":100,"sampleSize":600,"linkVarIntercept":1.7304114395207546,"linkVarSlope":0.0006584854600180772,"linkCovIS":-0.033219248447585524}},"configs":[{"id":"anthropic:claude-fable-5.2:max:direct:native","provider":"anthropic","modelSlug":"claude-fable-5.2","displayName":"Claude Fable 5.2","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":2.009519,"shrinkageFactor":0.99265,"score":80.14,"se":1.97,"samplingSe":1.2185,"residualSe":1.5428,"coverageCount":16,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":26.538558,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1000000,"releaseDate":"2026-11-09","paretoProb":0.4655,"benchmarks":[{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_briefcase:593f2d23-247b-41c9-a679-cebd225f0a9a","observedHarness":"Stirrup","rawScore":1783.12,"zScore":1.558376,"seModel":"upstream_ci","ciHalfWidth":10.56,"ciLower":1773.2,"ciUpper":1794.32,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase"},{"benchmarkId":"aa_mka","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_mka:593f2d23-247b-41c9-a679-cebd225f0a9a","observedHarness":"Stirrup","rawScore":56.56,"zScore":1.828458,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/mka"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:593f2d23-247b-41c9-a679-cebd225f0a9a","observedHarness":"Direct API","rawScore":79.13,"zScore":2.717892,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"aa_vials","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_vials:593f2d23-247b-41c9-a679-cebd225f0a9a","observedHarness":"Stirrup","rawScore":67.08,"zScore":1.59027,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/vials-aa"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:593f2d23-247b-41c9-a679-cebd225f0a9a","observedHarness":"Stirrup","rawScore":76.5,"zScore":1.982532,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:nD5Vaw2G6nRyLNsjfrpaYc","observedHarness":"Direct API","rawScore":94,"zScore":2.99435,"seModel":"upstream_se","seRaw":0.751375,"ciHalfWidth":1.4727,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-06","publishedDate":"2026-11-06","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:593f2d23-247b-41c9-a679-cebd225f0a9a","observedHarness":"Direct API","rawScore":24.2,"zScore":0.603185,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:593f2d23-247b-41c9-a679-cebd225f0a9a","observedHarness":"Direct API","rawScore":80.2,"zScore":2.815909,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Claude Fable 5.2 (Max)","observedHarness":"Direct API","rawScore":1483.08,"zScore":0.994438,"seModel":"upstream_ci","ciHalfWidth":7.3652,"ciLower":1475.7129,"ciUpper":1490.4433,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:593f2d23-247b-41c9-a679-cebd225f0a9a","observedHarness":"Direct API","rawScore":43.89,"zScore":2.559042,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Claude Fable 5.2 (Adaptive/Max)","observedHarness":"Direct API","rawScore":43.7,"zScore":1.950022,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Claude Fable 5.2 (Adaptive/Max)","observedHarness":"Direct API","rawScore":45,"zScore":2.366539,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:Claude Fable 5.2 (Adaptive/Max)","observedHarness":"Direct API","rawScore":26.5,"zScore":1.207206,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:Claude Fable 5.2 (Adaptive/Max)","observedHarness":"Direct API","rawScore":32.9,"zScore":1.514975,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:Claude Fable 5.2 (Adaptive/Max)","observedHarness":"Direct API","rawScore":83.2,"zScore":1.421945,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:593f2d23-247b-41c9-a679-cebd225f0a9a","observedHarness":"mini-SWE-agent","rawScore":73.33,"zScore":2.379774,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"anthropic:claude-fable-5.2:xhigh:direct:native","provider":"anthropic","modelSlug":"claude-fable-5.2","displayName":"Claude Fable 5.2","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.935928,"shrinkageFactor":0.988424,"score":79.04,"se":4.02,"samplingSe":1.5292,"residualSe":3.7132,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":14.828588,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1000000,"releaseDate":"2026-11-09","paretoProb":0.6335,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:76942526-cc96-46b0-bc27-29561c53ed70","observedHarness":"Direct API","rawScore":78.03,"zScore":2.639373,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:76942526-cc96-46b0-bc27-29561c53ed70","observedHarness":"Stirrup","rawScore":73.5,"zScore":1.827805,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:76942526-cc96-46b0-bc27-29561c53ed70","observedHarness":"Direct API","rawScore":21.4,"zScore":0.272724,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:76942526-cc96-46b0-bc27-29561c53ed70","observedHarness":"Direct API","rawScore":75.8,"zScore":2.576044,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:76942526-cc96-46b0-bc27-29561c53ed70","observedHarness":"mini-SWE-agent","rawScore":67.62,"zScore":2.117796,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"meta:muse-wonder:xhigh:direct:native","provider":"meta","modelSlug":"muse-wonder","displayName":"Muse Wonder","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.793196,"shrinkageFactor":0.967146,"score":76.9,"se":6.72,"samplingSe":2.5761,"residualSe":6.2118,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":5.650104,"costLogSd":0.5619,"costMinUsd":4.560546,"costMaxUsd":6.952036,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.872,"benchmarks":[{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:Muse Wonder 1.0 (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":58.67,"zScore":1.566747,"seModel":"upstream_ci","ciHalfWidth":1.69,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-14","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Muse Wonder 1.0 (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":58.4,"zScore":1.709091,"seModel":"upstream_ci","ciHalfWidth":1.76,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-14","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"}]},{"id":"google:gemini-4-neon:high:direct:native","provider":"google","modelSlug":"gemini-4-neon","displayName":"Gemini 4 Neon","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.76239,"shrinkageFactor":0.990372,"score":76.44,"se":2.3,"samplingSe":1.3946,"residualSe":1.8327,"coverageCount":14,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.74275,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1048576,"releaseDate":"2026-11-18","paretoProb":0.9609,"benchmarks":[{"benchmarkId":"aa_mka","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_mka:9b696f5b-b5d2-4b1f-9ed5-7c1446ec8421","observedHarness":"Stirrup","rawScore":54.59,"zScore":1.601651,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/mka"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:9b696f5b-b5d2-4b1f-9ed5-7c1446ec8421","observedHarness":"Direct API","rawScore":59.4,"zScore":1.309536,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:9b696f5b-b5d2-4b1f-9ed5-7c1446ec8421","observedHarness":"Stirrup","rawScore":59,"zScore":1.079956,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:rec0gHY2LSfDKLVDo","observedHarness":"APEX-Agents Harness","rawScore":82.7,"zScore":1.89171,"seModel":"upstream_se","seRaw":4.6,"ciHalfWidth":9.016,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:9b696f5b-b5d2-4b1f-9ed5-7c1446ec8421","observedHarness":"Direct API","rawScore":33.4,"zScore":1.688987,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:9b696f5b-b5d2-4b1f-9ed5-7c1446ec8421","observedHarness":"Direct API","rawScore":68.4,"zScore":2.172635,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Gemini 4 Neon (High)","observedHarness":"Direct API","rawScore":1509.28,"zScore":2.226628,"seModel":"upstream_ci","ciHalfWidth":7.1621,"ciLower":1502.1218,"ciUpper":1516.446,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:9b696f5b-b5d2-4b1f-9ed5-7c1446ec8421","observedHarness":"Direct API","rawScore":28.89,"zScore":0.881904,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Gemini 4 Neon (High reasoning)","observedHarness":"Direct API","rawScore":50.5,"zScore":2.468702,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Gemini 4 Neon (High reasoning)","observedHarness":"Direct API","rawScore":47.2,"zScore":2.535091,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:Gemini 4 Neon (High reasoning)","observedHarness":"Direct API","rawScore":26.2,"zScore":1.189605,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:Gemini 4 Neon (High reasoning)","observedHarness":"Direct API","rawScore":43.1,"zScore":2.845325,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:Gemini 4 Neon (High reasoning)","observedHarness":"Direct API","rawScore":84,"zScore":1.463529,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:9b696f5b-b5d2-4b1f-9ed5-7c1446ec8421","observedHarness":"mini-SWE-agent","rawScore":65.71,"zScore":2.030164,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"openai:gpt-6.2-astra:high:direct:native","provider":"openai","modelSlug":"gpt-6.2-astra","displayName":"GPT-6.2 Astra","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.725729,"shrinkageFactor":0.985502,"score":75.89,"se":3.53,"samplingSe":1.7113,"residualSe":3.0927,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":2.073875,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.7133,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:fe959a7a-944c-4ba8-8e41-349712599b39","observedHarness":"AA Analyst Harness","rawScore":61.25,"zScore":1.428248,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:fe959a7a-944c-4ba8-8e41-349712599b39","observedHarness":"Direct API","rawScore":66.33,"zScore":1.804209,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:fe959a7a-944c-4ba8-8e41-349712599b39","observedHarness":"Stirrup","rawScore":71.5,"zScore":1.724653,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:fe959a7a-944c-4ba8-8e41-349712599b39","observedHarness":"Direct API","rawScore":28.6,"zScore":1.122482,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:fe959a7a-944c-4ba8-8e41-349712599b39","observedHarness":"Direct API","rawScore":65.9,"zScore":2.036348,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:fe959a7a-944c-4ba8-8e41-349712599b39","observedHarness":"Direct API","rawScore":35.56,"zScore":1.627671,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:fe959a7a-944c-4ba8-8e41-349712599b39","observedHarness":"mini-SWE-agent","rawScore":75.24,"zScore":2.467405,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"anthropic:claude-fable-5.2:medium:direct:native","provider":"anthropic","modelSlug":"claude-fable-5.2","displayName":"Claude Fable 5.2","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.596708,"shrinkageFactor":0.986883,"score":73.95,"se":4.05,"samplingSe":1.6277,"residualSe":3.7074,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":3.802605,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1000000,"releaseDate":"2026-11-09","paretoProb":0.5228,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:0c9b4e05-9259-42cc-b40e-d6acbbeaa7e8","observedHarness":"Direct API","rawScore":76.97,"zScore":2.563708,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:0c9b4e05-9259-42cc-b40e-d6acbbeaa7e8","observedHarness":"Stirrup","rawScore":61.5,"zScore":1.208896,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:0c9b4e05-9259-42cc-b40e-d6acbbeaa7e8","observedHarness":"Direct API","rawScore":23.8,"zScore":0.555977,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:0c9b4e05-9259-42cc-b40e-d6acbbeaa7e8","observedHarness":"Direct API","rawScore":66.3,"zScore":2.058154,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:0c9b4e05-9259-42cc-b40e-d6acbbeaa7e8","observedHarness":"mini-SWE-agent","rawScore":60.48,"zScore":1.790209,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"google:gemini-4-neon:xhigh:direct:native","provider":"google","modelSlug":"gemini-4-neon","displayName":"Gemini 4 Neon","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.580561,"shrinkageFactor":0.991029,"score":73.71,"se":2.75,"samplingSe":1.3461,"residualSe":2.3974,"coverageCount":10,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.444838,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1048576,"releaseDate":"2026-11-18","paretoProb":0.194,"benchmarks":[{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_briefcase:7c667f2d-0149-4eb1-864c-7e66c166ad88","observedHarness":"Stirrup","rawScore":1698.69,"zScore":0.998987,"seModel":"upstream_ci","ciHalfWidth":11.205,"ciLower":1687.24,"ciUpper":1709.65,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase"},{"benchmarkId":"aa_mka","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_mka:7c667f2d-0149-4eb1-864c-7e66c166ad88","observedHarness":"Stirrup","rawScore":56.48,"zScore":1.819247,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/mka"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:7c667f2d-0149-4eb1-864c-7e66c166ad88","observedHarness":"Direct API","rawScore":60.47,"zScore":1.385914,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:7c667f2d-0149-4eb1-864c-7e66c166ad88","observedHarness":"Stirrup","rawScore":60.5,"zScore":1.15732,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:rec9YdMYfCNljZ7RO","observedHarness":"APEX-Agents Harness","rawScore":82.9,"zScore":1.903736,"seModel":"upstream_se","seRaw":4.6,"ciHalfWidth":9.016,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:oNv3qiZwkezTmFHngNKsne","observedHarness":"Direct API","rawScore":75.2,"zScore":1.656908,"seModel":"upstream_se","seRaw":1.366319,"ciHalfWidth":2.678,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-19","publishedDate":"2026-11-19","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:7c667f2d-0149-4eb1-864c-7e66c166ad88","observedHarness":"Direct API","rawScore":28,"zScore":1.051669,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:7c667f2d-0149-4eb1-864c-7e66c166ad88","observedHarness":"Direct API","rawScore":71.3,"zScore":2.330728,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:7c667f2d-0149-4eb1-864c-7e66c166ad88","observedHarness":"Direct API","rawScore":31.67,"zScore":1.192734,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:7c667f2d-0149-4eb1-864c-7e66c166ad88","observedHarness":"mini-SWE-agent","rawScore":70,"zScore":2.226991,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"google:gemini-4-carbon:high:direct:native","provider":"google","modelSlug":"gemini-4-carbon","displayName":"Gemini 4 Carbon","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.491502,"shrinkageFactor":0.994844,"score":72.37,"se":1.29,"samplingSe":1.0205,"residualSe":0.7842,"coverageCount":22,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":7.755546,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-10-12","paretoProb":0.0321,"benchmarks":[{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_briefcase:909ed0ba-bcc5-4f38-8162-08a4e187a772","observedHarness":"Stirrup","rawScore":1610.63,"zScore":0.415548,"seModel":"upstream_ci","ciHalfWidth":10.245,"ciLower":1600.75,"ciUpper":1621.24,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase"},{"benchmarkId":"aa_mka","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_mka:909ed0ba-bcc5-4f38-8162-08a4e187a772","observedHarness":"Stirrup","rawScore":54.34,"zScore":1.572868,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/mka"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:909ed0ba-bcc5-4f38-8162-08a4e187a772","observedHarness":"Direct API","rawScore":58.27,"zScore":1.228875,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"aa_vials","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_vials:909ed0ba-bcc5-4f38-8162-08a4e187a772","observedHarness":"Stirrup","rawScore":63.98,"zScore":1.385386,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/vials-aa"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:909ed0ba-bcc5-4f38-8162-08a4e187a772","observedHarness":"Stirrup","rawScore":69,"zScore":1.595714,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:CUUbVTzr7whCGEzPgrTozQ","observedHarness":"Direct API","rawScore":77.7,"zScore":1.834759,"seModel":"upstream_se","seRaw":1.316983,"ciHalfWidth":2.5813,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-02","publishedDate":"2026-11-02","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:909ed0ba-bcc5-4f38-8162-08a4e187a772","observedHarness":"Direct API","rawScore":28.4,"zScore":1.098877,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdpval_aa","benchmarkVersion":"2.1","sourceId":"artificial-analysis","sourceRowId":"aa:gdpval_aa:909ed0ba-bcc5-4f38-8162-08a4e187a772","observedHarness":"Stirrup","rawScore":1732.68,"zScore":0.681118,"seModel":"upstream_ci","ciHalfWidth":24.325,"ciLower":1708.36,"ciUpper":1757.01,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:909ed0ba-bcc5-4f38-8162-08a4e187a772","observedHarness":"Direct API","rawScore":69.1,"zScore":2.210795,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Gemini 4 Carbon (High)","observedHarness":"Direct API","rawScore":1522.23,"zScore":2.835668,"seModel":"upstream_ci","ciHalfWidth":8.4005,"ciLower":1513.827,"ciUpper":1530.6281,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:909ed0ba-bcc5-4f38-8162-08a4e187a772","observedHarness":"Direct API","rawScore":19.44,"zScore":-0.174693,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Gemini 4 Carbon (one-pass-agent) high","observedHarness":"Scale one-pass-agent","rawScore":53.35,"zScore":0.724676,"seModel":"upstream_ci","ciHalfWidth":0.33,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-11","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Gemini 4 Carbon (High reasoning)","observedHarness":"Direct API","rawScore":23.8,"zScore":0.432119,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Gemini 4 Carbon (High reasoning)","observedHarness":"Direct API","rawScore":35.2,"zScore":1.615721,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:Gemini 4 Carbon (High reasoning)","observedHarness":"Direct API","rawScore":13.8,"zScore":0.462103,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:Gemini 4 Carbon (High reasoning)","observedHarness":"Direct API","rawScore":38.9,"zScore":2.297534,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:Gemini 4 Carbon (High reasoning)","observedHarness":"Direct API","rawScore":80,"zScore":1.255608,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:909ed0ba-bcc5-4f38-8162-08a4e187a772","observedHarness":"mini-SWE-agent","rawScore":64.76,"zScore":1.986577,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:google/gemini-4-carbon","observedHarness":"Direct API","rawScore":76.47,"zScore":1.490904,"seModel":"upstream_se","seRaw":2.205,"ciHalfWidth":4.3218,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.755811,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:google/gemini-4-carbon","observedHarness":"Vals Finance Agent","rawScore":67.16,"zScore":1.220824,"seModel":"upstream_se","seRaw":2.024,"ciHalfWidth":3.967,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.290456,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:google/gemini-4-carbon","observedHarness":"Direct API","rawScore":55.77,"zScore":1.506266,"seModel":"upstream_se","seRaw":3.452,"ciHalfWidth":6.7659,"ciLower":null,"ciUpper":null,"measuredCostUsd":8.743524,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:google/gemini-4-carbon","observedHarness":"Vals Tax Agent","rawScore":51.78,"zScore":1.912089,"seModel":"upstream_se","seRaw":3.604,"ciHalfWidth":7.0638,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.340026,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-6.2-astra:medium:direct:native","provider":"openai","modelSlug":"gpt-6.2-astra","displayName":"GPT-6.2 Astra","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.454205,"shrinkageFactor":0.985439,"score":71.81,"se":3.54,"samplingSe":1.715,"residualSe":3.0925,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":1.617688,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.5098,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:8cec011f-c959-4726-8668-2ee2409a6bca","observedHarness":"AA Analyst Harness","rawScore":52.5,"zScore":0.791212,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:8cec011f-c959-4726-8668-2ee2409a6bca","observedHarness":"Direct API","rawScore":64.17,"zScore":1.650026,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:8cec011f-c959-4726-8668-2ee2409a6bca","observedHarness":"Stirrup","rawScore":67.5,"zScore":1.51835,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:8cec011f-c959-4726-8668-2ee2409a6bca","observedHarness":"Direct API","rawScore":28.4,"zScore":1.098877,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:8cec011f-c959-4726-8668-2ee2409a6bca","observedHarness":"Direct API","rawScore":56,"zScore":1.496652,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:8cec011f-c959-4726-8668-2ee2409a6bca","observedHarness":"Direct API","rawScore":32.78,"zScore":1.316842,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:8cec011f-c959-4726-8668-2ee2409a6bca","observedHarness":"mini-SWE-agent","rawScore":74.76,"zScore":2.445383,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"anthropic:claude-fable-5.2:low:direct:native","provider":"anthropic","modelSlug":"claude-fable-5.2","displayName":"Claude Fable 5.2","effort":"low","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.435444,"shrinkageFactor":0.986414,"score":71.53,"se":4.06,"samplingSe":1.6566,"residualSe":3.7056,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.097242,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1000000,"releaseDate":"2026-11-09","paretoProb":0.6262,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:cfc7acb1-d903-4858-9b6c-6c7776caf8f9","observedHarness":"Direct API","rawScore":76.3,"zScore":2.515883,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:cfc7acb1-d903-4858-9b6c-6c7776caf8f9","observedHarness":"Stirrup","rawScore":60,"zScore":1.131532,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:cfc7acb1-d903-4858-9b6c-6c7776caf8f9","observedHarness":"Direct API","rawScore":26.8,"zScore":0.910042,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:cfc7acb1-d903-4858-9b6c-6c7776caf8f9","observedHarness":"Direct API","rawScore":60.8,"zScore":1.758323,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:cfc7acb1-d903-4858-9b6c-6c7776caf8f9","observedHarness":"mini-SWE-agent","rawScore":53.81,"zScore":1.484185,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"anthropic:claude-opus-5.5:max:direct:native","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.43252,"shrinkageFactor":0.992671,"score":71.49,"se":1.85,"samplingSe":1.2167,"residualSe":1.3903,"coverageCount":19,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":10.03641,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.0773,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:2f3c4dc9-a450-4303-8697-0237257cf08f","observedHarness":"AA Analyst Harness","rawScore":56.25,"zScore":1.064228,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_briefcase:2f3c4dc9-a450-4303-8697-0237257cf08f","observedHarness":"Stirrup","rawScore":1797.44,"zScore":1.653253,"seModel":"upstream_ci","ciHalfWidth":10.26,"ciLower":1787.51,"ciUpper":1808.03,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase"},{"benchmarkId":"aa_mka","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_mka:2f3c4dc9-a450-4303-8697-0237257cf08f","observedHarness":"Stirrup","rawScore":53.01,"zScore":1.419745,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/mka"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:2f3c4dc9-a450-4303-8697-0237257cf08f","observedHarness":"Direct API","rawScore":65.5,"zScore":1.744963,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"aa_vials","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_vials:2f3c4dc9-a450-4303-8697-0237257cf08f","observedHarness":"Stirrup","rawScore":61.49,"zScore":1.220818,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/vials-aa"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:2f3c4dc9-a450-4303-8697-0237257cf08f","observedHarness":"Stirrup","rawScore":68,"zScore":1.544138,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:rec6TErVzXnuNRUoV","observedHarness":"APEX-Agents Harness","rawScore":73.5,"zScore":1.338524,"seModel":"upstream_se","seRaw":4.9,"ciHalfWidth":9.604,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:CgLfiGcNwMPVzxr8GdNUAK","observedHarness":"Direct API","rawScore":72.2,"zScore":1.443487,"seModel":"upstream_se","seRaw":1.417452,"ciHalfWidth":2.7782,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-22","publishedDate":"2026-09-22","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:2f3c4dc9-a450-4303-8697-0237257cf08f","observedHarness":"Direct API","rawScore":26.2,"zScore":0.839229,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:2f3c4dc9-a450-4303-8697-0237257cf08f","observedHarness":"Direct API","rawScore":67.9,"zScore":2.145377,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:2f3c4dc9-a450-4303-8697-0237257cf08f","observedHarness":"Direct API","rawScore":27.78,"zScore":0.757796,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Claude Opus 5.5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":42.5,"zScore":1.85849,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Claude Opus 5.5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":48.3,"zScore":2.619366,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:Claude Opus 5.5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":23.7,"zScore":1.042931,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:Claude Opus 5.5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":30.3,"zScore":1.175867,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:Claude Opus 5.5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":80,"zScore":1.255608,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:2f3c4dc9-a450-4303-8697-0237257cf08f","observedHarness":"mini-SWE-agent","rawScore":59.05,"zScore":1.724599,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:anthropic/claude-opus-5-5","observedHarness":"Direct API","rawScore":75.84,"zScore":1.318035,"seModel":"upstream_se","seRaw":2.41,"ciHalfWidth":4.7236,"ciLower":null,"ciUpper":null,"measuredCostUsd":9.330415,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:anthropic/claude-opus-5-5","observedHarness":"Direct API","rawScore":49.8,"zScore":0.590617,"seModel":"upstream_se","seRaw":2.273,"ciHalfWidth":4.4551,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.634301,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"}]},{"id":"openai:gpt-6.2-astra:xhigh:direct:native","provider":"openai","modelSlug":"gpt-6.2-astra","displayName":"GPT-6.2 Astra","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.408299,"shrinkageFactor":0.98939,"score":71.12,"se":3.08,"samplingSe":1.464,"residualSe":2.7062,"coverageCount":8,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.521143,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.0547,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:592572d5-f789-4106-abbc-fadf2206d10b","observedHarness":"AA Analyst Harness","rawScore":62.5,"zScore":1.519253,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:592572d5-f789-4106-abbc-fadf2206d10b","observedHarness":"Direct API","rawScore":67.2,"zScore":1.866311,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:592572d5-f789-4106-abbc-fadf2206d10b","observedHarness":"Stirrup","rawScore":70.5,"zScore":1.673078,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:592572d5-f789-4106-abbc-fadf2206d10b","observedHarness":"Direct API","rawScore":28,"zScore":1.051669,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:592572d5-f789-4106-abbc-fadf2206d10b","observedHarness":"Direct API","rawScore":67.3,"zScore":2.112669,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:592572d5-f789-4106-abbc-fadf2206d10b","observedHarness":"Direct API","rawScore":37.78,"zScore":1.875888,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:GPT 6.2 Astra (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":43.08,"zScore":-1.277293,"seModel":"upstream_ci","ciHalfWidth":0.96,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-18","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:592572d5-f789-4106-abbc-fadf2206d10b","observedHarness":"mini-SWE-agent","rawScore":78.57,"zScore":2.620188,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"anthropic:claude-fable-5.2:high:direct:native","provider":"anthropic","modelSlug":"claude-fable-5.2","displayName":"Claude Fable 5.2","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.401315,"shrinkageFactor":0.992773,"score":71.02,"se":2.36,"samplingSe":1.2082,"residualSe":2.026,"coverageCount":11,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":9.078415,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1000000,"releaseDate":"2026-11-09","paretoProb":0.0655,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:fdc1913b-9497-45ef-83f7-17dae756f37e","observedHarness":"Direct API","rawScore":77.23,"zScore":2.582267,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:fdc1913b-9497-45ef-83f7-17dae756f37e","observedHarness":"Stirrup","rawScore":73,"zScore":1.802017,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:fdc1913b-9497-45ef-83f7-17dae756f37e","observedHarness":"Direct API","rawScore":21.8,"zScore":0.319933,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:fdc1913b-9497-45ef-83f7-17dae756f37e","observedHarness":"Direct API","rawScore":73.8,"zScore":2.467014,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:Claude Fable 5.2 (one-pass-agent) high","observedHarness":"Scale one-pass-agent","rawScore":47.07,"zScore":0.026549,"seModel":"upstream_ci","ciHalfWidth":0.13,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-12","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Claude Fable 5.2 (one-pass-agent) high","observedHarness":"Scale one-pass-agent","rawScore":48.97,"zScore":-0.129133,"seModel":"upstream_ci","ciHalfWidth":0.03,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-12","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:fdc1913b-9497-45ef-83f7-17dae756f37e","observedHarness":"mini-SWE-agent","rawScore":70,"zScore":2.226991,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:anthropic/claude-fable-5-2","observedHarness":"Direct API","rawScore":77.38,"zScore":1.740604,"seModel":"upstream_se","seRaw":2.187,"ciHalfWidth":4.2865,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.672375,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:anthropic/claude-fable-5-2","observedHarness":"Vals Finance Agent","rawScore":69.47,"zScore":1.882562,"seModel":"upstream_se","seRaw":1.977,"ciHalfWidth":3.8749,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.833915,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:anthropic/claude-fable-5-2","observedHarness":"Direct API","rawScore":51.8,"zScore":2.194304,"seModel":"upstream_se","seRaw":3.361,"ciHalfWidth":6.5876,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.552812,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:anthropic/claude-fable-5-2","observedHarness":"Vals Tax Agent","rawScore":32.09,"zScore":0.17651,"seModel":"upstream_se","seRaw":3.367,"ciHalfWidth":6.5993,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.309035,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-6.2-sol:high:direct:native","provider":"openai","modelSlug":"gpt-6.2-sol","displayName":"GPT-6.2 Sol","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.361362,"shrinkageFactor":0.984688,"score":70.42,"se":3.56,"samplingSe":1.7587,"residualSe":3.0901,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.563645,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.7532,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:cd85e412-c43f-4535-bf8e-5ba113d97ca3","observedHarness":"AA Analyst Harness","rawScore":52.5,"zScore":0.791212,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:cd85e412-c43f-4535-bf8e-5ba113d97ca3","observedHarness":"Direct API","rawScore":61.43,"zScore":1.45444,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:cd85e412-c43f-4535-bf8e-5ba113d97ca3","observedHarness":"Stirrup","rawScore":61.5,"zScore":1.208896,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:cd85e412-c43f-4535-bf8e-5ba113d97ca3","observedHarness":"Direct API","rawScore":28.8,"zScore":1.146086,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:cd85e412-c43f-4535-bf8e-5ba113d97ca3","observedHarness":"Direct API","rawScore":58.3,"zScore":1.622036,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:cd85e412-c43f-4535-bf8e-5ba113d97ca3","observedHarness":"Direct API","rawScore":33.89,"zScore":1.44095,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:cd85e412-c43f-4535-bf8e-5ba113d97ca3","observedHarness":"mini-SWE-agent","rawScore":67.62,"zScore":2.117796,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"anthropic:claude-fable-5.1:max:direct:native","provider":"anthropic","modelSlug":"claude-fable-5.1","displayName":"Claude Fable 5.1","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.326305,"shrinkageFactor":0.988751,"score":69.89,"se":2.67,"samplingSe":1.5074,"residualSe":2.2048,"coverageCount":12,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":27.818269,"costLogSd":1.0653,"costImputed":true,"costEstimated":true,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-09-12","paretoProb":0.0762,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:ExyL3GN8s34tizrXV3VmS3","observedHarness":"Direct API","rawScore":70.8,"zScore":1.34389,"seModel":"upstream_se","seRaw":1.438551,"ciHalfWidth":2.8196,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-01","publishedDate":"2026-09-01","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdpval_aa","benchmarkVersion":"2.1","sourceId":"artificial-analysis","sourceRowId":"aa:gdpval_aa:3e87c73e-a257-495e-9730-367a66229811","observedHarness":"Stirrup","rawScore":1764.5,"zScore":0.883617,"seModel":"upstream_ci","ciHalfWidth":18,"ciLower":1746.5,"ciUpper":1782.5,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Claude Fable 5.1 (Max)","observedHarness":"Direct API","rawScore":1498.69,"zScore":1.728579,"seModel":"upstream_ci","ciHalfWidth":4.4976,"ciLower":1494.1952,"ciUpper":1503.1903,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Claude Fable 5.1 (Adaptive/Max)","observedHarness":"Direct API","rawScore":36.5,"zScore":1.400831,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Claude Fable 5.1 (Adaptive/Max)","observedHarness":"Direct API","rawScore":31.2,"zScore":1.309264,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:Claude Fable 5.1 (Adaptive/Max)","observedHarness":"Direct API","rawScore":22.5,"zScore":0.972528,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:Claude Fable 5.1 (Adaptive/Max)","observedHarness":"Direct API","rawScore":23.1,"zScore":0.236796,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:Claude Fable 5.1 (Adaptive/Max)","observedHarness":"Direct API","rawScore":71.2,"zScore":0.798183,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:anthropic/claude-fable-5-1","observedHarness":"Direct API","rawScore":76.73,"zScore":1.562247,"seModel":"upstream_se","seRaw":2.129,"ciHalfWidth":4.1728,"ciLower":null,"ciUpper":null,"measuredCostUsd":15.93249,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:anthropic/claude-fable-5-1","observedHarness":"Direct API","rawScore":53.51,"zScore":1.132484,"seModel":"upstream_se","seRaw":2.165,"ciHalfWidth":4.2434,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.116862,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:anthropic/claude-fable-5-1","observedHarness":"Direct API","rawScore":47.75,"zScore":1.766747,"seModel":"upstream_se","seRaw":3.36,"ciHalfWidth":6.5856,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.631518,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:anthropic/claude-fable-5-1","observedHarness":"Vals Tax Agent","rawScore":50.75,"zScore":1.821299,"seModel":"upstream_se","seRaw":3.606,"ciHalfWidth":7.0678,"ciLower":null,"ciUpper":null,"measuredCostUsd":13.180073,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-6.2-astra:max:direct:native","provider":"openai","modelSlug":"gpt-6.2-astra","displayName":"GPT-6.2 Astra","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.318346,"shrinkageFactor":0.994194,"score":69.78,"se":1.41,"samplingSe":1.083,"residualSe":0.9061,"coverageCount":23,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.997906,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.0094,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:75c1b0b8-9db3-4c7e-95be-d74e59defee4","observedHarness":"AA Analyst Harness","rawScore":62.5,"zScore":1.519253,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_briefcase:75c1b0b8-9db3-4c7e-95be-d74e59defee4","observedHarness":"Stirrup","rawScore":1643.07,"zScore":0.630478,"seModel":"upstream_ci","ciHalfWidth":11.605,"ciLower":1632.19,"ciUpper":1655.4,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase"},{"benchmarkId":"aa_mka","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_mka:75c1b0b8-9db3-4c7e-95be-d74e59defee4","observedHarness":"Stirrup","rawScore":52.67,"zScore":1.380601,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/mka"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:75c1b0b8-9db3-4c7e-95be-d74e59defee4","observedHarness":"Direct API","rawScore":68.57,"zScore":1.964104,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"aa_vials","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_vials:75c1b0b8-9db3-4c7e-95be-d74e59defee4","observedHarness":"Stirrup","rawScore":65.84,"zScore":1.508316,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/vials-aa"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:75c1b0b8-9db3-4c7e-95be-d74e59defee4","observedHarness":"Stirrup","rawScore":71.5,"zScore":1.724653,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recsqgjVWEICCIkzD","observedHarness":"APEX-Agents Harness","rawScore":72.2,"zScore":1.260356,"seModel":"upstream_se","seRaw":5.2,"ciHalfWidth":10.192,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:kErTfQGFajG4FTzYgrEysk","observedHarness":"Direct API","rawScore":78.5,"zScore":1.891672,"seModel":"upstream_se","seRaw":1.299784,"ciHalfWidth":2.5476,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-27","publishedDate":"2026-10-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:75c1b0b8-9db3-4c7e-95be-d74e59defee4","observedHarness":"Direct API","rawScore":29.8,"zScore":1.264108,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:75c1b0b8-9db3-4c7e-95be-d74e59defee4","observedHarness":"Direct API","rawScore":72.2,"zScore":2.379791,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-6.2 Astra (Max)","observedHarness":"Direct API","rawScore":1439.15,"zScore":-1.071595,"seModel":"upstream_ci","ciHalfWidth":10.6703,"ciLower":1428.4824,"ciUpper":1449.823,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:75c1b0b8-9db3-4c7e-95be-d74e59defee4","observedHarness":"Direct API","rawScore":39.44,"zScore":2.061491,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:GPT 6.2 Astra (Max reasoning)","observedHarness":"Direct API","rawScore":45.8,"zScore":2.110202,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:GPT 6.2 Astra (Max reasoning)","observedHarness":"Direct API","rawScore":34.4,"zScore":1.554429,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:GPT 6.2 Astra (Max reasoning)","observedHarness":"Direct API","rawScore":41.4,"zScore":2.081382,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:GPT 6.2 Astra (Max reasoning)","observedHarness":"Direct API","rawScore":29.7,"zScore":1.097611,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:GPT 6.2 Astra (Max reasoning)","observedHarness":"Direct API","rawScore":76,"zScore":1.047688,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:75c1b0b8-9db3-4c7e-95be-d74e59defee4","observedHarness":"mini-SWE-agent","rawScore":78.1,"zScore":2.598624,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:openai/gpt-6.2-astra","observedHarness":"Direct API","rawScore":76.03,"zScore":1.37017,"seModel":"upstream_se","seRaw":2.243,"ciHalfWidth":4.3963,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.801461,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:openai/gpt-6.2-astra","observedHarness":"Vals Finance Agent","rawScore":60.75,"zScore":-0.615425,"seModel":"upstream_se","seRaw":2.124,"ciHalfWidth":4.163,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.976553,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:openai/gpt-6.2-astra","observedHarness":"Direct API","rawScore":42.79,"zScore":0.354873,"seModel":"upstream_se","seRaw":3.439,"ciHalfWidth":6.7404,"ciLower":null,"ciUpper":null,"measuredCostUsd":12.836241,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:openai/gpt-6.2-astra","observedHarness":"Direct API","rawScore":53.15,"zScore":2.336823,"seModel":"upstream_se","seRaw":3.357,"ciHalfWidth":6.5797,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.720342,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:openai/gpt-6.2-astra","observedHarness":"Vals Tax Agent","rawScore":21.84,"zScore":-0.726978,"seModel":"upstream_se","seRaw":2.794,"ciHalfWidth":5.4762,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.898144,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"google:gemini-4-neon:medium:direct:native","provider":"google","modelSlug":"gemini-4-neon","displayName":"Gemini 4 Neon","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.2985,"shrinkageFactor":0.984596,"score":69.48,"se":3.57,"samplingSe":1.7639,"residualSe":3.0998,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":1.904556,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1048576,"releaseDate":"2026-11-18","paretoProb":0.8652,"benchmarks":[{"benchmarkId":"aa_mka","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_mka:2ba1a8f6-e052-4a33-b178-4db1e86157e4","observedHarness":"Stirrup","rawScore":50.91,"zScore":1.177972,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/mka"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:2ba1a8f6-e052-4a33-b178-4db1e86157e4","observedHarness":"Direct API","rawScore":57.93,"zScore":1.204605,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:2ba1a8f6-e052-4a33-b178-4db1e86157e4","observedHarness":"Stirrup","rawScore":61.5,"zScore":1.208896,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:2ba1a8f6-e052-4a33-b178-4db1e86157e4","observedHarness":"Direct API","rawScore":32.8,"zScore":1.618174,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:2ba1a8f6-e052-4a33-b178-4db1e86157e4","observedHarness":"Direct API","rawScore":63.6,"zScore":1.910964,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:2ba1a8f6-e052-4a33-b178-4db1e86157e4","observedHarness":"Direct API","rawScore":30.56,"zScore":1.068625,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:2ba1a8f6-e052-4a33-b178-4db1e86157e4","observedHarness":"mini-SWE-agent","rawScore":54.76,"zScore":1.527772,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"anthropic:claude-opus-4.6:high:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.6","displayName":"Claude Opus 4.6","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.278873,"shrinkageFactor":0.995579,"score":69.18,"se":9.25,"samplingSe":0.945,"residualSe":9.2016,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.432844,"costLogSd":1.0492,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-02-05","paretoProb":0.3754,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Claude Opus 4.6 (High)","observedHarness":"Direct API","rawScore":1492.7,"zScore":1.446868,"seModel":"upstream_ci","ciHalfWidth":2.6313,"ciLower":1490.0648,"ciUpper":1495.3275,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"meta:muse-spark-1.3:xhigh:direct:native","provider":"meta","modelSlug":"muse-spark-1.3","displayName":"Muse Spark 1.3","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.235401,"shrinkageFactor":0.977971,"score":68.53,"se":4.29,"samplingSe":2.1094,"residualSe":3.7374,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.165655,"costLogSd":1.051,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.8838,"benchmarks":[{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:Muse Spark 1.3 (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":59.54,"zScore":1.682262,"seModel":"upstream_ci","ciHalfWidth":1.66,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-14","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Muse Spark 1.3 (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":61.56,"zScore":2.325081,"seModel":"upstream_ci","ciHalfWidth":1.72,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-14","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:Muse Spark 1.3 (xHigh reasoning)","observedHarness":"Direct API","rawScore":53.6,"zScore":-0.116666,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:meta/muse_spark_1_3","observedHarness":"Direct API","rawScore":40.87,"zScore":0.184559,"seModel":"upstream_se","seRaw":3.417,"ciHalfWidth":6.6973,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.650063,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:meta/muse_spark_1_3","observedHarness":"Vals Tax Agent","rawScore":37.89,"zScore":0.687752,"seModel":"upstream_se","seRaw":3.45,"ciHalfWidth":6.762,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.319558,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"anthropic:claude-fable-5:xhigh:direct:native","provider":"anthropic","modelSlug":"claude-fable-5","displayName":"Claude Fable 5","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.231218,"shrinkageFactor":0.988445,"score":68.47,"se":9.23,"samplingSe":1.5277,"residualSe":9.1042,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":19.394239,"costLogSd":1.0664,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-06-09","paretoProb":0.2198,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:SEnvgtunRa5GwAtU2sHa87","observedHarness":"Direct API","rawScore":70.7,"zScore":1.336776,"seModel":"upstream_se","seRaw":1.439994,"ciHalfWidth":2.8224,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-10","publishedDate":"2026-08-10","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"}]},{"id":"google:gemini-4-carbon:medium:direct:native","provider":"google","modelSlug":"gemini-4-carbon","displayName":"Gemini 4 Carbon","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.227383,"shrinkageFactor":0.985608,"score":68.41,"se":3.74,"samplingSe":1.705,"residualSe":3.3317,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.585777,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-10-12","paretoProb":0.0216,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:0d3eaafd-54ed-4f78-bdd7-05a2db883986","observedHarness":"Direct API","rawScore":48.93,"zScore":0.562172,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:0d3eaafd-54ed-4f78-bdd7-05a2db883986","observedHarness":"Stirrup","rawScore":69,"zScore":1.595714,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:0d3eaafd-54ed-4f78-bdd7-05a2db883986","observedHarness":"Direct API","rawScore":29.6,"zScore":1.240504,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:0d3eaafd-54ed-4f78-bdd7-05a2db883986","observedHarness":"Direct API","rawScore":67.4,"zScore":2.11812,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:0d3eaafd-54ed-4f78-bdd7-05a2db883986","observedHarness":"Direct API","rawScore":18.33,"zScore":-0.298801,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:0d3eaafd-54ed-4f78-bdd7-05a2db883986","observedHarness":"mini-SWE-agent","rawScore":62.38,"zScore":1.877382,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"google:gemini-4-hydrogen:xhigh:direct:native","provider":"google","modelSlug":"gemini-4-hydrogen","displayName":"Gemini 4 Hydrogen","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.197209,"shrinkageFactor":0.992253,"score":67.96,"se":1.94,"samplingSe":1.2509,"residualSe":1.4786,"coverageCount":16,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.160159,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":1.5,"outputPer1mUsd":7.5,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.3256,"benchmarks":[{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_briefcase:dad0e3b1-e3e7-477a-99f9-c14457a91114","observedHarness":"Stirrup","rawScore":1568.48,"zScore":0.136284,"seModel":"upstream_ci","ciHalfWidth":10.645,"ciLower":1557.9,"ciUpper":1579.19,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:dad0e3b1-e3e7-477a-99f9-c14457a91114","observedHarness":"Direct API","rawScore":50.47,"zScore":0.672099,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:dad0e3b1-e3e7-477a-99f9-c14457a91114","observedHarness":"Stirrup","rawScore":66,"zScore":1.440987,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:BTWvEjbekh8TC7kWurJnrR","observedHarness":"Direct API","rawScore":70.9,"zScore":1.351004,"seModel":"upstream_se","seRaw":1.4371,"ciHalfWidth":2.8167,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-13","publishedDate":"2026-11-13","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:dad0e3b1-e3e7-477a-99f9-c14457a91114","observedHarness":"Direct API","rawScore":27.6,"zScore":1.00446,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:dad0e3b1-e3e7-477a-99f9-c14457a91114","observedHarness":"Direct API","rawScore":58.8,"zScore":1.649293,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:dad0e3b1-e3e7-477a-99f9-c14457a91114","observedHarness":"Direct API","rawScore":21.11,"zScore":0.012028,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Gemini 4 Hydrogen (xHigh reasoning)","observedHarness":"Direct API","rawScore":38.5,"zScore":1.553384,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Gemini 4 Hydrogen (xHigh reasoning)","observedHarness":"Direct API","rawScore":24,"zScore":0.757642,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:Gemini 4 Hydrogen (xHigh reasoning)","observedHarness":"Direct API","rawScore":9.7,"zScore":0.221558,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:Gemini 4 Hydrogen (xHigh reasoning)","observedHarness":"Direct API","rawScore":40.9,"zScore":2.558387,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:dad0e3b1-e3e7-477a-99f9-c14457a91114","observedHarness":"mini-SWE-agent","rawScore":49.52,"zScore":1.287358,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:google/gemini-4-hydrogen","observedHarness":"Direct API","rawScore":75.8,"zScore":1.307059,"seModel":"upstream_se","seRaw":2.034,"ciHalfWidth":3.9866,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.170206,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:google/gemini-4-hydrogen","observedHarness":"Vals Finance Agent","rawScore":65.12,"zScore":0.636433,"seModel":"upstream_se","seRaw":2.036,"ciHalfWidth":3.9906,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.358646,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:google/gemini-4-hydrogen","observedHarness":"Direct API","rawScore":53.85,"zScore":1.335952,"seModel":"upstream_se","seRaw":3.465,"ciHalfWidth":6.7914,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.268559,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:google/gemini-4-hydrogen","observedHarness":"Vals Tax Agent","rawScore":52.3,"zScore":1.957924,"seModel":"upstream_se","seRaw":3.603,"ciHalfWidth":7.0619,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.870924,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"anthropic:claude-opus-5.5:xhigh:direct:native","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.178981,"shrinkageFactor":0.987789,"score":67.68,"se":3.29,"samplingSe":1.5705,"residualSe":2.8876,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.047194,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.0369,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:df5bf99d-d228-4715-b41b-19ca86118777","observedHarness":"Direct API","rawScore":65.53,"zScore":1.747104,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:df5bf99d-d228-4715-b41b-19ca86118777","observedHarness":"Stirrup","rawScore":67.5,"zScore":1.51835,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:df5bf99d-d228-4715-b41b-19ca86118777","observedHarness":"Direct API","rawScore":26.6,"zScore":0.886438,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:df5bf99d-d228-4715-b41b-19ca86118777","observedHarness":"Direct API","rawScore":64,"zScore":1.93277,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:df5bf99d-d228-4715-b41b-19ca86118777","observedHarness":"Direct API","rawScore":25,"zScore":0.446966,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Claude Opus 5.5 (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":47.58,"zScore":-0.400091,"seModel":"upstream_ci","ciHalfWidth":2.2,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-25","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:df5bf99d-d228-4715-b41b-19ca86118777","observedHarness":"mini-SWE-agent","rawScore":61.9,"zScore":1.855359,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"openai:gpt-6.2-sol:medium:direct:native","provider":"openai","modelSlug":"gpt-6.2-sol","displayName":"GPT-6.2 Sol","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.149149,"shrinkageFactor":0.984651,"score":67.24,"se":3.56,"samplingSe":1.7608,"residualSe":3.09,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.406056,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.8727,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:82b37079-56d9-4451-8a1a-22b0a921f4f8","observedHarness":"AA Analyst Harness","rawScore":50,"zScore":0.609202,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:82b37079-56d9-4451-8a1a-22b0a921f4f8","observedHarness":"Direct API","rawScore":59.17,"zScore":1.293118,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:82b37079-56d9-4451-8a1a-22b0a921f4f8","observedHarness":"Stirrup","rawScore":64,"zScore":1.337835,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:82b37079-56d9-4451-8a1a-22b0a921f4f8","observedHarness":"Direct API","rawScore":29.2,"zScore":1.193295,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:82b37079-56d9-4451-8a1a-22b0a921f4f8","observedHarness":"Direct API","rawScore":50.7,"zScore":1.207724,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:82b37079-56d9-4451-8a1a-22b0a921f4f8","observedHarness":"Direct API","rawScore":31.67,"zScore":1.192734,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:82b37079-56d9-4451-8a1a-22b0a921f4f8","observedHarness":"mini-SWE-agent","rawScore":60.48,"zScore":1.790209,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"anthropic:claude-sonnet-5.5:max:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.147487,"shrinkageFactor":0.992822,"score":67.21,"se":1.73,"samplingSe":1.2041,"residualSe":1.2458,"coverageCount":21,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":10.020621,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.0015,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:b171d979-5ec1-45de-b8b9-db1ee65e77ec","observedHarness":"AA Analyst Harness","rawScore":57.5,"zScore":1.155233,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_briefcase:b171d979-5ec1-45de-b8b9-db1ee65e77ec","observedHarness":"Stirrup","rawScore":1813.08,"zScore":1.756875,"seModel":"upstream_ci","ciHalfWidth":10.15,"ciLower":1803.54,"ciUpper":1823.84,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase"},{"benchmarkId":"aa_mka","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_mka:b171d979-5ec1-45de-b8b9-db1ee65e77ec","observedHarness":"Stirrup","rawScore":51.99,"zScore":1.302312,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/mka"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:b171d979-5ec1-45de-b8b9-db1ee65e77ec","observedHarness":"Direct API","rawScore":51.47,"zScore":0.743481,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"aa_vials","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_vials:b171d979-5ec1-45de-b8b9-db1ee65e77ec","observedHarness":"Stirrup","rawScore":59.63,"zScore":1.097888,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/vials-aa"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:b171d979-5ec1-45de-b8b9-db1ee65e77ec","observedHarness":"Stirrup","rawScore":59,"zScore":1.079956,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recwsmBXqLEgJjkrE","observedHarness":"APEX-Agents Harness","rawScore":75.5,"zScore":1.458782,"seModel":"upstream_se","seRaw":4.7,"ciHalfWidth":9.212,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:GtJ5yd3ZPQ7HVcm6Q9ssQz","observedHarness":"Direct API","rawScore":46.5,"zScore":-0.384825,"seModel":"upstream_se","seRaw":1.57805,"ciHalfWidth":3.093,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-29","publishedDate":"2026-09-29","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:b171d979-5ec1-45de-b8b9-db1ee65e77ec","observedHarness":"Direct API","rawScore":25.8,"zScore":0.79202,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdpval_aa","benchmarkVersion":"2.1","sourceId":"artificial-analysis","sourceRowId":"aa:gdpval_aa:b171d979-5ec1-45de-b8b9-db1ee65e77ec","observedHarness":"Stirrup","rawScore":1837.82,"zScore":1.350217,"seModel":"upstream_ci","ciHalfWidth":23.47,"ciLower":1814.35,"ciUpper":1861.29,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:b171d979-5ec1-45de-b8b9-db1ee65e77ec","observedHarness":"Direct API","rawScore":53.3,"zScore":1.349462,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:b171d979-5ec1-45de-b8b9-db1ee65e77ec","observedHarness":"Direct API","rawScore":41.67,"zScore":2.310826,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Claude Sonnet 5.5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":40.5,"zScore":1.705937,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Claude Sonnet 5.5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":34,"zScore":1.523784,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:Claude Sonnet 5.5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":26.8,"zScore":1.224807,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:Claude Sonnet 5.5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":29.1,"zScore":1.019355,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:Claude Sonnet 5.5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":76.8,"zScore":1.089272,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:b171d979-5ec1-45de-b8b9-db1ee65e77ec","observedHarness":"mini-SWE-agent","rawScore":53.33,"zScore":1.462163,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:anthropic/claude-sonnet-5-5","observedHarness":"Direct API","rawScore":75.9,"zScore":1.334498,"seModel":"upstream_se","seRaw":2.468,"ciHalfWidth":4.8373,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.526758,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:anthropic/claude-sonnet-5-5","observedHarness":"Direct API","rawScore":52.92,"zScore":1.046311,"seModel":"upstream_se","seRaw":2.119,"ciHalfWidth":4.1532,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.391592,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:anthropic/claude-sonnet-5-5","observedHarness":"Direct API","rawScore":49.1,"zScore":1.909266,"seModel":"upstream_se","seRaw":3.363,"ciHalfWidth":6.5915,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.984657,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"}]},{"id":"google:gemini-4-carbon:low:direct:native","provider":"google","modelSlug":"gemini-4-carbon","displayName":"Gemini 4 Carbon","effort":"low","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.110079,"shrinkageFactor":0.985273,"score":66.65,"se":3.75,"samplingSe":1.7247,"residualSe":3.3306,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":7.186577,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-10-12","paretoProb":0.0064,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:b3866703-a9f7-4ea2-8f3f-b361c90aaf0e","observedHarness":"Direct API","rawScore":49.23,"zScore":0.583586,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:b3866703-a9f7-4ea2-8f3f-b361c90aaf0e","observedHarness":"Stirrup","rawScore":65,"zScore":1.389411,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:b3866703-a9f7-4ea2-8f3f-b361c90aaf0e","observedHarness":"Direct API","rawScore":28.6,"zScore":1.122482,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:b3866703-a9f7-4ea2-8f3f-b361c90aaf0e","observedHarness":"Direct API","rawScore":61.9,"zScore":1.818289,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:b3866703-a9f7-4ea2-8f3f-b361c90aaf0e","observedHarness":"Direct API","rawScore":18.89,"zScore":-0.236188,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:b3866703-a9f7-4ea2-8f3f-b361c90aaf0e","observedHarness":"mini-SWE-agent","rawScore":61.9,"zScore":1.855359,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"anthropic:claude-opus-5.5:high:direct:native","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.105408,"shrinkageFactor":0.989181,"score":66.58,"se":3.27,"samplingSe":1.4783,"residualSe":2.9147,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.464662,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.0814,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:f314eade-2232-44bf-a3ab-9b326bc67de6","observedHarness":"Direct API","rawScore":64.83,"zScore":1.697137,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:f314eade-2232-44bf-a3ab-9b326bc67de6","observedHarness":"Stirrup","rawScore":59.5,"zScore":1.105744,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:f314eade-2232-44bf-a3ab-9b326bc67de6","observedHarness":"Direct API","rawScore":28.8,"zScore":1.146086,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:f314eade-2232-44bf-a3ab-9b326bc67de6","observedHarness":"Direct API","rawScore":55.6,"zScore":1.474846,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Claude Opus 5.5 (High)","observedHarness":"Direct API","rawScore":1504.85,"zScore":2.018284,"seModel":"upstream_ci","ciHalfWidth":4.8455,"ciLower":1500.0006,"ciUpper":1509.6917,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:f314eade-2232-44bf-a3ab-9b326bc67de6","observedHarness":"Direct API","rawScore":15,"zScore":-0.671126,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:f314eade-2232-44bf-a3ab-9b326bc67de6","observedHarness":"mini-SWE-agent","rawScore":49.05,"zScore":1.265794,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"openai:gpt-6.2-sol:xhigh:direct:native","provider":"openai","modelSlug":"gpt-6.2-sol","displayName":"GPT-6.2 Sol","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.085822,"shrinkageFactor":0.988706,"score":66.29,"se":3.1,"samplingSe":1.5104,"residualSe":2.7043,"coverageCount":8,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.647968,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.091,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:27c326a6-ec9b-415f-a29e-b1f4927484dc","observedHarness":"AA Analyst Harness","rawScore":53.75,"zScore":0.882217,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:27c326a6-ec9b-415f-a29e-b1f4927484dc","observedHarness":"Direct API","rawScore":60.7,"zScore":1.402332,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:27c326a6-ec9b-415f-a29e-b1f4927484dc","observedHarness":"Stirrup","rawScore":62.5,"zScore":1.260472,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:27c326a6-ec9b-415f-a29e-b1f4927484dc","observedHarness":"Direct API","rawScore":28.4,"zScore":1.098877,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:27c326a6-ec9b-415f-a29e-b1f4927484dc","observedHarness":"Direct API","rawScore":61.8,"zScore":1.812837,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:27c326a6-ec9b-415f-a29e-b1f4927484dc","observedHarness":"Direct API","rawScore":35,"zScore":1.565058,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:GPT 6.2 Sol (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":43.34,"zScore":-1.22661,"seModel":"upstream_ci","ciHalfWidth":0.93,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-18","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:27c326a6-ec9b-415f-a29e-b1f4927484dc","observedHarness":"mini-SWE-agent","rawScore":66.67,"zScore":2.074209,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"anthropic:claude-opus-5:max:direct:native","provider":"anthropic","modelSlug":"claude-opus-5","displayName":"Claude Opus 5","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.084698,"shrinkageFactor":0.990786,"score":66.27,"se":2.81,"samplingSe":1.3642,"residualSe":2.4538,"coverageCount":10,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":14.827231,"costLogSd":1.0558,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-07-28","paretoProb":0.1015,"benchmarks":[{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:rec1AxRjtBjTtTCHM","observedHarness":"APEX-Agents Harness","rawScore":65.8,"zScore":0.875532,"seModel":"upstream_se","seRaw":5.1,"ciHalfWidth":9.996,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:Zftz2yk4X5NF28JQzjN3hZ","observedHarness":"Direct API","rawScore":59.9,"zScore":0.568458,"seModel":"upstream_se","seRaw":1.550611,"ciHalfWidth":3.0392,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-10","publishedDate":"2026-08-10","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdpval_aa","benchmarkVersion":"2.1","sourceId":"artificial-analysis","sourceRowId":"aa:gdpval_aa:b8fc61f7-5e9a-49e6-8547-6ac56db24627","observedHarness":"Stirrup","rawScore":1729,"zScore":0.657699,"seModel":"upstream_ci","ciHalfWidth":18.175,"ciLower":1710.82,"ciUpper":1747.17,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Claude Opus 5 (Max)","observedHarness":"Direct API","rawScore":1487.79,"zScore":1.21595,"seModel":"upstream_ci","ciHalfWidth":3.7875,"ciLower":1484.004,"ciUpper":1491.579,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Claude Opus 5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":23.8,"zScore":0.432119,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Claude Opus 5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":22,"zScore":0.604414,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:anthropic/claude-opus-5","observedHarness":"Direct API","rawScore":55.29,"zScore":1.463688,"seModel":"upstream_se","seRaw":3.456,"ciHalfWidth":6.7738,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.762498,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:anthropic/claude-opus-5","observedHarness":"Direct API","rawScore":63.57,"zScore":2.601805,"seModel":"upstream_se","seRaw":1.993,"ciHalfWidth":3.9063,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.156845,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:anthropic/claude-opus-5","observedHarness":"Direct API","rawScore":37.39,"zScore":0.673047,"seModel":"upstream_se","seRaw":3.255,"ciHalfWidth":6.3798,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.280693,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:anthropic/claude-opus-5","observedHarness":"Vals Tax Agent","rawScore":47.1,"zScore":1.499569,"seModel":"upstream_se","seRaw":3.599,"ciHalfWidth":7.054,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.136554,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"google:gemini-4-argon:high:direct:native","provider":"google","modelSlug":"gemini-4-argon","displayName":"Gemini 4 Argon","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.036489,"shrinkageFactor":0.995428,"score":65.55,"se":1.06,"samplingSe":0.961,"residualSe":0.4581,"coverageCount":24,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":9.946609,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:d366dfa1-d940-4788-aa73-d298c45699b3","observedHarness":"AA Analyst Harness","rawScore":48.75,"zScore":0.518197,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_mka","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_mka:d366dfa1-d940-4788-aa73-d298c45699b3","observedHarness":"Stirrup","rawScore":51.26,"zScore":1.218267,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/mka"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:d366dfa1-d940-4788-aa73-d298c45699b3","observedHarness":"Direct API","rawScore":48.3,"zScore":0.517202,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"aa_vials","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_vials:d366dfa1-d940-4788-aa73-d298c45699b3","observedHarness":"Stirrup","rawScore":59.63,"zScore":1.097888,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/vials-aa"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:d366dfa1-d940-4788-aa73-d298c45699b3","observedHarness":"Stirrup","rawScore":52.5,"zScore":0.744714,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:iq4Y9tXpiyLP25yotySxJW","observedHarness":"Direct API","rawScore":76,"zScore":1.713821,"seModel":"upstream_se","seRaw":1.351231,"ciHalfWidth":2.6484,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-13","publishedDate":"2026-10-13","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:d366dfa1-d940-4788-aa73-d298c45699b3","observedHarness":"Direct API","rawScore":21.8,"zScore":0.319933,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:d366dfa1-d940-4788-aa73-d298c45699b3","observedHarness":"Direct API","rawScore":56.7,"zScore":1.534812,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Gemini 4 Argon (High)","observedHarness":"Direct API","rawScore":1492.26,"zScore":1.426175,"seModel":"upstream_ci","ciHalfWidth":4.3753,"ciLower":1487.8895,"ciUpper":1496.6401,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:d366dfa1-d940-4788-aa73-d298c45699b3","observedHarness":"Direct API","rawScore":13.89,"zScore":-0.795234,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:Gemini 4 Argon (one-pass-agent) high","observedHarness":"Scale one-pass-agent","rawScore":49.9,"zScore":0.402304,"seModel":"upstream_ci","ciHalfWidth":0.42,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-16","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Gemini 4 Argon (one-pass-agent) high","observedHarness":"Scale one-pass-agent","rawScore":49.41,"zScore":-0.043363,"seModel":"upstream_ci","ciHalfWidth":0.37,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-16","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Gemini 4 Argon (High reasoning)","observedHarness":"Direct API","rawScore":30.5,"zScore":0.943172,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Gemini 4 Argon (High reasoning)","observedHarness":"Direct API","rawScore":23.6,"zScore":0.726997,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:Gemini 4 Argon (High reasoning)","observedHarness":"Direct API","rawScore":5.9,"zScore":-0.001386,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:Gemini 4 Argon (High reasoning)","observedHarness":"Direct API","rawScore":38.3,"zScore":2.219278,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:Gemini 4 Argon (High reasoning)","observedHarness":"Direct API","rawScore":74.4,"zScore":0.96452,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:d366dfa1-d940-4788-aa73-d298c45699b3","observedHarness":"mini-SWE-agent","rawScore":52.86,"zScore":1.440599,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:google/gemini-4-argon","observedHarness":"Direct API","rawScore":74.83,"zScore":1.040895,"seModel":"upstream_se","seRaw":2.494,"ciHalfWidth":4.8882,"ciLower":null,"ciUpper":null,"measuredCostUsd":11.8029,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:google/gemini-4-argon","observedHarness":"Vals Finance Agent","rawScore":67.66,"zScore":1.364058,"seModel":"upstream_se","seRaw":2,"ciHalfWidth":3.92,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.315759,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:google/gemini-4-argon","observedHarness":"Direct API","rawScore":54.81,"zScore":1.421109,"seModel":"upstream_se","seRaw":3.459,"ciHalfWidth":6.7796,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.44908,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:google/gemini-4-argon","observedHarness":"Direct API","rawScore":58.8,"zScore":1.905119,"seModel":"upstream_se","seRaw":2.1,"ciHalfWidth":4.116,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.13716,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:google/gemini-4-argon","observedHarness":"Direct API","rawScore":45.49,"zScore":1.52816,"seModel":"upstream_se","seRaw":3.35,"ciHalfWidth":6.566,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.594426,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:google/gemini-4-argon","observedHarness":"Vals Tax Agent","rawScore":49.22,"zScore":1.686437,"seModel":"upstream_se","seRaw":3.608,"ciHalfWidth":7.0717,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.473645,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-6.2-astra:low:direct:native","provider":"openai","modelSlug":"gpt-6.2-astra","displayName":"GPT-6.2 Astra","effort":"low","harness":"direct","harnessRegime":"native","domainId":"research","theta":1.021849,"shrinkageFactor":0.984505,"score":65.33,"se":3.56,"samplingSe":1.7691,"residualSe":3.0896,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.661705,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.0965,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:cc93ecfd-845e-455d-9642-6799d7133b7e","observedHarness":"AA Analyst Harness","rawScore":56.25,"zScore":1.064228,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:cc93ecfd-845e-455d-9642-6799d7133b7e","observedHarness":"Direct API","rawScore":60.63,"zScore":1.397335,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:cc93ecfd-845e-455d-9642-6799d7133b7e","observedHarness":"Stirrup","rawScore":61,"zScore":1.183108,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:cc93ecfd-845e-455d-9642-6799d7133b7e","observedHarness":"Direct API","rawScore":28,"zScore":1.051669,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:cc93ecfd-845e-455d-9642-6799d7133b7e","observedHarness":"Direct API","rawScore":47.7,"zScore":1.044179,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:cc93ecfd-845e-455d-9642-6799d7133b7e","observedHarness":"Direct API","rawScore":30.56,"zScore":1.068625,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:cc93ecfd-845e-455d-9642-6799d7133b7e","observedHarness":"mini-SWE-agent","rawScore":50.95,"zScore":1.352967,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"anthropic:claude-opus-4.6:none:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.6","displayName":"Claude Opus 4.6","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.934062,"shrinkageFactor":0.99365,"score":64.01,"se":5.14,"samplingSe":1.1325,"residualSe":5.0138,"coverageCount":3,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":2.977031,"costLogSd":1.049,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-02-05","paretoProb":0.4783,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Claude Opus 4.6","observedHarness":"Direct API","rawScore":1488.23,"zScore":1.236644,"seModel":"upstream_ci","ciHalfWidth":2.5799,"ciLower":1485.6497,"ciUpper":1490.8094,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:Claude Opus 4.6 (one-pass-agent)","observedHarness":"Scale one-pass-agent","rawScore":53.28,"zScore":0.851086,"seModel":"upstream_ci","ciHalfWidth":0.1802,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-02-17","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Claude Opus 4.6 (one-pass-agent)","observedHarness":"Scale one-pass-agent","rawScore":52.27,"zScore":0.514148,"seModel":"upstream_ci","ciHalfWidth":0.6555,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-02-17","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"}]},{"id":"google:gemini-4-hydrogen:high:direct:native","provider":"google","modelSlug":"gemini-4-hydrogen","displayName":"Gemini 4 Hydrogen","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.933574,"shrinkageFactor":0.991794,"score":64,"se":2.63,"samplingSe":1.2875,"residualSe":2.2912,"coverageCount":9,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":1.873314,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":1.5,"outputPer1mUsd":7.5,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.6529,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:dae0262d-26a4-44d1-925a-6f2731cbda6a","observedHarness":"Direct API","rawScore":48.17,"zScore":0.507922,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:dae0262d-26a4-44d1-925a-6f2731cbda6a","observedHarness":"Stirrup","rawScore":61.5,"zScore":1.208896,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:dae0262d-26a4-44d1-925a-6f2731cbda6a","observedHarness":"Direct API","rawScore":30,"zScore":1.287712,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:dae0262d-26a4-44d1-925a-6f2731cbda6a","observedHarness":"Direct API","rawScore":54.9,"zScore":1.436686,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Gemini 4 Hydrogen (High)","observedHarness":"Direct API","rawScore":1493.59,"zScore":1.488725,"seModel":"upstream_ci","ciHalfWidth":7.6107,"ciLower":1485.9758,"ciUpper":1501.1972,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:dae0262d-26a4-44d1-925a-6f2731cbda6a","observedHarness":"Direct API","rawScore":27.22,"zScore":0.695183,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:Gemini 4 Hydrogen (one-pass-agent) high","observedHarness":"Scale one-pass-agent","rawScore":52.29,"zScore":0.719638,"seModel":"upstream_ci","ciHalfWidth":0.11,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-19","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Gemini 4 Hydrogen (one-pass-agent) high","observedHarness":"Scale one-pass-agent","rawScore":53.09,"zScore":0.673993,"seModel":"upstream_ci","ciHalfWidth":0.47,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-17","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:dae0262d-26a4-44d1-925a-6f2731cbda6a","observedHarness":"mini-SWE-agent","rawScore":41.43,"zScore":0.916184,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"openai:gpt-5.5:high:direct:native","provider":"openai","modelSlug":"gpt-5.5","displayName":"GPT-5.5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.92673,"shrinkageFactor":0.994722,"score":63.9,"se":9.25,"samplingSe":1.0325,"residualSe":9.1937,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":9.848117,"costLogSd":1.0508,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":11.25,"inputPer1mUsd":5,"outputPer1mUsd":30,"contextWindow":1050000,"releaseDate":"2026-04-23","paretoProb":0.1059,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-5.5 (High)","observedHarness":"Direct API","rawScore":1485.2,"zScore":1.094142,"seModel":"upstream_ci","ciHalfWidth":2.8763,"ciLower":1482.3287,"ciUpper":1488.0812,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-fable-5:high:direct:native","provider":"anthropic","modelSlug":"claude-fable-5","displayName":"Claude Fable 5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.873194,"shrinkageFactor":0.993003,"score":63.1,"se":4.61,"samplingSe":1.1888,"residualSe":4.4527,"coverageCount":4,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":15.786777,"costLogSd":1.0567,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-06-09","paretoProb":0.0611,"benchmarks":[{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:reckaD0wz9XOvArFA","observedHarness":"APEX-Agents Harness","rawScore":63.6,"zScore":0.743248,"seModel":"upstream_se","seRaw":5.3,"ciHalfWidth":10.388,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Claude Fable 5 (High)","observedHarness":"Direct API","rawScore":1483.28,"zScore":1.003844,"seModel":"upstream_ci","ciHalfWidth":3.3189,"ciLower":1479.9612,"ciUpper":1486.5991,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:Claude Fable 5 (one-pass-agent)","observedHarness":"Scale one-pass-agent","rawScore":53.86,"zScore":0.928096,"seModel":"upstream_ci","ciHalfWidth":0.14,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-07-28","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Claude Fable 5 (one-pass-agent)","observedHarness":"Scale one-pass-agent","rawScore":52.56,"zScore":0.570678,"seModel":"upstream_ci","ciHalfWidth":0.54,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-07-28","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"}]},{"id":"openai:gpt-6.2-sol:max:direct:native","provider":"openai","modelSlug":"gpt-6.2-sol","displayName":"GPT-6.2 Sol","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.862616,"shrinkageFactor":0.993433,"score":62.94,"se":1.62,"samplingSe":1.1517,"residualSe":1.1463,"coverageCount":21,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.01438,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.0018,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:a1a84b18-e8c9-48fb-8e04-6003bd9f6ff4","observedHarness":"AA Analyst Harness","rawScore":53.75,"zScore":0.882217,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_briefcase:a1a84b18-e8c9-48fb-8e04-6003bd9f6ff4","observedHarness":"Stirrup","rawScore":1611.82,"zScore":0.423432,"seModel":"upstream_ci","ciHalfWidth":11.715,"ciLower":1600.86,"ciUpper":1624.29,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:a1a84b18-e8c9-48fb-8e04-6003bd9f6ff4","observedHarness":"Direct API","rawScore":61.63,"zScore":1.468717,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"aa_vials","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_vials:a1a84b18-e8c9-48fb-8e04-6003bd9f6ff4","observedHarness":"Stirrup","rawScore":62.73,"zScore":1.302772,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/vials-aa"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:a1a84b18-e8c9-48fb-8e04-6003bd9f6ff4","observedHarness":"Stirrup","rawScore":62.5,"zScore":1.260472,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:rec6FHye6omYQVfM8","observedHarness":"APEX-Agents Harness","rawScore":66.6,"zScore":0.923635,"seModel":"upstream_se","seRaw":5.3,"ciHalfWidth":10.388,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:a1a84b18-e8c9-48fb-8e04-6003bd9f6ff4","observedHarness":"Direct API","rawScore":30,"zScore":1.287712,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:a1a84b18-e8c9-48fb-8e04-6003bd9f6ff4","observedHarness":"Direct API","rawScore":64,"zScore":1.93277,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-6.2 Sol (Max)","observedHarness":"Direct API","rawScore":1443.28,"zScore":-0.877361,"seModel":"upstream_ci","ciHalfWidth":10.1437,"ciLower":1433.1405,"ciUpper":1453.428,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:a1a84b18-e8c9-48fb-8e04-6003bd9f6ff4","observedHarness":"Direct API","rawScore":37.22,"zScore":1.813275,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:GPT 6.2 Sol (Max reasoning)","observedHarness":"Direct API","rawScore":32.8,"zScore":1.118608,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:GPT 6.2 Sol (Max reasoning)","observedHarness":"Direct API","rawScore":17.2,"zScore":0.236666,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:GPT 6.2 Sol (Max reasoning)","observedHarness":"Direct API","rawScore":32.7,"zScore":1.570957,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:GPT 6.2 Sol (Max reasoning)","observedHarness":"Direct API","rawScore":22.3,"zScore":0.132455,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:GPT 6.2 Sol (Max reasoning)","observedHarness":"Direct API","rawScore":80,"zScore":1.255608,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:a1a84b18-e8c9-48fb-8e04-6003bd9f6ff4","observedHarness":"mini-SWE-agent","rawScore":67.62,"zScore":2.117796,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:openai/gpt-6.2-sol","observedHarness":"Direct API","rawScore":74.29,"zScore":0.892721,"seModel":"upstream_se","seRaw":2.247,"ciHalfWidth":4.4041,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.840823,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:openai/gpt-6.2-sol","observedHarness":"Vals Finance Agent","rawScore":58.91,"zScore":-1.142523,"seModel":"upstream_se","seRaw":2.096,"ciHalfWidth":4.1082,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.062252,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:openai/gpt-6.2-sol","observedHarness":"Direct API","rawScore":37.02,"zScore":-0.156956,"seModel":"upstream_se","seRaw":3.356,"ciHalfWidth":6.5778,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.766098,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:openai/gpt-6.2-sol","observedHarness":"Direct API","rawScore":36.94,"zScore":0.62554,"seModel":"upstream_se","seRaw":3.247,"ciHalfWidth":6.3641,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.486648,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:openai/gpt-6.2-sol","observedHarness":"Vals Tax Agent","rawScore":24.84,"zScore":-0.462542,"seModel":"upstream_se","seRaw":3.116,"ciHalfWidth":6.1074,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.401632,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"anthropic:claude-opus-5.5:medium:direct:native","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.795928,"shrinkageFactor":0.985886,"score":61.94,"se":3.53,"samplingSe":1.6885,"residualSe":3.0946,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.451089,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.2222,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:ea9e0da6-169f-43c6-92f5-730a10d6ff8d","observedHarness":"Direct API","rawScore":63.9,"zScore":1.630753,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:ea9e0da6-169f-43c6-92f5-730a10d6ff8d","observedHarness":"Stirrup","rawScore":54,"zScore":0.822078,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recjdN7zdajrdYE2h","observedHarness":"APEX-Agents Harness","rawScore":52.5,"zScore":0.075817,"seModel":"upstream_se","seRaw":5.6,"ciHalfWidth":10.976,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:ea9e0da6-169f-43c6-92f5-730a10d6ff8d","observedHarness":"Direct API","rawScore":25.6,"zScore":0.768416,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:ea9e0da6-169f-43c6-92f5-730a10d6ff8d","observedHarness":"Direct API","rawScore":51.4,"zScore":1.245884,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:ea9e0da6-169f-43c6-92f5-730a10d6ff8d","observedHarness":"Direct API","rawScore":22.22,"zScore":0.136136,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:ea9e0da6-169f-43c6-92f5-730a10d6ff8d","observedHarness":"mini-SWE-agent","rawScore":43.33,"zScore":1.003357,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"google:gemini-4-neon:low:direct:native","provider":"google","modelSlug":"gemini-4-neon","displayName":"Gemini 4 Neon","effort":"low","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.775764,"shrinkageFactor":0.985776,"score":61.64,"se":3.54,"samplingSe":1.695,"residualSe":3.1035,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.745228,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1048576,"releaseDate":"2026-11-18","paretoProb":0.9014,"benchmarks":[{"benchmarkId":"aa_mka","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_mka:09fea5f5-7217-452d-b693-fcca01e07bdf","observedHarness":"Stirrup","rawScore":43.35,"zScore":0.307587,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/mka"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:09fea5f5-7217-452d-b693-fcca01e07bdf","observedHarness":"Direct API","rawScore":56.63,"zScore":1.111809,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:09fea5f5-7217-452d-b693-fcca01e07bdf","observedHarness":"Stirrup","rawScore":62,"zScore":1.234684,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:09fea5f5-7217-452d-b693-fcca01e07bdf","observedHarness":"Direct API","rawScore":31.4,"zScore":1.452943,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:09fea5f5-7217-452d-b693-fcca01e07bdf","observedHarness":"Direct API","rawScore":54.2,"zScore":1.398525,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:09fea5f5-7217-452d-b693-fcca01e07bdf","observedHarness":"Direct API","rawScore":26.11,"zScore":0.571074,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:09fea5f5-7217-452d-b693-fcca01e07bdf","observedHarness":"mini-SWE-agent","rawScore":27.14,"zScore":0.260551,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"meta:muse-spark-1.3:max:direct:native","provider":"meta","modelSlug":"muse-spark-1.3","displayName":"Muse Spark 1.3","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.748081,"shrinkageFactor":0.987718,"score":61.22,"se":3.06,"samplingSe":1.575,"residualSe":2.6275,"coverageCount":10,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.092883,"costLogSd":1.0457,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.3694,"benchmarks":[{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recXul8NIwqqTNIrA","observedHarness":"APEX-Agents Harness","rawScore":47.6,"zScore":-0.218814,"seModel":"upstream_se","seRaw":5.7,"ciHalfWidth":11.172,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"gdpval_aa","benchmarkVersion":"2.1","sourceId":"artificial-analysis","sourceRowId":"aa:gdpval_aa:9999672c-b687-4026-9c15-5cef02ff53bb","observedHarness":"Stirrup","rawScore":1680.99,"zScore":0.352169,"seModel":"upstream_ci","ciHalfWidth":19.225,"ciLower":1661.76,"ciUpper":1700.21,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Muse Spark 1.3 (Max)","observedHarness":"Direct API","rawScore":1481.37,"zScore":0.914017,"seModel":"upstream_ci","ciHalfWidth":4.091,"ciLower":1477.282,"ciUpper":1485.464,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Muse Spark 1.3 (Max reasoning)","observedHarness":"Direct API","rawScore":27,"zScore":0.676204,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Muse Spark 1.3 (Max reasoning)","observedHarness":"Direct API","rawScore":22,"zScore":0.604414,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:Muse Spark 1.3 (Max reasoning)","observedHarness":"Direct API","rawScore":11.8,"zScore":0.344764,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:Muse Spark 1.3 (Max reasoning)","observedHarness":"Direct API","rawScore":22.6,"zScore":0.171583,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:meta/muse_spark_1_3_max","observedHarness":"Direct API","rawScore":55.29,"zScore":1.463688,"seModel":"upstream_se","seRaw":3.456,"ciHalfWidth":6.7738,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.588812,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:meta/muse_spark_1_3_max","observedHarness":"Direct API","rawScore":36.04,"zScore":0.530528,"seModel":"upstream_se","seRaw":3.23,"ciHalfWidth":6.3308,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.851846,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:meta/muse_spark_1_3_max","observedHarness":"Vals Tax Agent","rawScore":42.44,"zScore":1.088813,"seModel":"upstream_se","seRaw":3.511,"ciHalfWidth":6.8816,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.389917,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-5.5:medium:direct:native","provider":"openai","modelSlug":"gpt-5.5","displayName":"GPT-5.5","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.74767,"shrinkageFactor":0.994842,"score":61.22,"se":9.25,"samplingSe":1.0207,"residualSe":9.1948,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":5.234615,"costLogSd":1.0599,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":11.25,"inputPer1mUsd":5,"outputPer1mUsd":30,"contextWindow":1050000,"releaseDate":"2026-04-23","paretoProb":0.0943,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-5.5","observedHarness":"Direct API","rawScore":1481.37,"zScore":0.914017,"seModel":"upstream_ci","ciHalfWidth":2.8433,"ciLower":1478.5298,"ciUpper":1484.2164,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-6-astra:max:direct:native","provider":"openai","modelSlug":"gpt-6-astra","displayName":"GPT-6 Astra","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.717728,"shrinkageFactor":0.989811,"score":60.77,"se":2.54,"samplingSe":1.4346,"residualSe":2.0905,"coverageCount":13,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":27.818269,"costLogSd":1.0653,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0001,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:PppwnX7EjXx2SeTcFN8Ujn","observedHarness":"Direct API","rawScore":75.6,"zScore":1.685364,"seModel":"upstream_se","seRaw":1.358855,"ciHalfWidth":2.6634,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-30","publishedDate":"2026-08-30","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-6 Astra (Max)","observedHarness":"Direct API","rawScore":1467.85,"zScore":0.278169,"seModel":"upstream_ci","ciHalfWidth":4.4555,"ciLower":1463.3948,"ciUpper":1472.3058,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:GPT 6 Astra (Max reasoning)","observedHarness":"Direct API","rawScore":37.3,"zScore":1.461852,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:GPT 6 Astra (Max reasoning)","observedHarness":"Direct API","rawScore":28.4,"zScore":1.094745,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:GPT 6 Astra (Max reasoning)","observedHarness":"Direct API","rawScore":36.6,"zScore":1.799768,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:GPT 6 Astra (Max reasoning)","observedHarness":"Direct API","rawScore":22.9,"zScore":0.210711,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:GPT 6 Astra (Max reasoning)","observedHarness":"Direct API","rawScore":81.6,"zScore":1.338777,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:openai/gpt-6-astra","observedHarness":"Direct API","rawScore":72.68,"zScore":0.450944,"seModel":"upstream_se","seRaw":2.297,"ciHalfWidth":4.5021,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.827063,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:openai/gpt-6-astra","observedHarness":"Vals Finance Agent","rawScore":60.43,"zScore":-0.707094,"seModel":"upstream_se","seRaw":2.12,"ciHalfWidth":4.1552,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.823464,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:openai/gpt-6-astra","observedHarness":"Direct API","rawScore":39.42,"zScore":0.055937,"seModel":"upstream_se","seRaw":3.397,"ciHalfWidth":6.6581,"ciLower":null,"ciUpper":null,"measuredCostUsd":10.577143,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:openai/gpt-6-astra","observedHarness":"Direct API","rawScore":48.49,"zScore":0.399284,"seModel":"upstream_se","seRaw":2.131,"ciHalfWidth":4.1768,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.451358,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:openai/gpt-6-astra","observedHarness":"Direct API","rawScore":53.15,"zScore":2.336823,"seModel":"upstream_se","seRaw":3.357,"ciHalfWidth":6.5797,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.559704,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:openai/gpt-6-astra","observedHarness":"Vals Tax Agent","rawScore":20.72,"zScore":-0.825701,"seModel":"upstream_se","seRaw":2.924,"ciHalfWidth":5.731,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.7963,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"google:gemini-4-argon:medium:direct:native","provider":"google","modelSlug":"gemini-4-argon","displayName":"Gemini 4 Argon","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.659462,"shrinkageFactor":0.986212,"score":59.89,"se":4.06,"samplingSe":1.6688,"residualSe":3.7049,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.629243,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:48e6f735-f2bc-418f-b99f-3dc5a0ddda77","observedHarness":"Direct API","rawScore":48.97,"zScore":0.565027,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:48e6f735-f2bc-418f-b99f-3dc5a0ddda77","observedHarness":"Stirrup","rawScore":53.5,"zScore":0.79629,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:48e6f735-f2bc-418f-b99f-3dc5a0ddda77","observedHarness":"Direct API","rawScore":25.6,"zScore":0.768416,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:48e6f735-f2bc-418f-b99f-3dc5a0ddda77","observedHarness":"Direct API","rawScore":42.5,"zScore":0.760703,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:48e6f735-f2bc-418f-b99f-3dc5a0ddda77","observedHarness":"mini-SWE-agent","rawScore":48.57,"zScore":1.243771,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"anthropic:claude-fable-5.1:xhigh:direct:native","provider":"anthropic","modelSlug":"claude-fable-5.1","displayName":"Claude Fable 5.1","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.64567,"shrinkageFactor":0.98879,"score":59.69,"se":6.53,"samplingSe":1.5047,"residualSe":6.3508,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":19.394239,"costLogSd":1.0664,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-09-12","paretoProb":0.0354,"benchmarks":[{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:Claude Fable 5.1 (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":50.8,"zScore":0.521802,"seModel":"upstream_ci","ciHalfWidth":0.22,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-03","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Claude Fable 5.1 (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":51.6,"zScore":0.383542,"seModel":"upstream_ci","ciHalfWidth":0.24,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-03","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"}]},{"id":"anthropic:claude-sonnet-5.5:xhigh:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.644999,"shrinkageFactor":0.988586,"score":59.67,"se":3.28,"samplingSe":1.5184,"residualSe":2.9129,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.091985,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.0002,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:975aeaef-50bd-4a65-9c73-f1015adcd4d4","observedHarness":"Direct API","rawScore":50.9,"zScore":0.702793,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:975aeaef-50bd-4a65-9c73-f1015adcd4d4","observedHarness":"Stirrup","rawScore":57.5,"zScore":1.002593,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:975aeaef-50bd-4a65-9c73-f1015adcd4d4","observedHarness":"Direct API","rawScore":24.6,"zScore":0.650394,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:975aeaef-50bd-4a65-9c73-f1015adcd4d4","observedHarness":"Direct API","rawScore":44.1,"zScore":0.847926,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Claude Sonnet 5.5 (xHigh)","observedHarness":"Direct API","rawScore":1473.27,"zScore":0.533073,"seModel":"upstream_ci","ciHalfWidth":4.8377,"ciLower":1468.4333,"ciUpper":1478.1088,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:975aeaef-50bd-4a65-9c73-f1015adcd4d4","observedHarness":"Direct API","rawScore":20.56,"zScore":-0.049467,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:975aeaef-50bd-4a65-9c73-f1015adcd4d4","observedHarness":"mini-SWE-agent","rawScore":52.38,"zScore":1.418576,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"meta:muse-wonder:max:direct:native","provider":"meta","modelSlug":"muse-wonder","displayName":"Muse Wonder","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.642421,"shrinkageFactor":0.993729,"score":59.64,"se":1.56,"samplingSe":1.1254,"residualSe":1.0799,"coverageCount":22,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.952036,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.0005,"benchmarks":[{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_briefcase:ff2f3f3d-783f-4e27-8a39-47237501a313","observedHarness":"Stirrup","rawScore":1658.65,"zScore":0.733703,"seModel":"upstream_ci","ciHalfWidth":10.17,"ciLower":1648.48,"ciUpper":1668.82,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase"},{"benchmarkId":"aa_mka","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_mka:ff2f3f3d-783f-4e27-8a39-47237501a313","observedHarness":"Stirrup","rawScore":47.49,"zScore":0.784226,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/mka"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:ff2f3f3d-783f-4e27-8a39-47237501a313","observedHarness":"Direct API","rawScore":38.03,"zScore":-0.215886,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"aa_vials","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_vials:ff2f3f3d-783f-4e27-8a39-47237501a313","observedHarness":"Stirrup","rawScore":55.28,"zScore":0.810389,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/vials-aa"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:ff2f3f3d-783f-4e27-8a39-47237501a313","observedHarness":"Stirrup","rawScore":50.5,"zScore":0.641562,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recJ1sZSqwFwfdlZ4","observedHarness":"APEX-Agents Harness","rawScore":66.1,"zScore":0.89357,"seModel":"upstream_se","seRaw":5.3,"ciHalfWidth":10.388,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:ff2f3f3d-783f-4e27-8a39-47237501a313","observedHarness":"Direct API","rawScore":30.6,"zScore":1.358525,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdpval_aa","benchmarkVersion":"2.1","sourceId":"artificial-analysis","sourceRowId":"aa:gdpval_aa:ff2f3f3d-783f-4e27-8a39-47237501a313","observedHarness":"Stirrup","rawScore":1770.34,"zScore":0.920782,"seModel":"upstream_ci","ciHalfWidth":23.13,"ciLower":1747.21,"ciUpper":1793.47,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:ff2f3f3d-783f-4e27-8a39-47237501a313","observedHarness":"Direct API","rawScore":39,"zScore":0.569901,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Muse Wonder (Max)","observedHarness":"Direct API","rawScore":1493.67,"zScore":1.492487,"seModel":"upstream_ci","ciHalfWidth":4.955,"ciLower":1488.7165,"ciUpper":1498.6265,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:ff2f3f3d-783f-4e27-8a39-47237501a313","observedHarness":"Direct API","rawScore":22.22,"zScore":0.136136,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Muse Wonder 1.0 (Max reasoning)","observedHarness":"Direct API","rawScore":38.8,"zScore":1.576267,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Muse Wonder 1.0 (Max reasoning)","observedHarness":"Direct API","rawScore":31.2,"zScore":1.309264,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:Muse Wonder 1.0 (Max reasoning)","observedHarness":"Direct API","rawScore":20.6,"zScore":0.861056,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:Muse Wonder 1.0 (Max reasoning)","observedHarness":"Direct API","rawScore":21.8,"zScore":0.067242,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:Muse Wonder 1.0 (Max reasoning)","observedHarness":"Direct API","rawScore":59.2,"zScore":0.174422,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:ff2f3f3d-783f-4e27-8a39-47237501a313","observedHarness":"mini-SWE-agent","rawScore":25.71,"zScore":0.194942,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:meta/muse-wonder","observedHarness":"Direct API","rawScore":72.5,"zScore":0.401553,"seModel":"upstream_se","seRaw":2.414,"ciHalfWidth":4.7314,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.599337,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:meta/muse-wonder","observedHarness":"Vals Finance Agent","rawScore":65.55,"zScore":0.759613,"seModel":"upstream_se","seRaw":2.031,"ciHalfWidth":3.9808,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.620705,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:meta/muse-wonder","observedHarness":"Direct API","rawScore":50.48,"zScore":1.037016,"seModel":"upstream_se","seRaw":3.475,"ciHalfWidth":6.811,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.980242,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:meta/muse-wonder","observedHarness":"Direct API","rawScore":27.48,"zScore":-0.373148,"seModel":"upstream_se","seRaw":3.003,"ciHalfWidth":5.8859,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.389451,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:meta/muse-wonder","observedHarness":"Vals Tax Agent","rawScore":36.27,"zScore":0.544957,"seModel":"upstream_se","seRaw":3.47,"ciHalfWidth":6.8012,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.884439,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-6.1-terra:xhigh:direct:native","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.639352,"shrinkageFactor":0.986492,"score":59.59,"se":4.06,"samplingSe":1.6518,"residualSe":3.7059,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.400689,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.4413,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:e8d443ba-8aca-4830-a186-ae869bb9c6fe","observedHarness":"Direct API","rawScore":48.03,"zScore":0.497929,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:e8d443ba-8aca-4830-a186-ae869bb9c6fe","observedHarness":"Stirrup","rawScore":60.5,"zScore":1.15732,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:e8d443ba-8aca-4830-a186-ae869bb9c6fe","observedHarness":"Direct API","rawScore":26.6,"zScore":0.886438,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:e8d443ba-8aca-4830-a186-ae869bb9c6fe","observedHarness":"Direct API","rawScore":42.3,"zScore":0.7498,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:e8d443ba-8aca-4830-a186-ae869bb9c6fe","observedHarness":"mini-SWE-agent","rawScore":41.9,"zScore":0.937748,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"openai:gpt-6.1-sol:max:direct:native","provider":"openai","modelSlug":"gpt-6.1-sol","displayName":"GPT-6.1 Sol","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.605716,"shrinkageFactor":0.990182,"score":59.09,"se":2.45,"samplingSe":1.4082,"residualSe":2.0024,"coverageCount":14,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.487251,"costLogSd":1.048,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.0069,"benchmarks":[{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:reciOAOiuYHbbtCcU","observedHarness":"APEX-Agents Harness","rawScore":60,"zScore":0.526784,"seModel":"upstream_se","seRaw":5.6,"ciHalfWidth":10.976,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:dfT5VD7cQug8XW6GQHyJUF","observedHarness":"Direct API","rawScore":73.9,"zScore":1.564425,"seModel":"upstream_se","seRaw":1.389504,"ciHalfWidth":2.7234,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-29","publishedDate":"2026-09-29","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-6.1 Sol (Max)","observedHarness":"Direct API","rawScore":1469.83,"zScore":0.371289,"seModel":"upstream_ci","ciHalfWidth":4.9724,"ciLower":1464.8602,"ciUpper":1474.8049,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:GPT 6.1 Sol (Max reasoning)","observedHarness":"Direct API","rawScore":33.5,"zScore":1.172001,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:GPT 6.1 Sol (Max reasoning)","observedHarness":"Direct API","rawScore":25.2,"zScore":0.849579,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:GPT 6.1 Sol (Max reasoning)","observedHarness":"Direct API","rawScore":31.6,"zScore":1.506421,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:GPT 6.1 Sol (Max reasoning)","observedHarness":"Direct API","rawScore":22,"zScore":0.093327,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:GPT 6.1 Sol (Max reasoning)","observedHarness":"Direct API","rawScore":79.2,"zScore":1.214024,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:openai/gpt-6.1-sol","observedHarness":"Direct API","rawScore":71.09,"zScore":0.014655,"seModel":"upstream_se","seRaw":2.748,"ciHalfWidth":5.3861,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.143199,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:openai/gpt-6.1-sol","observedHarness":"Vals Finance Agent","rawScore":61.82,"zScore":-0.308906,"seModel":"upstream_se","seRaw":2.112,"ciHalfWidth":4.1395,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.626543,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:openai/gpt-6.1-sol","observedHarness":"Direct API","rawScore":38.46,"zScore":-0.02922,"seModel":"upstream_se","seRaw":3.381,"ciHalfWidth":6.6268,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.384561,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:openai/gpt-6.1-sol","observedHarness":"Direct API","rawScore":48.84,"zScore":0.450404,"seModel":"upstream_se","seRaw":2.12,"ciHalfWidth":4.1552,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.084726,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:openai/gpt-6.1-sol","observedHarness":"Direct API","rawScore":46.4,"zScore":1.624228,"seModel":"upstream_se","seRaw":3.355,"ciHalfWidth":6.5758,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.284881,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:openai/gpt-6.1-sol","observedHarness":"Vals Tax Agent","rawScore":21.75,"zScore":-0.734911,"seModel":"upstream_se","seRaw":2.977,"ciHalfWidth":5.8349,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.323601,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"anthropic:claude-opus-4.7:high:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.7","displayName":"Claude Opus 4.7","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.59503,"shrinkageFactor":0.994123,"score":58.93,"se":9.25,"samplingSe":1.0895,"residualSe":9.1881,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.432844,"costLogSd":1.0492,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-04-16","paretoProb":0.1276,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Claude Opus 4.7 (High)","observedHarness":"Direct API","rawScore":1478.12,"zScore":0.761169,"seModel":"upstream_ci","ciHalfWidth":3.036,"ciLower":1475.0841,"ciUpper":1481.1561,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"meta:muse-spark-1.1:high:direct:native","provider":"meta","modelSlug":"muse-spark-1.1","displayName":"Muse Spark 1.1","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.585295,"shrinkageFactor":0.995143,"score":58.78,"se":3.92,"samplingSe":0.9905,"residualSe":3.7973,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.768757,"costLogSd":1.0444,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.4441,"benchmarks":[{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recf1h0VzvdEIHDhj","observedHarness":"APEX-Agents Harness","rawScore":31.8,"zScore":-1.168851,"seModel":"upstream_se","seRaw":5.1,"ciHalfWidth":9.996,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:bCdwpRXgZtvQLYC4wA9JXr","observedHarness":"Direct API","rawScore":57.79,"zScore":0.418351,"seModel":"upstream_se","seRaw":1.566543,"ciHalfWidth":3.0704,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-31","publishedDate":"2026-08-31","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Muse Spark 1.1","observedHarness":"Direct API","rawScore":1463.64,"zScore":0.080173,"seModel":"upstream_ci","ciHalfWidth":3.5007,"ciLower":1460.1428,"ciUpper":1467.1441,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:Muse Spark 1.1 (one-pass-agent)","observedHarness":"Scale one-pass-agent","rawScore":55.01,"zScore":1.080788,"seModel":"upstream_ci","ciHalfWidth":0.14,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-07-09","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Muse Spark 1.1 (one-pass-agent)","observedHarness":"Scale one-pass-agent","rawScore":57.05,"zScore":1.44593,"seModel":"upstream_ci","ciHalfWidth":0.22,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-07-09","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"}]},{"id":"anthropic:claude-fable-5:max:direct:native","provider":"anthropic","modelSlug":"claude-fable-5","displayName":"Claude Fable 5","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.57607,"shrinkageFactor":0.969845,"score":58.64,"se":5.01,"samplingSe":2.468,"residualSe":4.3625,"coverageCount":4,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":27.818269,"costLogSd":1.0653,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-06-09","paretoProb":0.012,"benchmarks":[{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Claude Fable 5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":18.3,"zScore":0.012598,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Claude Fable 5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":11.2,"zScore":-0.223019,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:anthropic/claude-fable-5","observedHarness":"Direct API","rawScore":56.07,"zScore":1.506387,"seModel":"upstream_se","seRaw":2.203,"ciHalfWidth":4.3179,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.591071,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:anthropic/claude-fable-5","observedHarness":"Vals Tax Agent","rawScore":38.28,"zScore":0.722129,"seModel":"upstream_se","seRaw":3.503,"ciHalfWidth":6.8659,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.696087,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"anthropic:claude-opus-5:high:direct:native","provider":"anthropic","modelSlug":"claude-opus-5","displayName":"Claude Opus 5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.572536,"shrinkageFactor":0.994652,"score":58.59,"se":9.25,"samplingSe":1.0393,"residualSe":9.193,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.432844,"costLogSd":1.0492,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-07-28","paretoProb":0.1301,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Claude Opus 5 (High)","observedHarness":"Direct API","rawScore":1477.63,"zScore":0.738124,"seModel":"upstream_ci","ciHalfWidth":2.8954,"ciLower":1474.7323,"ciUpper":1480.5231,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-6.2-sol:low:direct:native","provider":"openai","modelSlug":"gpt-6.2-sol","displayName":"GPT-6.2 Sol","effort":"low","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.556648,"shrinkageFactor":0.984627,"score":58.35,"se":3.56,"samplingSe":1.7622,"residualSe":3.0899,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.183585,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.7797,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:a77965fb-2117-4e2c-bd8b-80b5579a7e7e","observedHarness":"AA Analyst Harness","rawScore":38.75,"zScore":-0.209844,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:a77965fb-2117-4e2c-bd8b-80b5579a7e7e","observedHarness":"Direct API","rawScore":54.63,"zScore":0.969046,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:a77965fb-2117-4e2c-bd8b-80b5579a7e7e","observedHarness":"Stirrup","rawScore":58,"zScore":1.028381,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:a77965fb-2117-4e2c-bd8b-80b5579a7e7e","observedHarness":"Direct API","rawScore":28.6,"zScore":1.122482,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:a77965fb-2117-4e2c-bd8b-80b5579a7e7e","observedHarness":"Direct API","rawScore":34.5,"zScore":0.324585,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:a77965fb-2117-4e2c-bd8b-80b5579a7e7e","observedHarness":"Direct API","rawScore":32.22,"zScore":1.254229,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:a77965fb-2117-4e2c-bd8b-80b5579a7e7e","observedHarness":"mini-SWE-agent","rawScore":37.14,"zScore":0.719356,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"alibaba:qwen4-max:xhigh:direct:native","provider":"alibaba","modelSlug":"qwen4-max","displayName":"Qwen4 Max","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.535547,"shrinkageFactor":0.991877,"score":58.03,"se":1.99,"samplingSe":1.2809,"residualSe":1.5181,"coverageCount":17,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":3.268133,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2.25,"inputPer1mUsd":1.5,"outputPer1mUsd":4.5,"contextWindow":1000000,"releaseDate":"2026-10-22","paretoProb":0.6304,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:fef6625d-9097-45f6-95c0-bcaa06e3d8e1","observedHarness":"AA Analyst Harness","rawScore":46.25,"zScore":0.336187,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_briefcase:fef6625d-9097-45f6-95c0-bcaa06e3d8e1","observedHarness":"Stirrup","rawScore":1686.07,"zScore":0.915374,"seModel":"upstream_ci","ciHalfWidth":10.48,"ciLower":1675.59,"ciUpper":1696.55,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:fef6625d-9097-45f6-95c0-bcaa06e3d8e1","observedHarness":"Direct API","rawScore":43.7,"zScore":0.188847,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:fef6625d-9097-45f6-95c0-bcaa06e3d8e1","observedHarness":"Stirrup","rawScore":43,"zScore":0.254744,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:fef6625d-9097-45f6-95c0-bcaa06e3d8e1","observedHarness":"Direct API","rawScore":26.2,"zScore":0.839229,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdpval_aa","benchmarkVersion":"2.1","sourceId":"artificial-analysis","sourceRowId":"aa:gdpval_aa:fef6625d-9097-45f6-95c0-bcaa06e3d8e1","observedHarness":"Stirrup","rawScore":1808.69,"zScore":1.164837,"seModel":"upstream_ci","ciHalfWidth":20.825,"ciLower":1787.86,"ciUpper":1829.51,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:fef6625d-9097-45f6-95c0-bcaa06e3d8e1","observedHarness":"Direct API","rawScore":44.9,"zScore":0.891538,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:fef6625d-9097-45f6-95c0-bcaa06e3d8e1","observedHarness":"Direct API","rawScore":19.44,"zScore":-0.174693,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Qwen 4 Max (xHigh reasoning)","observedHarness":"Direct API","rawScore":25.3,"zScore":0.546534,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Qwen 4 Max (xHigh reasoning)","observedHarness":"Direct API","rawScore":24.4,"zScore":0.788288,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:Qwen 4 Max (xHigh reasoning)","observedHarness":"Direct API","rawScore":17.5,"zScore":0.67918,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:Qwen 4 Max (xHigh reasoning)","observedHarness":"Direct API","rawScore":23.7,"zScore":0.315052,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:fef6625d-9097-45f6-95c0-bcaa06e3d8e1","observedHarness":"mini-SWE-agent","rawScore":23.33,"zScore":0.085746,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:alibaba/qwen4-max","observedHarness":"Direct API","rawScore":70.83,"zScore":-0.056688,"seModel":"upstream_se","seRaw":2.361,"ciHalfWidth":4.6276,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.997845,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:alibaba/qwen4-max","observedHarness":"Vals Finance Agent","rawScore":66.68,"zScore":1.08332,"seModel":"upstream_se","seRaw":2.02,"ciHalfWidth":3.9592,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.741142,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:alibaba/qwen4-max","observedHarness":"Direct API","rawScore":41.35,"zScore":0.227138,"seModel":"upstream_se","seRaw":3.423,"ciHalfWidth":6.7091,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.59377,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:alibaba/qwen4-max","observedHarness":"Vals Tax Agent","rawScore":40.35,"zScore":0.904589,"seModel":"upstream_se","seRaw":3.536,"ciHalfWidth":6.9306,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.031379,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"alibaba:qwen3.8-max-0902:xhigh:direct:native","provider":"alibaba","modelSlug":"qwen3.8-max-0902","displayName":"Qwen3.8 Max (0902)","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.530494,"shrinkageFactor":0.979161,"score":57.96,"se":4.17,"samplingSe":2.0517,"residualSe":3.632,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.977532,"costLogSd":1.0508,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":1000000,"releaseDate":"2026-09-02","paretoProb":0.8628,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:nBcLJJomVoeg2zh5F8Mnsu","observedHarness":"Direct API","rawScore":47.3,"zScore":-0.327913,"seModel":"upstream_se","seRaw":1.579622,"ciHalfWidth":3.0961,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-02","publishedDate":"2026-09-02","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdpval_aa","benchmarkVersion":"2.1","sourceId":"artificial-analysis","sourceRowId":"aa:gdpval_aa:b112af07-3bd5-4647-b09f-b23204361bb2","observedHarness":"Stirrup","rawScore":1674.05,"zScore":0.308004,"seModel":"upstream_ci","ciHalfWidth":18.725,"ciLower":1655.33,"ciUpper":1692.78,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:alibaba/qwen3.8-max-internal","observedHarness":"Direct API","rawScore":71.98,"zScore":0.258867,"seModel":"upstream_se","seRaw":2.387,"ciHalfWidth":4.6785,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.430287,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:alibaba/qwen3.8-max-internal","observedHarness":"Vals Finance Agent","rawScore":58.83,"zScore":-1.165441,"seModel":"upstream_se","seRaw":2.117,"ciHalfWidth":4.1493,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.992239,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:alibaba/qwen3.8-max-internal","observedHarness":"Direct API","rawScore":46.15,"zScore":0.652922,"seModel":"upstream_se","seRaw":3.465,"ciHalfWidth":6.7914,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.312295,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:alibaba/qwen3.8-max-internal","observedHarness":"Vals Tax Agent","rawScore":49.66,"zScore":1.725221,"seModel":"upstream_se","seRaw":3.603,"ciHalfWidth":7.0619,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.068548,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-5.6-sol:xhigh:direct:native","provider":"openai","modelSlug":"gpt-5.6-sol","displayName":"GPT-5.6 Sol","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.503447,"shrinkageFactor":0.992892,"score":57.55,"se":9.25,"samplingSe":1.1982,"residualSe":9.1768,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.461528,"costLogSd":1.056,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0356,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-5.6 Sol (xHigh)","observedHarness":"Direct API","rawScore":1476.18,"zScore":0.66993,"seModel":"upstream_ci","ciHalfWidth":3.3409,"ciLower":1472.8409,"ciUpper":1479.5227,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-opus-4.7:none:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.7","displayName":"Claude Opus 4.7","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.500433,"shrinkageFactor":0.993907,"score":57.51,"se":9.25,"samplingSe":1.1094,"residualSe":9.1861,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.977031,"costLogSd":1.049,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-04-16","paretoProb":0.3064,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Claude Opus 4.7","observedHarness":"Direct API","rawScore":1476.1,"zScore":0.666168,"seModel":"upstream_ci","ciHalfWidth":3.0918,"ciLower":1473.0129,"ciUpper":1479.1964,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-5.4:none:direct:native","provider":"openai","modelSlug":"gpt-5.4","displayName":"GPT-5.4","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.444298,"shrinkageFactor":0.994554,"score":56.66,"se":9.25,"samplingSe":1.0488,"residualSe":9.1921,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.859635,"costLogSd":1.0443,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":5.625,"inputPer1mUsd":2.5,"outputPer1mUsd":15,"contextWindow":1050000,"releaseDate":"2026-03-05","paretoProb":0.1265,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-5.4","observedHarness":"Direct API","rawScore":1474.89,"zScore":0.609262,"seModel":"upstream_ci","ciHalfWidth":2.9219,"ciLower":1471.9686,"ciUpper":1477.8124,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-6.1-terra:high:direct:native","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.438846,"shrinkageFactor":0.98695,"score":56.58,"se":4.05,"samplingSe":1.6236,"residualSe":3.7076,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.25928,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.3049,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:ea3e1f50-f62d-41d2-849a-9af0413e4d37","observedHarness":"Direct API","rawScore":47.27,"zScore":0.443679,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:ea3e1f50-f62d-41d2-849a-9af0413e4d37","observedHarness":"Stirrup","rawScore":61.5,"zScore":1.208896,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:ea3e1f50-f62d-41d2-849a-9af0413e4d37","observedHarness":"Direct API","rawScore":28,"zScore":1.051669,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:ea3e1f50-f62d-41d2-849a-9af0413e4d37","observedHarness":"Direct API","rawScore":35.4,"zScore":0.373648,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:ea3e1f50-f62d-41d2-849a-9af0413e4d37","observedHarness":"mini-SWE-agent","rawScore":33.33,"zScore":0.544551,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"google:gemini-4-hydrogen:medium:direct:native","provider":"google","modelSlug":"gemini-4-hydrogen","displayName":"Gemini 4 Hydrogen","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.431694,"shrinkageFactor":0.986482,"score":56.48,"se":3.72,"samplingSe":1.6524,"residualSe":3.3347,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.797987,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":1.5,"outputPer1mUsd":7.5,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.1316,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:f0537484-af30-4a8a-bf34-368c552e5cfd","observedHarness":"Direct API","rawScore":48.33,"zScore":0.519343,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:f0537484-af30-4a8a-bf34-368c552e5cfd","observedHarness":"Stirrup","rawScore":61,"zScore":1.183108,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:f0537484-af30-4a8a-bf34-368c552e5cfd","observedHarness":"Direct API","rawScore":32.8,"zScore":1.618174,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:f0537484-af30-4a8a-bf34-368c552e5cfd","observedHarness":"Direct API","rawScore":47.7,"zScore":1.044179,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:f0537484-af30-4a8a-bf34-368c552e5cfd","observedHarness":"Direct API","rawScore":20,"zScore":-0.11208,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:f0537484-af30-4a8a-bf34-368c552e5cfd","observedHarness":"mini-SWE-agent","rawScore":15.24,"zScore":-0.285428,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"anthropic:claude-fable-5.1:high:direct:native","provider":"anthropic","modelSlug":"claude-fable-5.1","displayName":"Claude Fable 5.1","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.418167,"shrinkageFactor":0.887865,"score":56.27,"se":9.7,"samplingSe":4.7592,"residualSe":8.4542,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":15.786777,"costLogSd":1.0567,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-09-12","paretoProb":0.0535,"benchmarks":[{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recNL6V1sB2zRy2vL","observedHarness":"APEX-Agents Harness","rawScore":59.7,"zScore":0.508745,"seModel":"upstream_se","seRaw":5.6,"ciHalfWidth":10.976,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"}]},{"id":"openai:gpt-5.6-sol:max:direct:native","provider":"openai","modelSlug":"gpt-5.6-sol","displayName":"GPT-5.6 Sol","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.37672,"shrinkageFactor":0.992528,"score":55.65,"se":2.65,"samplingSe":1.2285,"residualSe":2.3448,"coverageCount":10,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":12.117077,"costLogSd":1.0534,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.001,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:UnRUSuZzm2X27TUbpeMPva","observedHarness":"Direct API","rawScore":69.7,"zScore":1.265635,"seModel":"upstream_se","seRaw":1.453968,"ciHalfWidth":2.8498,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-10","publishedDate":"2026-08-10","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:GPT 5.6 Sol (one-pass-agent) max","observedHarness":"Scale one-pass-agent","rawScore":50.45,"zScore":0.475331,"seModel":"upstream_ci","ciHalfWidth":0.33,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-07-28","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:GPT 5.6 Sol (one-pass-agent) max","observedHarness":"Scale one-pass-agent","rawScore":50.5,"zScore":0.169115,"seModel":"upstream_ci","ciHalfWidth":0.54,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-07-28","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:GPT 5.6 Sol (Max reasoning)","observedHarness":"Direct API","rawScore":11.8,"zScore":-0.483199,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:GPT 5.6 Sol (Max reasoning)","observedHarness":"Direct API","rawScore":4.4,"zScore":-0.743995,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:GPT 5.6 Sol (Max reasoning)","observedHarness":"Direct API","rawScore":83.2,"zScore":1.421945,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:openai/gpt-5.6-sol","observedHarness":"Direct API","rawScore":48.08,"zScore":0.824123,"seModel":"upstream_se","seRaw":3.473,"ciHalfWidth":6.8071,"ciLower":null,"ciUpper":null,"measuredCostUsd":21.605566,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:openai/gpt-5.6-sol","observedHarness":"Direct API","rawScore":43.97,"zScore":-0.260888,"seModel":"upstream_se","seRaw":2.258,"ciHalfWidth":4.4257,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.280517,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:openai/gpt-5.6-sol","observedHarness":"Direct API","rawScore":33.33,"zScore":0.244434,"seModel":"upstream_se","seRaw":3.171,"ciHalfWidth":6.2152,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.96335,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:openai/gpt-5.6-sol","observedHarness":"Vals Tax Agent","rawScore":31.09,"zScore":0.088365,"seModel":"upstream_se","seRaw":3.34,"ciHalfWidth":6.5464,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.513965,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"anthropic:claude-opus-5.5:low:direct:native","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"low","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.335061,"shrinkageFactor":0.986086,"score":55.03,"se":3.73,"samplingSe":1.6765,"residualSe":3.3333,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.975529,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.3425,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:4416ca93-fde6-4b7b-abbc-3a6be109cc3d","observedHarness":"Direct API","rawScore":63.33,"zScore":1.590065,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:4416ca93-fde6-4b7b-abbc-3a6be109cc3d","observedHarness":"Stirrup","rawScore":36,"zScore":-0.106286,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:4416ca93-fde6-4b7b-abbc-3a6be109cc3d","observedHarness":"Direct API","rawScore":25.6,"zScore":0.768416,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:4416ca93-fde6-4b7b-abbc-3a6be109cc3d","observedHarness":"Direct API","rawScore":41.7,"zScore":0.717091,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:4416ca93-fde6-4b7b-abbc-3a6be109cc3d","observedHarness":"Direct API","rawScore":19.44,"zScore":-0.174693,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:4416ca93-fde6-4b7b-abbc-3a6be109cc3d","observedHarness":"mini-SWE-agent","rawScore":24.29,"zScore":0.129791,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"moonshot:kimi-k3.1-pro:max:direct:native","provider":"moonshot","modelSlug":"kimi-k3.1-pro","displayName":"Kimi K3.1 Pro","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.331837,"shrinkageFactor":0.994456,"score":54.98,"se":1.43,"samplingSe":1.0582,"residualSe":0.9571,"coverageCount":22,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.6812,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-10-10","paretoProb":0.7241,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:665c67ca-09eb-48f0-a7a3-0096e1d5de51","observedHarness":"AA Analyst Harness","rawScore":38.75,"zScore":-0.209844,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_briefcase:665c67ca-09eb-48f0-a7a3-0096e1d5de51","observedHarness":"Stirrup","rawScore":1673.56,"zScore":0.832489,"seModel":"upstream_ci","ciHalfWidth":10.95,"ciLower":1663.56,"ciUpper":1685.46,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase"},{"benchmarkId":"aa_mka","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_mka:665c67ca-09eb-48f0-a7a3-0096e1d5de51","observedHarness":"Stirrup","rawScore":43.86,"zScore":0.366303,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/mka"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:665c67ca-09eb-48f0-a7a3-0096e1d5de51","observedHarness":"Direct API","rawScore":48.93,"zScore":0.562172,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"aa_vials","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_vials:665c67ca-09eb-48f0-a7a3-0096e1d5de51","observedHarness":"Stirrup","rawScore":45.34,"zScore":0.153439,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/vials-aa"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:665c67ca-09eb-48f0-a7a3-0096e1d5de51","observedHarness":"Stirrup","rawScore":49.5,"zScore":0.589987,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:HumfsXVh5BZWuTo6Z8Gqnf","observedHarness":"Direct API","rawScore":62.2,"zScore":0.732081,"seModel":"upstream_se","seRaw":1.534117,"ciHalfWidth":3.0069,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-06","publishedDate":"2026-11-06","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:665c67ca-09eb-48f0-a7a3-0096e1d5de51","observedHarness":"Direct API","rawScore":31.4,"zScore":1.452943,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:665c67ca-09eb-48f0-a7a3-0096e1d5de51","observedHarness":"Direct API","rawScore":37.3,"zScore":0.477226,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Kimi K3.1 Pro (Max)","observedHarness":"Direct API","rawScore":1478.82,"zScore":0.79409,"seModel":"upstream_ci","ciHalfWidth":6.6657,"ciLower":1472.1523,"ciUpper":1485.4836,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:665c67ca-09eb-48f0-a7a3-0096e1d5de51","observedHarness":"Direct API","rawScore":38.33,"zScore":1.937383,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Kimi K3.1 Pro (Max reasoning)","observedHarness":"Direct API","rawScore":17,"zScore":-0.086561,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Kimi K3.1 Pro (Max reasoning)","observedHarness":"Direct API","rawScore":8.8,"zScore":-0.406893,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:Kimi K3.1 Pro (Max reasoning)","observedHarness":"Direct API","rawScore":22,"zScore":0.943193,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:Kimi K3.1 Pro (Max reasoning)","observedHarness":"Direct API","rawScore":14,"zScore":-0.950084,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:Kimi K3.1 Pro (Max reasoning)","observedHarness":"Direct API","rawScore":60.8,"zScore":0.25759,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:665c67ca-09eb-48f0-a7a3-0096e1d5de51","observedHarness":"mini-SWE-agent","rawScore":34.29,"zScore":0.588597,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:kimi/kimi-k3.1-pro","observedHarness":"Direct API","rawScore":70,"zScore":-0.284437,"seModel":"upstream_se","seRaw":2.63,"ciHalfWidth":5.1548,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.588749,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:kimi/kimi-k3.1-pro","observedHarness":"Vals Finance Agent","rawScore":61.49,"zScore":-0.40344,"seModel":"upstream_se","seRaw":2.1,"ciHalfWidth":4.116,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.290605,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:kimi/kimi-k3.1-pro","observedHarness":"Direct API","rawScore":37.5,"zScore":-0.114377,"seModel":"upstream_se","seRaw":3.365,"ciHalfWidth":6.5954,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.231463,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:kimi/kimi-k3.1-pro","observedHarness":"Direct API","rawScore":26.13,"zScore":-0.515667,"seModel":"upstream_se","seRaw":2.955,"ciHalfWidth":5.7918,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.62244,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:kimi/kimi-k3.1-pro","observedHarness":"Vals Tax Agent","rawScore":24.87,"zScore":-0.459898,"seModel":"upstream_se","seRaw":3.12,"ciHalfWidth":6.1152,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.643476,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"anthropic:claude-sonnet-5.5:high:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.286007,"shrinkageFactor":0.985248,"score":54.29,"se":3.75,"samplingSe":1.7262,"residualSe":3.3305,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.636129,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.0836,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:268525e2-f873-4178-bc07-e6a0ee1f002d","observedHarness":"Direct API","rawScore":50.33,"zScore":0.662106,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:268525e2-f873-4178-bc07-e6a0ee1f002d","observedHarness":"Stirrup","rawScore":51.5,"zScore":0.693138,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:268525e2-f873-4178-bc07-e6a0ee1f002d","observedHarness":"Direct API","rawScore":25.2,"zScore":0.721207,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:268525e2-f873-4178-bc07-e6a0ee1f002d","observedHarness":"Direct API","rawScore":33.8,"zScore":0.286424,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:268525e2-f873-4178-bc07-e6a0ee1f002d","observedHarness":"Direct API","rawScore":18.89,"zScore":-0.236188,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:268525e2-f873-4178-bc07-e6a0ee1f002d","observedHarness":"mini-SWE-agent","rawScore":34.29,"zScore":0.588597,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"openai:gpt-6.1-terra:max:direct:native","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.267893,"shrinkageFactor":0.994295,"score":54.02,"se":1.74,"samplingSe":1.0735,"residualSe":1.3734,"coverageCount":17,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.961108,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0,"benchmarks":[{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_briefcase:b0c462cf-f2d0-4126-9434-28db9502219f","observedHarness":"Stirrup","rawScore":1399.26,"zScore":-0.984879,"seModel":"upstream_ci","ciHalfWidth":9.79,"ciLower":1389.57,"ciUpper":1409.15,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase"},{"benchmarkId":"aa_mka","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_mka:b0c462cf-f2d0-4126-9434-28db9502219f","observedHarness":"Stirrup","rawScore":43.76,"zScore":0.35479,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/mka"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:b0c462cf-f2d0-4126-9434-28db9502219f","observedHarness":"Direct API","rawScore":47.63,"zScore":0.469376,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"aa_vials","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_vials:b0c462cf-f2d0-4126-9434-28db9502219f","observedHarness":"Stirrup","rawScore":60.25,"zScore":1.138864,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/vials-aa"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:b0c462cf-f2d0-4126-9434-28db9502219f","observedHarness":"Stirrup","rawScore":68,"zScore":1.544138,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recbZ5jYnCHQNhXed","observedHarness":"APEX-Agents Harness","rawScore":68.3,"zScore":1.025854,"seModel":"upstream_se","seRaw":5.2,"ciHalfWidth":10.192,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:Bixeio8A6SejZtUzFXFSXK","observedHarness":"Direct API","rawScore":57.3,"zScore":0.383493,"seModel":"upstream_se","seRaw":1.564979,"ciHalfWidth":3.0674,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-26","publishedDate":"2026-10-26","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:b0c462cf-f2d0-4126-9434-28db9502219f","observedHarness":"Direct API","rawScore":28.6,"zScore":1.122482,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdpval_aa","benchmarkVersion":"2.1","sourceId":"artificial-analysis","sourceRowId":"aa:gdpval_aa:b0c462cf-f2d0-4126-9434-28db9502219f","observedHarness":"Stirrup","rawScore":1475.84,"zScore":-0.95338,"seModel":"upstream_ci","ciHalfWidth":22.68,"ciLower":1453.16,"ciUpper":1498.52,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:b0c462cf-f2d0-4126-9434-28db9502219f","observedHarness":"Direct API","rawScore":47.3,"zScore":1.022374,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-6.1 Terra (Max)","observedHarness":"Direct API","rawScore":1455.44,"zScore":-0.305474,"seModel":"upstream_ci","ciHalfWidth":6.5422,"ciLower":1448.896,"ciUpper":1461.9805,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:b0c462cf-f2d0-4126-9434-28db9502219f","observedHarness":"Direct API","rawScore":30.56,"zScore":1.068625,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:b0c462cf-f2d0-4126-9434-28db9502219f","observedHarness":"mini-SWE-agent","rawScore":57.62,"zScore":1.65899,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:openai/gpt-6.1-terra","observedHarness":"Direct API","rawScore":73.33,"zScore":0.629301,"seModel":"upstream_se","seRaw":2.235,"ciHalfWidth":4.3806,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.559258,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:openai/gpt-6.1-terra","observedHarness":"Vals Finance Agent","rawScore":55.96,"zScore":-1.987599,"seModel":"upstream_se","seRaw":2.094,"ciHalfWidth":4.1042,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.922848,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:openai/gpt-6.1-terra","observedHarness":"Direct API","rawScore":25.96,"zScore":-1.138035,"seModel":"upstream_se","seRaw":3.047,"ciHalfWidth":5.9721,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.220614,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:openai/gpt-6.1-terra","observedHarness":"Vals Tax Agent","rawScore":17.08,"zScore":-1.146549,"seModel":"upstream_se","seRaw":2.714,"ciHalfWidth":5.3194,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.159819,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"google:gemini-3.5-flash:high:direct:native","provider":"google","modelSlug":"gemini-3.5-flash","displayName":"Gemini 3.5 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.258417,"shrinkageFactor":0.989354,"score":53.88,"se":3.92,"samplingSe":1.4664,"residualSe":3.6359,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.335721,"costLogSd":1.0439,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3.375,"inputPer1mUsd":1.5,"outputPer1mUsd":9,"contextWindow":1048576,"releaseDate":"2026-04-12","paretoProb":0.1305,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:GJ6YsSuzyRD9YXe6SBwvsu","observedHarness":"Direct API","rawScore":66.2,"zScore":1.016643,"seModel":"upstream_se","seRaw":1.496596,"ciHalfWidth":2.9333,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Gemini 3.5 Flash (High)","observedHarness":"Direct API","rawScore":1478.29,"zScore":0.769164,"seModel":"upstream_ci","ciHalfWidth":3.3461,"ciLower":1474.9474,"ciUpper":1481.6396,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:google/gemini-3.5-flash","observedHarness":"Direct API","rawScore":30.77,"zScore":-0.711363,"seModel":"upstream_se","seRaw":3.208,"ciHalfWidth":6.2877,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.305281,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:google/gemini-3.5-flash","observedHarness":"Direct API","rawScore":55.83,"zScore":1.471334,"seModel":"upstream_se","seRaw":2.113,"ciHalfWidth":4.1415,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.073716,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:google/gemini-3.5-flash","observedHarness":"Vals Tax Agent","rawScore":21.65,"zScore":-0.743726,"seModel":"upstream_se","seRaw":2.963,"ciHalfWidth":5.8075,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.859679,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-5.1:max:direct:native","provider":"openai","modelSlug":"gpt-5.1","displayName":"GPT-5.1","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.238067,"shrinkageFactor":0.987761,"score":53.57,"se":6.54,"samplingSe":1.5723,"residualSe":6.3442,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.350421,"costLogSd":1.0479,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3.4375,"inputPer1mUsd":1.25,"outputPer1mUsd":10,"contextWindow":400000,"releaseDate":"2025-11-13","paretoProb":0.0069,"benchmarks":[{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:GPT 5.1 (one-pass-agent) max","observedHarness":"Scale one-pass-agent","rawScore":48.01,"zScore":0.151358,"seModel":"upstream_ci","ciHalfWidth":0.09,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2025-12-11","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:GPT 5.1 (one-pass-agent) max","observedHarness":"Scale one-pass-agent","rawScore":49.33,"zScore":-0.058957,"seModel":"upstream_ci","ciHalfWidth":0.38,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2025-12-11","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"}]},{"id":"openai:gpt-6.1-terra:medium:direct:native","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.233954,"shrinkageFactor":0.987525,"score":53.51,"se":4.04,"samplingSe":1.5874,"residualSe":3.7098,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.168805,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.84,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:c441e158-a87d-4116-911b-93325e4b4f1c","observedHarness":"Direct API","rawScore":47.43,"zScore":0.4551,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:c441e158-a87d-4116-911b-93325e4b4f1c","observedHarness":"Stirrup","rawScore":56.5,"zScore":0.951017,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:c441e158-a87d-4116-911b-93325e4b4f1c","observedHarness":"Direct API","rawScore":28.4,"zScore":1.098877,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:c441e158-a87d-4116-911b-93325e4b4f1c","observedHarness":"Direct API","rawScore":31.3,"zScore":0.150138,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:c441e158-a87d-4116-911b-93325e4b4f1c","observedHarness":"mini-SWE-agent","rawScore":24.76,"zScore":0.151355,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"meta:muse-spark-1.2:xhigh:direct:native","provider":"meta","modelSlug":"muse-spark-1.2","displayName":"Muse Spark 1.2","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.232534,"shrinkageFactor":0.991376,"score":53.49,"se":3.28,"samplingSe":1.3198,"residualSe":3.0022,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.165655,"costLogSd":1.051,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.1915,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:mLLmVVvo6t23MM5yxq7ryW","observedHarness":"Direct API","rawScore":60.3,"zScore":0.596914,"seModel":"upstream_se","seRaw":1.551879,"ciHalfWidth":3.0417,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Muse Spark 1.2 (xHigh)","observedHarness":"Direct API","rawScore":1478.86,"zScore":0.795971,"seModel":"upstream_ci","ciHalfWidth":4.7191,"ciLower":1474.1431,"ciUpper":1483.5813,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Muse Spark 1.2 (xHigh reasoning)","observedHarness":"Direct API","rawScore":6.8,"zScore":-0.864581,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Muse Spark 1.2 (xHigh reasoning)","observedHarness":"Direct API","rawScore":8.8,"zScore":-0.406893,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:meta/muse_spark_1_2","observedHarness":"Direct API","rawScore":43.75,"zScore":0.44003,"seModel":"upstream_se","seRaw":3.448,"ciHalfWidth":6.7581,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.508097,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:meta/muse_spark_1_2","observedHarness":"Direct API","rawScore":49.35,"zScore":0.524892,"seModel":"upstream_se","seRaw":2.187,"ciHalfWidth":4.2865,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.039023,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:meta/muse_spark_1_2","observedHarness":"Vals Tax Agent","rawScore":32.8,"zScore":0.239093,"seModel":"upstream_se","seRaw":2.751,"ciHalfWidth":5.392,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.269212,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"anthropic:claude-opus-4.8:high:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.8","displayName":"Claude Opus 4.8","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.227436,"shrinkageFactor":0.993872,"score":53.41,"se":9.25,"samplingSe":1.1126,"residualSe":9.1858,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.432844,"costLogSd":1.0492,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-06-12","paretoProb":0.0632,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Claude Opus 4.8 (High)","observedHarness":"Direct API","rawScore":1470.26,"zScore":0.391512,"seModel":"upstream_ci","ciHalfWidth":3.1007,"ciLower":1467.1597,"ciUpper":1473.3611,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-3.8-flash:high:direct:native","provider":"google","modelSlug":"gemini-3.8-flash","displayName":"Gemini 3.8 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.217444,"shrinkageFactor":0.993496,"score":53.26,"se":2.08,"samplingSe":1.1462,"residualSe":1.7381,"coverageCount":15,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.542136,"costLogSd":1.045,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.5,"inputPer1mUsd":0.75,"outputPer1mUsd":3.75,"contextWindow":1048576,"releaseDate":"2026-08-10","paretoProb":0.2951,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:gZ4bwozTKtpmZTqnqX3a33","observedHarness":"Direct API","rawScore":69.7,"zScore":1.265635,"seModel":"upstream_se","seRaw":1.453968,"ciHalfWidth":2.8498,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-02","publishedDate":"2026-09-02","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Gemini 3.8 Flash (High)","observedHarness":"Direct API","rawScore":1469.07,"zScore":0.335546,"seModel":"upstream_ci","ciHalfWidth":3.2976,"ciLower":1465.7694,"ciUpper":1472.3645,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:Gemini 3.8 Flash (one-pass-agent) high","observedHarness":"Scale one-pass-agent","rawScore":49.51,"zScore":0.350522,"seModel":"upstream_ci","ciHalfWidth":1.66,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-09","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Gemini 3.8 Flash (one-pass-agent) high","observedHarness":"Scale one-pass-agent","rawScore":48.9,"zScore":-0.142779,"seModel":"upstream_ci","ciHalfWidth":1.65,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-09","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Gemini 3.8 Flash (High reasoning)","observedHarness":"Direct API","rawScore":4.5,"zScore":-1.040017,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Gemini 3.8 Flash (High reasoning)","observedHarness":"Direct API","rawScore":2,"zScore":-0.927869,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:Gemini 3.8 Flash (High reasoning)","observedHarness":"Direct API","rawScore":0,"zScore":-0.347537,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:Gemini 3.8 Flash (High reasoning)","observedHarness":"Direct API","rawScore":18.3,"zScore":-0.38925,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:Gemini 3.8 Flash (High reasoning)","observedHarness":"Direct API","rawScore":56.8,"zScore":0.04967,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:google/gemini-3.8-flash","observedHarness":"Direct API","rawScore":72.87,"zScore":0.503079,"seModel":"upstream_se","seRaw":2.424,"ciHalfWidth":4.751,"ciLower":null,"ciUpper":null,"measuredCostUsd":8.243862,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:google/gemini-3.8-flash","observedHarness":"Vals Finance Agent","rawScore":65.76,"zScore":0.819771,"seModel":"upstream_se","seRaw":2.039,"ciHalfWidth":3.9964,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.991077,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:google/gemini-3.8-flash","observedHarness":"Direct API","rawScore":38.94,"zScore":0.013358,"seModel":"upstream_se","seRaw":3.389,"ciHalfWidth":6.6424,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.630498,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:google/gemini-3.8-flash","observedHarness":"Direct API","rawScore":48.13,"zScore":0.346704,"seModel":"upstream_se","seRaw":2.18,"ciHalfWidth":4.2728,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.0177,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:google/gemini-3.8-flash","observedHarness":"Direct API","rawScore":36.49,"zScore":0.578034,"seModel":"upstream_se","seRaw":3.238,"ciHalfWidth":6.3465,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.731041,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:google/gemini-3.8-flash","observedHarness":"Vals Tax Agent","rawScore":32.12,"zScore":0.179154,"seModel":"upstream_se","seRaw":3.37,"ciHalfWidth":6.6052,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.857214,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"anthropic:claude-opus-4.8:max:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.8","displayName":"Claude Opus 4.8","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.211776,"shrinkageFactor":0.98338,"score":53.18,"se":4.26,"samplingSe":1.8322,"residualSe":3.8418,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":14.827231,"costLogSd":1.0558,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-06-12","paretoProb":0.0131,"benchmarks":[{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:reckFRpJRQsQ8Z4Sm","observedHarness":"APEX-Agents Harness","rawScore":48.9,"zScore":-0.140647,"seModel":"upstream_se","seRaw":5.5,"ciHalfWidth":10.78,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:ViC6pxQZUVS3E9M9iEubF7","observedHarness":"Direct API","rawScore":53,"zScore":0.077588,"seModel":"upstream_se","seRaw":1.57908,"ciHalfWidth":3.095,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:anthropic/claude-opus-4-8","observedHarness":"Direct API","rawScore":43.75,"zScore":0.44003,"seModel":"upstream_se","seRaw":3.448,"ciHalfWidth":6.7581,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.820942,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:anthropic/claude-opus-4-8","observedHarness":"Direct API","rawScore":53.22,"zScore":1.090128,"seModel":"upstream_se","seRaw":2.165,"ciHalfWidth":4.2434,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.350925,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:anthropic/claude-opus-4-8","observedHarness":"Vals Tax Agent","rawScore":30,"zScore":-0.007713,"seModel":"upstream_se","seRaw":3.303,"ciHalfWidth":6.4739,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.68964,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-6-astra:xhigh:direct:native","provider":"openai","modelSlug":"gpt-6-astra","displayName":"GPT-6 Astra","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.10097,"shrinkageFactor":0.975996,"score":51.51,"se":6.64,"samplingSe":2.202,"residualSe":6.2686,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":19.394239,"costLogSd":1.0664,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0004,"benchmarks":[{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:GPT 6 Astra (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":47.54,"zScore":0.088954,"seModel":"upstream_ci","ciHalfWidth":1.69,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-09","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:GPT 6 Astra (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":48.36,"zScore":-0.248043,"seModel":"upstream_ci","ciHalfWidth":1.64,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-09","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"}]},{"id":"meta:muse-wonder:high:direct:native","provider":"meta","modelSlug":"muse-wonder","displayName":"Muse Wonder","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.099375,"shrinkageFactor":0.98769,"score":51.49,"se":4.03,"samplingSe":1.5769,"residualSe":3.7104,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.560546,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.0026,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:38a98e3d-779b-49ec-b005-230a3a72088a","observedHarness":"Direct API","rawScore":38.2,"zScore":-0.203751,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:38a98e3d-779b-49ec-b005-230a3a72088a","observedHarness":"Stirrup","rawScore":51,"zScore":0.66735,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:38a98e3d-779b-49ec-b005-230a3a72088a","observedHarness":"Direct API","rawScore":29.2,"zScore":1.193295,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:38a98e3d-779b-49ec-b005-230a3a72088a","observedHarness":"Direct API","rawScore":34.5,"zScore":0.324585,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:38a98e3d-779b-49ec-b005-230a3a72088a","observedHarness":"mini-SWE-agent","rawScore":21.9,"zScore":0.020137,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"google:gemini-3.5-flash:medium:direct:native","provider":"google","modelSlug":"gemini-3.5-flash","displayName":"Gemini 3.5 Flash","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.077547,"shrinkageFactor":0.992775,"score":51.16,"se":9.25,"samplingSe":1.208,"residualSe":9.1757,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.774672,"costLogSd":1.0539,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3.375,"inputPer1mUsd":1.5,"outputPer1mUsd":9,"contextWindow":1048576,"releaseDate":"2026-04-12","paretoProb":0.265,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Gemini 3.5 Flash (Medium)","observedHarness":"Direct API","rawScore":1467.06,"zScore":0.241016,"seModel":"upstream_ci","ciHalfWidth":3.3685,"ciLower":1463.6905,"ciUpper":1470.4275,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-3.7-flash:high:direct:native","provider":"google","modelSlug":"gemini-3.7-flash","displayName":"Gemini 3.7 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.076768,"shrinkageFactor":0.992097,"score":51.15,"se":3.26,"samplingSe":1.2635,"residualSe":3.0044,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.542136,"costLogSd":1.045,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.5,"inputPer1mUsd":0.75,"outputPer1mUsd":3.75,"contextWindow":1048576,"releaseDate":"2026-06-18","paretoProb":0.2347,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:84ARuGyhQsjgh8umRmBvas","observedHarness":"Direct API","rawScore":69.2,"zScore":1.230065,"seModel":"upstream_se","seRaw":1.460648,"ciHalfWidth":2.8629,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Gemini 3.7 Flash (High)","observedHarness":"Direct API","rawScore":1476.09,"zScore":0.665698,"seModel":"upstream_ci","ciHalfWidth":3.8852,"ciLower":1472.2078,"ciUpper":1479.9781,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Gemini 3.7 Flash (High reasoning)","observedHarness":"Direct API","rawScore":5.3,"zScore":-0.978996,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Gemini 3.7 Flash (High reasoning)","observedHarness":"Direct API","rawScore":2.4,"zScore":-0.897223,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:google/gemini-3.7-flash","observedHarness":"Direct API","rawScore":34.62,"zScore":-0.369848,"seModel":"upstream_se","seRaw":3.307,"ciHalfWidth":6.4817,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.760397,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:google/gemini-3.7-flash","observedHarness":"Direct API","rawScore":53.39,"zScore":1.114958,"seModel":"upstream_se","seRaw":2.12,"ciHalfWidth":4.1552,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.038331,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:google/gemini-3.7-flash","observedHarness":"Vals Tax Agent","rawScore":22.8,"zScore":-0.642359,"seModel":"upstream_se","seRaw":3.028,"ciHalfWidth":5.9349,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.483978,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"spacexai:grok-4.6:xhigh:direct:native","provider":"spacexai","modelSlug":"grok-4.6","displayName":"Grok 4.6","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.054238,"shrinkageFactor":0.982171,"score":50.81,"se":4.91,"samplingSe":1.8977,"residualSe":4.5235,"coverageCount":4,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":2.977532,"costLogSd":1.0508,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-08-25","paretoProb":0.6354,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:dKCWFaoGCWJTnD5CPWQGru","observedHarness":"Direct API","rawScore":48.9,"zScore":-0.214088,"seModel":"upstream_se","seRaw":1.581547,"ciHalfWidth":3.0998,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdpval_aa","benchmarkVersion":"2.1","sourceId":"artificial-analysis","sourceRowId":"aa:gdpval_aa:d6f43d0a-e8c4-447c-9ff7-5cf0a9d59aa3","observedHarness":"Stirrup","rawScore":1658.22,"zScore":0.207264,"seModel":"upstream_ci","ciHalfWidth":17.915,"ciLower":1640.31,"ciUpper":1676.14,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Grok 4.6 (xHigh reasoning)","observedHarness":"Direct API","rawScore":21,"zScore":0.218545,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Grok 4.6 (xHigh reasoning)","observedHarness":"Direct API","rawScore":14.8,"zScore":0.052792,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"}]},{"id":"moonshot:kimi-k3.1-pro:xhigh:direct:native","provider":"moonshot","modelSlug":"kimi-k3.1-pro","displayName":"Kimi K3.1 Pro","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.050466,"shrinkageFactor":0.987258,"score":50.76,"se":6.54,"samplingSe":1.6043,"residualSe":6.3409,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.062211,"costLogSd":0.6488,"costMaxUsd":3.6812,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-10-10","paretoProb":0.9192,"benchmarks":[{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:Kimi K3.1 Pro (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":47.54,"zScore":0.088954,"seModel":"upstream_ci","ciHalfWidth":0.48,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-12","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Kimi K3.1 Pro (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":47.87,"zScore":-0.34356,"seModel":"upstream_ci","ciHalfWidth":0.37,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-12","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"}]},{"id":"anthropic:claude-opus-4.7:max:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.7","displayName":"Claude Opus 4.7","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.044791,"shrinkageFactor":0.973944,"score":50.67,"se":5.03,"samplingSe":2.2942,"residualSe":4.479,"coverageCount":4,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":14.827231,"costLogSd":1.0558,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-04-16","paretoProb":0.0093,"benchmarks":[{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recDYoVbtdEHFX5Vv","observedHarness":"APEX-Agents Harness","rawScore":49.2,"zScore":-0.122608,"seModel":"upstream_se","seRaw":5.4,"ciHalfWidth":10.584,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:anthropic/claude-opus-4-7","observedHarness":"Direct API","rawScore":38.46,"zScore":-0.02922,"seModel":"upstream_se","seRaw":3.381,"ciHalfWidth":6.6268,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.05609,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:anthropic/claude-opus-4-7","observedHarness":"Direct API","rawScore":54.86,"zScore":1.32966,"seModel":"upstream_se","seRaw":2.205,"ciHalfWidth":4.3218,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.226314,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:anthropic/claude-opus-4-7","observedHarness":"Vals Tax Agent","rawScore":23.74,"zScore":-0.559502,"seModel":"upstream_se","seRaw":3.063,"ciHalfWidth":6.0035,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.274806,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-5.5:xhigh:direct:native","provider":"openai","modelSlug":"gpt-5.5","displayName":"GPT-5.5","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.042127,"shrinkageFactor":0.982699,"score":50.63,"se":4.62,"samplingSe":1.8694,"residualSe":4.2237,"coverageCount":4,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":12.089838,"costLogSd":1.0598,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":11.25,"inputPer1mUsd":5,"outputPer1mUsd":30,"contextWindow":1050000,"releaseDate":"2026-04-23","paretoProb":0.0006,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:TijCafTTJUFrJ6jPGvBbgq","observedHarness":"Direct API","rawScore":63,"zScore":0.788994,"seModel":"upstream_se","seRaw":1.527525,"ciHalfWidth":2.9939,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:openai/gpt-5.5","observedHarness":"Direct API","rawScore":40.38,"zScore":0.141094,"seModel":"upstream_se","seRaw":3.41,"ciHalfWidth":6.6836,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.402471,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:openai/gpt-5.5","observedHarness":"Direct API","rawScore":49.1,"zScore":0.488378,"seModel":"upstream_se","seRaw":2.188,"ciHalfWidth":4.2885,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.160759,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:openai/gpt-5.5","observedHarness":"Vals Tax Agent","rawScore":21.76,"zScore":-0.73403,"seModel":"upstream_se","seRaw":2.978,"ciHalfWidth":5.8369,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.073277,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"spacexai:grok-4.7:xhigh:direct:native","provider":"spacexai","modelSlug":"grok-4.7","displayName":"Grok 4.7","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.034268,"shrinkageFactor":0.995467,"score":50.51,"se":1.12,"samplingSe":0.9569,"residualSe":0.5813,"coverageCount":24,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":7.771539,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-10-18","paretoProb":0.2361,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:59627802-1546-422c-9a65-65b61c022d36","observedHarness":"AA Analyst Harness","rawScore":52.5,"zScore":0.791212,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_briefcase:59627802-1546-422c-9a65-65b61c022d36","observedHarness":"Stirrup","rawScore":1634.38,"zScore":0.572903,"seModel":"upstream_ci","ciHalfWidth":8.02,"ciLower":1626.37,"ciUpper":1642.41,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase"},{"benchmarkId":"aa_mka","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_mka:59627802-1546-422c-9a65-65b61c022d36","observedHarness":"Stirrup","rawScore":40.14,"zScore":-0.061981,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/mka"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:59627802-1546-422c-9a65-65b61c022d36","observedHarness":"Direct API","rawScore":42.7,"zScore":0.117465,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"aa_vials","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_vials:59627802-1546-422c-9a65-65b61c022d36","observedHarness":"Stirrup","rawScore":37.89,"zScore":-0.338943,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/vials-aa"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:59627802-1546-422c-9a65-65b61c022d36","observedHarness":"Stirrup","rawScore":33.5,"zScore":-0.235226,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:6izDCw5ygCLZJEKPS8jV5T","observedHarness":"Direct API","rawScore":56,"zScore":0.29101,"seModel":"upstream_se","seRaw":1.570499,"ciHalfWidth":3.0782,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-22","publishedDate":"2026-09-22","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:59627802-1546-422c-9a65-65b61c022d36","observedHarness":"Direct API","rawScore":20,"zScore":0.107494,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdpval_aa","benchmarkVersion":"2.1","sourceId":"artificial-analysis","sourceRowId":"aa:gdpval_aa:59627802-1546-422c-9a65-65b61c022d36","observedHarness":"Stirrup","rawScore":1716.83,"zScore":0.580251,"seModel":"upstream_ci","ciHalfWidth":20.205,"ciLower":1696.62,"ciUpper":1737.03,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:59627802-1546-422c-9a65-65b61c022d36","observedHarness":"Direct API","rawScore":23.2,"zScore":-0.291432,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Grok 4.7 (xHigh)","observedHarness":"Direct API","rawScore":1438.83,"zScore":-1.086645,"seModel":"upstream_ci","ciHalfWidth":4.3349,"ciLower":1434.4947,"ciUpper":1443.1645,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:59627802-1546-422c-9a65-65b61c022d36","observedHarness":"Direct API","rawScore":20,"zScore":-0.11208,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Grok 4.7 (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":47.62,"zScore":-0.392294,"seModel":"upstream_ci","ciHalfWidth":1.79,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-25","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Grok 4.7 (xHigh reasoning)","observedHarness":"Direct API","rawScore":26.5,"zScore":0.638066,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Grok 4.7 (xHigh reasoning)","observedHarness":"Direct API","rawScore":27.6,"zScore":1.033453,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:Grok 4.7 (xHigh reasoning)","observedHarness":"Direct API","rawScore":22.3,"zScore":0.132455,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:Grok 4.7 (xHigh reasoning)","observedHarness":"Direct API","rawScore":57.6,"zScore":0.091254,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:59627802-1546-422c-9a65-65b61c022d36","observedHarness":"mini-SWE-agent","rawScore":11.43,"zScore":-0.460233,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:grok/grok-4.7","observedHarness":"Direct API","rawScore":71.5,"zScore":0.127157,"seModel":"upstream_se","seRaw":2.328,"ciHalfWidth":4.5629,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.481284,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:grok/grok-4.7","observedHarness":"Vals Finance Agent","rawScore":60.59,"zScore":-0.66126,"seModel":"upstream_se","seRaw":2.114,"ciHalfWidth":4.1434,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.238652,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:grok/grok-4.7","observedHarness":"Direct API","rawScore":47.12,"zScore":0.738966,"seModel":"upstream_se","seRaw":3.469,"ciHalfWidth":6.7992,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.920812,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:grok/grok-4.7","observedHarness":"Direct API","rawScore":49.55,"zScore":0.554103,"seModel":"upstream_se","seRaw":2.171,"ciHalfWidth":4.2552,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.105497,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:grok/grok-4.7","observedHarness":"Direct API","rawScore":25.23,"zScore":-0.610679,"seModel":"upstream_se","seRaw":2.921,"ciHalfWidth":5.7252,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.841835,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:grok/grok-4.7","observedHarness":"Vals Tax Agent","rawScore":35.75,"zScore":0.499122,"seModel":"upstream_se","seRaw":3.459,"ciHalfWidth":6.7796,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.796539,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"meta:muse-ember:max:direct:native","provider":"meta","modelSlug":"muse-ember","displayName":"Muse Ember","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.022901,"shrinkageFactor":0.99131,"score":50.34,"se":2.15,"samplingSe":1.3249,"residualSe":1.687,"coverageCount":15,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.135936,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.4887,"benchmarks":[{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_briefcase:dbf63365-127f-41ee-9e51-81173adb3502","observedHarness":"Stirrup","rawScore":1519.75,"zScore":-0.186575,"seModel":"upstream_ci","ciHalfWidth":10.66,"ciLower":1509.51,"ciUpper":1530.83,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:dbf63365-127f-41ee-9e51-81173adb3502","observedHarness":"Direct API","rawScore":21.07,"zScore":-1.426516,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:dbf63365-127f-41ee-9e51-81173adb3502","observedHarness":"Stirrup","rawScore":39,"zScore":0.048441,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:dbf63365-127f-41ee-9e51-81173adb3502","observedHarness":"Direct API","rawScore":22.8,"zScore":0.437955,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:dbf63365-127f-41ee-9e51-81173adb3502","observedHarness":"Direct API","rawScore":25.1,"zScore":-0.187854,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:dbf63365-127f-41ee-9e51-81173adb3502","observedHarness":"Direct API","rawScore":18.33,"zScore":-0.298801,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Muse Ember 1.0 (Max reasoning)","observedHarness":"Direct API","rawScore":24.2,"zScore":0.46263,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Muse Ember 1.0 (Max reasoning)","observedHarness":"Direct API","rawScore":17.2,"zScore":0.236666,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:Muse Ember 1.0 (Max reasoning)","observedHarness":"Direct API","rawScore":15.4,"zScore":-0.767487,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:Muse Ember 1.0 (Max reasoning)","observedHarness":"Direct API","rawScore":53.6,"zScore":-0.116666,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:dbf63365-127f-41ee-9e51-81173adb3502","observedHarness":"mini-SWE-agent","rawScore":20,"zScore":-0.067036,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:meta/muse-ember","observedHarness":"Direct API","rawScore":72.17,"zScore":0.311002,"seModel":"upstream_se","seRaw":2.308,"ciHalfWidth":4.5237,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.091434,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:meta/muse-ember","observedHarness":"Vals Finance Agent","rawScore":63.83,"zScore":0.266891,"seModel":"upstream_se","seRaw":2.006,"ciHalfWidth":3.9318,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.056396,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:meta/muse-ember","observedHarness":"Direct API","rawScore":49.04,"zScore":0.90928,"seModel":"upstream_se","seRaw":3.475,"ciHalfWidth":6.811,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.056581,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:meta/muse-ember","observedHarness":"Vals Tax Agent","rawScore":38.23,"zScore":0.717722,"seModel":"upstream_se","seRaw":3.499,"ciHalfWidth":6.858,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.031788,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-5.4:high:direct:native","provider":"openai","modelSlug":"gpt-5.4","displayName":"GPT-5.4","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.014443,"shrinkageFactor":0.993062,"score":50.22,"se":5.15,"samplingSe":1.1838,"residualSe":5.0108,"coverageCount":3,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":5.273919,"costLogSd":1.0456,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":5.625,"inputPer1mUsd":2.5,"outputPer1mUsd":15,"contextWindow":1050000,"releaseDate":"2026-03-05","paretoProb":0.0028,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-5.4 (High)","observedHarness":"Direct API","rawScore":1484.4,"zScore":1.056518,"seModel":"upstream_ci","ciHalfWidth":2.9654,"ciLower":1481.4376,"ciUpper":1487.3683,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:GPT 5.4 (one-pass-agent) high","observedHarness":"Scale one-pass-agent","rawScore":45.63,"zScore":-0.164648,"seModel":"upstream_ci","ciHalfWidth":0.555,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-04-22","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:GPT 5.4 (one-pass-agent) high","observedHarness":"Scale one-pass-agent","rawScore":44.35,"zScore":-1.029727,"seModel":"upstream_ci","ciHalfWidth":0.278,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-04-22","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"}]},{"id":"meta:muse-ember:xhigh:direct:native","provider":"meta","modelSlug":"muse-ember","displayName":"Muse Ember","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.006019,"shrinkageFactor":0.992961,"score":50.09,"se":3.08,"samplingSe":1.1924,"residualSe":2.8348,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.116228,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.9488,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:7a64ffc4-36c3-4369-981e-73abdd6ff5e2","observedHarness":"Direct API","rawScore":21.37,"zScore":-1.405101,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:7a64ffc4-36c3-4369-981e-73abdd6ff5e2","observedHarness":"Stirrup","rawScore":38.5,"zScore":0.022653,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:7a64ffc4-36c3-4369-981e-73abdd6ff5e2","observedHarness":"Direct API","rawScore":22.4,"zScore":0.390746,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:7a64ffc4-36c3-4369-981e-73abdd6ff5e2","observedHarness":"Direct API","rawScore":23.6,"zScore":-0.269626,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:Muse Ember 1.0 (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":55.97,"zScore":1.208253,"seModel":"upstream_ci","ciHalfWidth":0.66,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-19","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Muse Ember 1.0 (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":55.6,"zScore":1.163277,"seModel":"upstream_ci","ciHalfWidth":0.59,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-19","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:7a64ffc4-36c3-4369-981e-73abdd6ff5e2","observedHarness":"mini-SWE-agent","rawScore":17.62,"zScore":-0.176232,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"anthropic:claude-haiku-5.5:max:direct:native","provider":"anthropic","modelSlug":"claude-haiku-5.5","displayName":"Claude Haiku 5.5","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.010663,"shrinkageFactor":0.993624,"score":49.84,"se":1.7,"samplingSe":1.1349,"residualSe":1.2677,"coverageCount":19,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.411007,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1000000,"releaseDate":"2026-10-07","paretoProb":0.5926,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:7d064a11-6e78-4030-b535-cdf940615041","observedHarness":"AA Analyst Harness","rawScore":53.75,"zScore":0.882217,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_briefcase:7d064a11-6e78-4030-b535-cdf940615041","observedHarness":"Stirrup","rawScore":1571.33,"zScore":0.155167,"seModel":"upstream_ci","ciHalfWidth":9.37,"ciLower":1562.38,"ciUpper":1581.12,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase"},{"benchmarkId":"aa_mka","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_mka:7d064a11-6e78-4030-b535-cdf940615041","observedHarness":"Stirrup","rawScore":40.96,"zScore":0.032426,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/mka"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:7d064a11-6e78-4030-b535-cdf940615041","observedHarness":"Direct API","rawScore":34.67,"zScore":-0.455728,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"aa_vials","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_vials:7d064a11-6e78-4030-b535-cdf940615041","observedHarness":"Stirrup","rawScore":49.07,"zScore":0.399961,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/vials-aa"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:7d064a11-6e78-4030-b535-cdf940615041","observedHarness":"Stirrup","rawScore":43.5,"zScore":0.280532,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recuIVFmW80Eh5kgM","observedHarness":"APEX-Agents Harness","rawScore":56.1,"zScore":0.292281,"seModel":"upstream_se","seRaw":5.2,"ciHalfWidth":10.192,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:NwnrUMtveqHDYB2Um4ycmA","observedHarness":"Direct API","rawScore":23.8,"zScore":-1.999715,"seModel":"upstream_se","seRaw":1.347359,"ciHalfWidth":2.6408,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-09","publishedDate":"2026-10-09","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:7d064a11-6e78-4030-b535-cdf940615041","observedHarness":"Direct API","rawScore":20.8,"zScore":0.201911,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdpval_aa","benchmarkVersion":"2.1","sourceId":"artificial-analysis","sourceRowId":"aa:gdpval_aa:7d064a11-6e78-4030-b535-cdf940615041","observedHarness":"Stirrup","rawScore":1628.98,"zScore":0.021184,"seModel":"upstream_ci","ciHalfWidth":21.355,"ciLower":1607.63,"ciUpper":1650.34,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:7d064a11-6e78-4030-b535-cdf940615041","observedHarness":"Direct API","rawScore":29.6,"zScore":0.057463,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:7d064a11-6e78-4030-b535-cdf940615041","observedHarness":"Direct API","rawScore":35,"zScore":1.565058,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:7d064a11-6e78-4030-b535-cdf940615041","observedHarness":"mini-SWE-agent","rawScore":20,"zScore":-0.067036,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:anthropic/claude-haiku-5-5","observedHarness":"Direct API","rawScore":65.48,"zScore":-1.524706,"seModel":"upstream_se","seRaw":3.691,"ciHalfWidth":7.2344,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.364559,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:anthropic/claude-haiku-5-5","observedHarness":"Vals Finance Agent","rawScore":65.37,"zScore":0.70805,"seModel":"upstream_se","seRaw":2.063,"ciHalfWidth":4.0435,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.637243,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:anthropic/claude-haiku-5-5","observedHarness":"Direct API","rawScore":43.75,"zScore":0.44003,"seModel":"upstream_se","seRaw":3.448,"ciHalfWidth":6.7581,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.647359,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:anthropic/claude-haiku-5-5","observedHarness":"Direct API","rawScore":51.54,"zScore":0.844754,"seModel":"upstream_se","seRaw":2.166,"ciHalfWidth":4.2454,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.031937,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:anthropic/claude-haiku-5-5","observedHarness":"Direct API","rawScore":35.13,"zScore":0.434459,"seModel":"upstream_se","seRaw":3.211,"ciHalfWidth":6.2936,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.541024,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:anthropic/claude-haiku-5-5","observedHarness":"Vals Tax Agent","rawScore":31.92,"zScore":0.161525,"seModel":"upstream_se","seRaw":3.352,"ciHalfWidth":6.5699,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.164207,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"anthropic:claude-opus-4.5:high:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.5","displayName":"Claude Opus 4.5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.010481,"shrinkageFactor":0.912248,"score":49.84,"se":9.5,"samplingSe":4.2101,"residualSe":8.5116,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.432844,"costLogSd":1.0492,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":200000,"releaseDate":"2025-11-24","paretoProb":0.0443,"benchmarks":[{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:anthropic/claude-opus-4-5-20251101-thinking","observedHarness":"Direct API","rawScore":49.16,"zScore":0.497142,"seModel":"upstream_se","seRaw":2.012,"ciHalfWidth":3.9435,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.095846,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"}]},{"id":"google:gemini-3.6-flash:high:direct:native","provider":"google","modelSlug":"gemini-3.6-flash","displayName":"Gemini 3.6 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.019989,"shrinkageFactor":0.989979,"score":49.7,"se":3.91,"samplingSe":1.4228,"residualSe":3.6382,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.542136,"costLogSd":1.045,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.5,"inputPer1mUsd":0.75,"outputPer1mUsd":3.75,"contextWindow":1048576,"releaseDate":"2026-05-05","paretoProb":0.193,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:F83pa7Gfsb4UNVttTnS5Hk","observedHarness":"Direct API","rawScore":66.2,"zScore":1.016643,"seModel":"upstream_se","seRaw":1.496596,"ciHalfWidth":2.9333,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Gemini 3.6 Flash (High)","observedHarness":"Direct API","rawScore":1472.1,"zScore":0.478048,"seModel":"upstream_ci","ciHalfWidth":3.6546,"ciLower":1468.4456,"ciUpper":1475.7547,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:google/gemini-3.6-flash","observedHarness":"Direct API","rawScore":25,"zScore":-1.223192,"seModel":"upstream_se","seRaw":3.01,"ciHalfWidth":5.8996,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.466741,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:google/gemini-3.6-flash","observedHarness":"Direct API","rawScore":53.15,"zScore":1.079904,"seModel":"upstream_se","seRaw":2.157,"ciHalfWidth":4.2277,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.044216,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:google/gemini-3.6-flash","observedHarness":"Vals Tax Agent","rawScore":18.04,"zScore":-1.06193,"seModel":"upstream_se","seRaw":2.766,"ciHalfWidth":5.4214,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.356293,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-6.1-sol:xhigh:direct:native","provider":"openai","modelSlug":"gpt-6.1-sol","displayName":"GPT-6.1 Sol","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.026365,"shrinkageFactor":0.974125,"score":49.6,"se":6.66,"samplingSe":2.2861,"residualSe":6.2566,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.535732,"costLogSd":1.0518,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.0059,"benchmarks":[{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:GPT 6.1 Sol (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":46.21,"zScore":-0.087638,"seModel":"upstream_ci","ciHalfWidth":1.72,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-07","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:GPT 6.1 Sol (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":47.87,"zScore":-0.34356,"seModel":"upstream_ci","ciHalfWidth":1.69,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-07","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"}]},{"id":"anthropic:claude-opus-4.8:none:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.8","displayName":"Claude Opus 4.8","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.036153,"shrinkageFactor":0.994078,"score":49.46,"se":9.25,"samplingSe":1.0937,"residualSe":9.1877,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.977031,"costLogSd":1.049,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-06-12","paretoProb":0.1437,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Claude Opus 4.8","observedHarness":"Direct API","rawScore":1464.62,"zScore":0.126262,"seModel":"upstream_ci","ciHalfWidth":3.0478,"ciLower":1461.575,"ciUpper":1467.6706,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"deepseek:deepseek-v4.1-flash-1017:max:direct:native","provider":"deepseek","modelSlug":"deepseek-v4.1-flash-1017","displayName":"DeepSeek V4.1 Flash (1017)","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.048532,"shrinkageFactor":0.994698,"score":49.27,"se":1.53,"samplingSe":1.0349,"residualSe":1.1319,"coverageCount":21,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.602105,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.525,"inputPer1mUsd":0.3,"outputPer1mUsd":1.2,"contextWindow":1048576,"releaseDate":"2026-10-17","paretoProb":0.9817,"benchmarks":[{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_briefcase:f151fd0b-2139-40cf-a3ec-995e8cedc592","observedHarness":"Stirrup","rawScore":1570.42,"zScore":0.149138,"seModel":"upstream_ci","ciHalfWidth":10.005,"ciLower":1560.41,"ciUpper":1580.42,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase"},{"benchmarkId":"aa_mka","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_mka:f151fd0b-2139-40cf-a3ec-995e8cedc592","observedHarness":"Stirrup","rawScore":38.54,"zScore":-0.246189,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/mka"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:f151fd0b-2139-40cf-a3ec-995e8cedc592","observedHarness":"Direct API","rawScore":39.6,"zScore":-0.103817,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"aa_vials","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_vials:f151fd0b-2139-40cf-a3ec-995e8cedc592","observedHarness":"Stirrup","rawScore":40.37,"zScore":-0.175036,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/vials-aa"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:f151fd0b-2139-40cf-a3ec-995e8cedc592","observedHarness":"Stirrup","rawScore":25,"zScore":-0.67362,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:rec4h0kAzxLz00RFT","observedHarness":"APEX-Agents Harness","rawScore":69.8,"zScore":1.116047,"seModel":"upstream_se","seRaw":4.9,"ciHalfWidth":9.604,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:Ei4XWaCH5drYUnDRo2MYCW","observedHarness":"Direct API","rawScore":42.5,"zScore":-0.669387,"seModel":"upstream_se","seRaw":1.564032,"ciHalfWidth":3.0655,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-19","publishedDate":"2026-10-19","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:f151fd0b-2139-40cf-a3ec-995e8cedc592","observedHarness":"Direct API","rawScore":17.2,"zScore":-0.222968,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdpval_aa","benchmarkVersion":"2.1","sourceId":"artificial-analysis","sourceRowId":"aa:gdpval_aa:f151fd0b-2139-40cf-a3ec-995e8cedc592","observedHarness":"Stirrup","rawScore":1745.77,"zScore":0.764422,"seModel":"upstream_ci","ciHalfWidth":20.79,"ciLower":1724.98,"ciUpper":1766.56,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:f151fd0b-2139-40cf-a3ec-995e8cedc592","observedHarness":"Direct API","rawScore":25.3,"zScore":-0.176951,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:f151fd0b-2139-40cf-a3ec-995e8cedc592","observedHarness":"Direct API","rawScore":26.67,"zScore":0.633687,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:DeepSeek V4.1 Flash 1017 (Max reasoning)","observedHarness":"Direct API","rawScore":12.5,"zScore":-0.429805,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:DeepSeek V4.1 Flash 1017 (Max reasoning)","observedHarness":"Direct API","rawScore":6,"zScore":-0.621412,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:DeepSeek V4.1 Flash 1017 (Max reasoning)","observedHarness":"Direct API","rawScore":2.8,"zScore":-0.183262,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:DeepSeek V4.1 Flash 1017 (Max reasoning)","observedHarness":"Direct API","rawScore":18.4,"zScore":-0.376208,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:DeepSeek V4.1 Flash 1017 (Max reasoning)","observedHarness":"Direct API","rawScore":38.4,"zScore":-0.906764,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:f151fd0b-2139-40cf-a3ec-995e8cedc592","observedHarness":"mini-SWE-agent","rawScore":12.86,"zScore":-0.394624,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:deepseek/deepseek-v4.1-flash-1017","observedHarness":"Direct API","rawScore":69.17,"zScore":-0.512185,"seModel":"upstream_se","seRaw":2.395,"ciHalfWidth":4.6942,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.330922,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:deepseek/deepseek-v4.1-flash-1017","observedHarness":"Vals Finance Agent","rawScore":62.93,"zScore":0.009072,"seModel":"upstream_se","seRaw":2.028,"ciHalfWidth":3.9749,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.466261,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:deepseek/deepseek-v4.1-flash-1017","observedHarness":"Direct API","rawScore":45.19,"zScore":0.567765,"seModel":"upstream_se","seRaw":3.459,"ciHalfWidth":6.7796,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.69856,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:deepseek/deepseek-v4.1-flash-1017","observedHarness":"Vals Tax Agent","rawScore":44.35,"zScore":1.25717,"seModel":"upstream_se","seRaw":3.574,"ciHalfWidth":7.005,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.459454,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:o3:high:direct:native","provider":"openai","modelSlug":"o3","displayName":"o3","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.063858,"shrinkageFactor":0.990344,"score":49.04,"se":4.43,"samplingSe":1.3966,"residualSe":4.2032,"coverageCount":4,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.072691,"costLogSd":1.0438,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3.5,"inputPer1mUsd":2,"outputPer1mUsd":8,"contextWindow":200000,"releaseDate":"2025-04-16","paretoProb":0.0115,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:CxHSa2gY7ySiPPGTZEZaRt","observedHarness":"Direct API","rawScore":49.4,"zScore":-0.178518,"seModel":"upstream_se","seRaw":1.581816,"ciHalfWidth":3.1004,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:o3 (one-pass-agent) high","observedHarness":"Scale one-pass-agent","rawScore":47.69,"zScore":0.10887,"seModel":"upstream_ci","ciHalfWidth":0.4458,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2025-11-13","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:o3 (one-pass-agent) high","observedHarness":"Scale one-pass-agent","rawScore":48.57,"zScore":-0.207107,"seModel":"upstream_ci","ciHalfWidth":0.5384,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2025-11-13","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:openai/o3-2025-04-16","observedHarness":"Direct API","rawScore":47.29,"zScore":0.224017,"seModel":"upstream_se","seRaw":2.161,"ciHalfWidth":4.2356,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.029818,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"}]},{"id":"openai:gpt-5.6-terra:max:direct:native","provider":"openai","modelSlug":"gpt-5.6-terra","displayName":"GPT-5.6 Terra","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.064547,"shrinkageFactor":0.981057,"score":49.03,"se":4.86,"samplingSe":1.9561,"residualSe":4.4529,"coverageCount":4,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":7.571751,"costLogSd":1.049,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4.5,"inputPer1mUsd":2,"outputPer1mUsd":12,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0014,"benchmarks":[{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recIl7G6eKw5AFuMv","observedHarness":"APEX-Agents Harness","rawScore":58.2,"zScore":0.418552,"seModel":"upstream_se","seRaw":5.3,"ciHalfWidth":10.388,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:DienatPbgfgn3EzM7Gvzxr","observedHarness":"Direct API","rawScore":43.2,"zScore":-0.619589,"seModel":"upstream_se","seRaw":1.567232,"ciHalfWidth":3.0718,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-10","publishedDate":"2026-08-10","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:openai/gpt-5.6-terra","observedHarness":"Direct API","rawScore":41.35,"zScore":0.227138,"seModel":"upstream_se","seRaw":3.423,"ciHalfWidth":6.7091,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.907299,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:openai/gpt-5.6-terra","observedHarness":"Vals Tax Agent","rawScore":30.04,"zScore":-0.004188,"seModel":"upstream_se","seRaw":3.307,"ciHalfWidth":6.4817,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.829225,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"spacexai:grok-4.6:high:direct:native","provider":"spacexai","modelSlug":"grok-4.6","displayName":"Grok 4.6","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.069149,"shrinkageFactor":0.988234,"score":48.96,"se":3.65,"samplingSe":1.5416,"residualSe":3.3106,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":2.430646,"costLogSd":1.0437,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-08-25","paretoProb":0.5326,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:TpHYtuPG6kYZFvMK7Z6Hqj","observedHarness":"Direct API","rawScore":49.3,"zScore":-0.185632,"seModel":"upstream_se","seRaw":1.581775,"ciHalfWidth":3.1003,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Grok 4.6 (High)","observedHarness":"Direct API","rawScore":1451.28,"zScore":-0.50112,"seModel":"upstream_ci","ciHalfWidth":3.7167,"ciLower":1447.5584,"ciUpper":1454.9917,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:grok/grok-4.6","observedHarness":"Direct API","rawScore":48.08,"zScore":0.824123,"seModel":"upstream_se","seRaw":3.473,"ciHalfWidth":6.8071,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.530204,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:grok/grok-4.6","observedHarness":"Direct API","rawScore":44.71,"zScore":-0.152807,"seModel":"upstream_se","seRaw":2.256,"ciHalfWidth":4.4218,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.050335,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:grok/grok-4.6","observedHarness":"Direct API","rawScore":30.63,"zScore":-0.040604,"seModel":"upstream_se","seRaw":3.101,"ciHalfWidth":6.078,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.309658,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:grok/grok-4.6","observedHarness":"Vals Tax Agent","rawScore":33.66,"zScore":0.314898,"seModel":"upstream_se","seRaw":3.409,"ciHalfWidth":6.6816,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.978328,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"google:gemini-4-hydrogen:low:direct:native","provider":"google","modelSlug":"gemini-4-hydrogen","displayName":"Gemini 4 Hydrogen","effort":"low","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.069704,"shrinkageFactor":0.987908,"score":48.95,"se":3.69,"samplingSe":1.5629,"residualSe":3.3395,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.292223,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":1.5,"outputPer1mUsd":7.5,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.5764,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:88c68c50-03c1-458a-9292-bd398c27adc5","observedHarness":"Direct API","rawScore":46.23,"zScore":0.369442,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:88c68c50-03c1-458a-9292-bd398c27adc5","observedHarness":"Stirrup","rawScore":48,"zScore":0.512623,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:88c68c50-03c1-458a-9292-bd398c27adc5","observedHarness":"Direct API","rawScore":28.2,"zScore":1.075273,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:88c68c50-03c1-458a-9292-bd398c27adc5","observedHarness":"Direct API","rawScore":32.2,"zScore":0.199201,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:88c68c50-03c1-458a-9292-bd398c27adc5","observedHarness":"Direct API","rawScore":22.22,"zScore":0.136136,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:88c68c50-03c1-458a-9292-bd398c27adc5","observedHarness":"mini-SWE-agent","rawScore":2.38,"zScore":-0.875452,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"moonshot:kimi-k3:max:direct:native","provider":"moonshot","modelSlug":"kimi-k3","displayName":"Kimi K3","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.081982,"shrinkageFactor":0.990965,"score":48.77,"se":2.59,"samplingSe":1.3509,"residualSe":2.2121,"coverageCount":12,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":9.345558,"costLogSd":1.0507,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1048576,"releaseDate":"2026-08-14","paretoProb":0.255,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:mZLWt2JSYvJyGhMvji7CT8","observedHarness":"Direct API","rawScore":50.6,"zScore":-0.093149,"seModel":"upstream_se","seRaw":1.581816,"ciHalfWidth":3.1004,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Kimi K3 (Max)","observedHarness":"Direct API","rawScore":1470.53,"zScore":0.40421,"seModel":"upstream_ci","ciHalfWidth":3.2952,"ciLower":1467.2336,"ciUpper":1473.824,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Kimi K3 (Max reasoning)","observedHarness":"Direct API","rawScore":5.5,"zScore":-0.963741,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Kimi K3 (Max reasoning)","observedHarness":"Direct API","rawScore":4,"zScore":-0.774641,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:Kimi K3 (Max reasoning)","observedHarness":"Direct API","rawScore":1.7,"zScore":-0.247799,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:Kimi K3 (Max reasoning)","observedHarness":"Direct API","rawScore":13.4,"zScore":-1.02834,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:Kimi K3 (Max reasoning)","observedHarness":"Direct API","rawScore":36.8,"zScore":-0.989932,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:kimi/kimi-k3","observedHarness":"Direct API","rawScore":71.26,"zScore":0.061302,"seModel":"upstream_se","seRaw":2.283,"ciHalfWidth":4.4747,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.074655,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:kimi/kimi-k3","observedHarness":"Vals Finance Agent","rawScore":60.01,"zScore":-0.82741,"seModel":"upstream_se","seRaw":2.09,"ciHalfWidth":4.0964,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.881233,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:kimi/kimi-k3","observedHarness":"Direct API","rawScore":46.15,"zScore":0.652922,"seModel":"upstream_se","seRaw":3.465,"ciHalfWidth":6.7914,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.469601,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:kimi/kimi-k3","observedHarness":"Direct API","rawScore":49.36,"zScore":0.526353,"seModel":"upstream_se","seRaw":2.202,"ciHalfWidth":4.3159,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.093857,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:kimi/kimi-k3","observedHarness":"Vals Tax Agent","rawScore":33.37,"zScore":0.289336,"seModel":"upstream_se","seRaw":3.386,"ciHalfWidth":6.6366,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.790127,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-5.1:high:direct:native","provider":"openai","modelSlug":"gpt-5.1","displayName":"GPT-5.1","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.103528,"shrinkageFactor":0.988377,"score":48.45,"se":5.21,"samplingSe":1.5322,"residualSe":4.9813,"coverageCount":3,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.622478,"costLogSd":1.0441,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3.4375,"inputPer1mUsd":1.25,"outputPer1mUsd":10,"contextWindow":400000,"releaseDate":"2025-11-13","paretoProb":0.0057,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:QCLnRy69NBH9ip3VYtQneh","observedHarness":"Direct API","rawScore":48,"zScore":-0.278114,"seModel":"upstream_se","seRaw":1.580664,"ciHalfWidth":3.0981,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-5.1 (High)","observedHarness":"Direct API","rawScore":1458.05,"zScore":-0.182726,"seModel":"upstream_ci","ciHalfWidth":3.3221,"ciLower":1454.7297,"ciUpper":1461.3738,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:openai/gpt-5.1-2025-11-13","observedHarness":"Direct API","rawScore":52.73,"zScore":1.018561,"seModel":"upstream_se","seRaw":2.151,"ciHalfWidth":4.216,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.014371,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"}]},{"id":"anthropic:claude-opus-4.7:xhigh:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.7","displayName":"Claude Opus 4.7","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.105077,"shrinkageFactor":0.986104,"score":48.42,"se":9.24,"samplingSe":1.6754,"residualSe":9.0827,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":10.349955,"costLogSd":1.058,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-04-16","paretoProb":0.0258,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:2c4rvmf5N7QXTugrFLMZ83","observedHarness":"Direct API","rawScore":51.7,"zScore":-0.014894,"seModel":"upstream_se","seRaw":1.581015,"ciHalfWidth":3.0988,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"}]},{"id":"openai:gpt-6.1-terra:low:direct:native","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"low","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.174217,"shrinkageFactor":0.990047,"score":47.39,"se":3.98,"samplingSe":1.4179,"residualSe":3.7193,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.071338,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.9254,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:c955b262-a17f-4f1a-b097-6860b866295b","observedHarness":"Direct API","rawScore":45.57,"zScore":0.32233,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:c955b262-a17f-4f1a-b097-6860b866295b","observedHarness":"Stirrup","rawScore":50.5,"zScore":0.641562,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:c955b262-a17f-4f1a-b097-6860b866295b","observedHarness":"Direct API","rawScore":25.6,"zScore":0.768416,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:c955b262-a17f-4f1a-b097-6860b866295b","observedHarness":"Direct API","rawScore":24.9,"zScore":-0.198757,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:c955b262-a17f-4f1a-b097-6860b866295b","observedHarness":"mini-SWE-agent","rawScore":7.62,"zScore":-0.635038,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"google:gemini-3-flash:high:direct:native","provider":"google","modelSlug":"gemini-3-flash","displayName":"Gemini 3 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.200606,"shrinkageFactor":0.991583,"score":46.99,"se":3.87,"samplingSe":1.3039,"residualSe":3.6441,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.253198,"costLogSd":1.0461,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.125,"inputPer1mUsd":0.5,"outputPer1mUsd":3,"contextWindow":1048576,"releaseDate":"2025-12-15","paretoProb":0.1886,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:7cc6cDCXjsPjtKt36bbeMR","observedHarness":"Direct API","rawScore":66.8,"zScore":1.059328,"seModel":"upstream_se","seRaw":1.48996,"ciHalfWidth":2.9203,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Gemini 3 Flash","observedHarness":"Direct API","rawScore":1468,"zScore":0.285224,"seModel":"upstream_ci","ciHalfWidth":3.956,"ciLower":1464.0463,"ciUpper":1471.9582,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:google/gemini-3-flash-preview","observedHarness":"Direct API","rawScore":18.27,"zScore":-1.820178,"seModel":"upstream_se","seRaw":2.686,"ciHalfWidth":5.2646,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.135395,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:google/gemini-3-flash-preview","observedHarness":"Direct API","rawScore":55.92,"zScore":1.484479,"seModel":"upstream_se","seRaw":2.112,"ciHalfWidth":4.1395,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.006187,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:google/gemini-3-flash-preview","observedHarness":"Vals Tax Agent","rawScore":11.84,"zScore":-1.60843,"seModel":"upstream_se","seRaw":2.324,"ciHalfWidth":4.555,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.100699,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"anthropic:claude-sonnet-4.6:none:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-4.6","displayName":"Claude Sonnet 4.6","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.217088,"shrinkageFactor":0.994666,"score":46.74,"se":9.25,"samplingSe":1.038,"residualSe":9.1932,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.87726,"costLogSd":1.0443,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1000000,"releaseDate":"2026-02-17","paretoProb":0.1813,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Claude Sonnet 4.6","observedHarness":"Direct API","rawScore":1460.75,"zScore":-0.055744,"seModel":"upstream_ci","ciHalfWidth":2.8918,"ciLower":1457.8595,"ciUpper":1463.643,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"spacexai:grok-4.7:high:direct:native","provider":"spacexai","modelSlug":"grok-4.7","displayName":"Grok 4.7","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.244657,"shrinkageFactor":0.988491,"score":46.33,"se":3.46,"samplingSe":1.5247,"residualSe":3.1021,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.183596,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-10-18","paretoProb":0.094,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:272f5f03-aea9-4675-a244-2b753b618cdf","observedHarness":"AA Analyst Harness","rawScore":55,"zScore":0.973223,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:272f5f03-aea9-4675-a244-2b753b618cdf","observedHarness":"Direct API","rawScore":41.77,"zScore":0.051081,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:272f5f03-aea9-4675-a244-2b753b618cdf","observedHarness":"Stirrup","rawScore":37,"zScore":-0.05471,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:272f5f03-aea9-4675-a244-2b753b618cdf","observedHarness":"Direct API","rawScore":23.2,"zScore":0.485164,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:272f5f03-aea9-4675-a244-2b753b618cdf","observedHarness":"Direct API","rawScore":23.6,"zScore":-0.269626,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:272f5f03-aea9-4675-a244-2b753b618cdf","observedHarness":"Direct API","rawScore":18.89,"zScore":-0.236188,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:272f5f03-aea9-4675-a244-2b753b618cdf","observedHarness":"mini-SWE-agent","rawScore":9.52,"zScore":-0.547865,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"openai:gpt-5.6-luna:xhigh:direct:native","provider":"openai","modelSlug":"gpt-5.6-luna","displayName":"GPT-5.6 Luna","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.251143,"shrinkageFactor":0.992538,"score":46.23,"se":9.26,"samplingSe":1.2277,"residualSe":9.1735,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.681333,"costLogSd":1.0589,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.45,"inputPer1mUsd":0.2,"outputPer1mUsd":1.2,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.142,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-5.6 Luna (xHigh)","observedHarness":"Direct API","rawScore":1460.02,"zScore":-0.090076,"seModel":"upstream_ci","ciHalfWidth":3.4238,"ciLower":1456.5984,"ciUpper":1463.446,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-opus-4.6:max:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.6","displayName":"Claude Opus 4.6","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.255692,"shrinkageFactor":0.980101,"score":46.16,"se":5.68,"samplingSe":2.0049,"residualSe":5.3128,"coverageCount":3,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":14.827231,"costLogSd":1.0558,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-02-05","paretoProb":0.0044,"benchmarks":[{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recfKa2RdTGFZh3gQ","observedHarness":"APEX-Agents Harness","rawScore":46.3,"zScore":-0.296982,"seModel":"upstream_se","seRaw":5.5,"ciHalfWidth":10.78,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:85fhMxLaebWtLoi6oLpTbi","observedHarness":"Direct API","rawScore":47,"zScore":-0.349255,"seModel":"upstream_se","seRaw":1.57908,"ciHalfWidth":3.095,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:anthropic/claude-opus-4-6-thinking","observedHarness":"Direct API","rawScore":49.13,"zScore":0.49276,"seModel":"upstream_se","seRaw":2.085,"ciHalfWidth":4.0866,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.244127,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"}]},{"id":"anthropic:claude-haiku-5.5:xhigh:direct:native","provider":"anthropic","modelSlug":"claude-haiku-5.5","displayName":"Claude Haiku 5.5","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.277559,"shrinkageFactor":0.987574,"score":45.84,"se":3.7,"samplingSe":1.5843,"residualSe":3.3384,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.248361,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1000000,"releaseDate":"2026-10-07","paretoProb":0.7781,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:41b5bf5e-daf6-42d9-b8d1-60e340a3028d","observedHarness":"Direct API","rawScore":33.77,"zScore":-0.519971,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:41b5bf5e-daf6-42d9-b8d1-60e340a3028d","observedHarness":"Stirrup","rawScore":47.5,"zScore":0.486835,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:41b5bf5e-daf6-42d9-b8d1-60e340a3028d","observedHarness":"Direct API","rawScore":18.2,"zScore":-0.104946,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:41b5bf5e-daf6-42d9-b8d1-60e340a3028d","observedHarness":"Direct API","rawScore":28.8,"zScore":0.013851,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:41b5bf5e-daf6-42d9-b8d1-60e340a3028d","observedHarness":"Direct API","rawScore":15,"zScore":-0.671126,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:41b5bf5e-daf6-42d9-b8d1-60e340a3028d","observedHarness":"mini-SWE-agent","rawScore":18.1,"zScore":-0.154209,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"anthropic:claude-opus-4.5:none:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.5","displayName":"Claude Opus 4.5","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.2873,"shrinkageFactor":0.9952,"score":45.69,"se":9.25,"samplingSe":0.9846,"residualSe":9.1981,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.977031,"costLogSd":1.049,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":200000,"releaseDate":"2025-11-24","paretoProb":0.0996,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Claude Opus 4.5 (20251101)","observedHarness":"Direct API","rawScore":1459.25,"zScore":-0.12629,"seModel":"upstream_ci","ciHalfWidth":2.7422,"ciLower":1456.5091,"ciUpper":1461.9935,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"zhipu:glm-5.3:max:direct:native","provider":"zhipu","modelSlug":"glm-5.3","displayName":"GLM-5.3","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.296294,"shrinkageFactor":0.995647,"score":45.56,"se":1.26,"samplingSe":0.9377,"residualSe":0.8436,"coverageCount":22,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":5.009265,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":1048576,"releaseDate":"2026-09-18","paretoProb":0.5467,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:cd684ea4-b475-4269-b001-d469d06d8a7a","observedHarness":"AA Analyst Harness","rawScore":38.75,"zScore":-0.209844,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_briefcase:cd684ea4-b475-4269-b001-d469d06d8a7a","observedHarness":"Stirrup","rawScore":1499.11,"zScore":-0.323325,"seModel":"upstream_ci","ciHalfWidth":7.405,"ciLower":1492.13,"ciUpper":1506.94,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:cd684ea4-b475-4269-b001-d469d06d8a7a","observedHarness":"Direct API","rawScore":32.33,"zScore":-0.62276,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:cd684ea4-b475-4269-b001-d469d06d8a7a","observedHarness":"Stirrup","rawScore":33.5,"zScore":-0.235226,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:TY8R8FV7zCwByEipkqyg2P","observedHarness":"Direct API","rawScore":41,"zScore":-0.776098,"seModel":"upstream_se","seRaw":1.556092,"ciHalfWidth":3.0499,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-28","publishedDate":"2026-08-28","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:cd684ea4-b475-4269-b001-d469d06d8a7a","observedHarness":"Direct API","rawScore":11.2,"zScore":-0.931099,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdpval_aa","benchmarkVersion":"2.1","sourceId":"artificial-analysis","sourceRowId":"aa:gdpval_aa:cd684ea4-b475-4269-b001-d469d06d8a7a","observedHarness":"Stirrup","rawScore":1654.78,"zScore":0.185372,"seModel":"upstream_ci","ciHalfWidth":17.9,"ciLower":1636.88,"ciUpper":1672.68,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:cd684ea4-b475-4269-b001-d469d06d8a7a","observedHarness":"Direct API","rawScore":17.9,"zScore":-0.58036,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GLM-5.3 (Max)","observedHarness":"Direct API","rawScore":1468.68,"zScore":0.317204,"seModel":"upstream_ci","ciHalfWidth":3.7832,"ciLower":1464.8976,"ciUpper":1472.4639,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:cd684ea4-b475-4269-b001-d469d06d8a7a","observedHarness":"Direct API","rawScore":17.22,"zScore":-0.42291,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:GLM 5.3 (Max reasoning)","observedHarness":"Direct API","rawScore":11,"zScore":-0.54422,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:GLM 5.3 (Max reasoning)","observedHarness":"Direct API","rawScore":4.4,"zScore":-0.743995,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:GLM 5.3 (Max reasoning)","observedHarness":"Direct API","rawScore":3.7,"zScore":-0.130459,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:GLM 5.3 (Max reasoning)","observedHarness":"Direct API","rawScore":21.4,"zScore":0.015072,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:GLM 5.3 (Max reasoning)","observedHarness":"Direct API","rawScore":20.8,"zScore":-1.821614,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:cd684ea4-b475-4269-b001-d469d06d8a7a","observedHarness":"mini-SWE-agent","rawScore":9.52,"zScore":-0.547865,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:zai/glm-5.3","observedHarness":"Direct API","rawScore":69.11,"zScore":-0.528649,"seModel":"upstream_se","seRaw":2.649,"ciHalfWidth":5.192,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.787827,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:zai/glm-5.3","observedHarness":"Vals Finance Agent","rawScore":65,"zScore":0.602057,"seModel":"upstream_se","seRaw":2.01,"ciHalfWidth":3.9396,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.030939,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:zai/glm-5.3","observedHarness":"Direct API","rawScore":49.04,"zScore":0.90928,"seModel":"upstream_se","seRaw":3.475,"ciHalfWidth":6.811,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.241388,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:zai/glm-5.3","observedHarness":"Direct API","rawScore":42.86,"zScore":-0.42301,"seModel":"upstream_se","seRaw":2.111,"ciHalfWidth":4.1376,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.081905,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:zai/glm-5.3","observedHarness":"Direct API","rawScore":22.97,"zScore":-0.849267,"seModel":"upstream_se","seRaw":2.83,"ciHalfWidth":5.5468,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.531337,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:zai/glm-5.3","observedHarness":"Vals Tax Agent","rawScore":43.33,"zScore":1.167262,"seModel":"upstream_se","seRaw":3.565,"ciHalfWidth":6.9874,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.80097,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"anthropic:claude-sonnet-5.5:medium:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.305126,"shrinkageFactor":0.988724,"score":45.42,"se":3.45,"samplingSe":1.5092,"residualSe":3.1035,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.780656,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.0365,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:042e8a37-fcd0-40f4-8f41-99215fb68eda","observedHarness":"Direct API","rawScore":45.77,"zScore":0.336607,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:042e8a37-fcd0-40f4-8f41-99215fb68eda","observedHarness":"Stirrup","rawScore":33.5,"zScore":-0.235226,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recWY8vFxRRaVvJK3","observedHarness":"APEX-Agents Harness","rawScore":44.6,"zScore":-0.399201,"seModel":"upstream_se","seRaw":5.6,"ciHalfWidth":10.976,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:042e8a37-fcd0-40f4-8f41-99215fb68eda","observedHarness":"Direct API","rawScore":20.2,"zScore":0.131098,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:042e8a37-fcd0-40f4-8f41-99215fb68eda","observedHarness":"Direct API","rawScore":23.8,"zScore":-0.258723,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:042e8a37-fcd0-40f4-8f41-99215fb68eda","observedHarness":"Direct API","rawScore":17.78,"zScore":-0.360297,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:042e8a37-fcd0-40f4-8f41-99215fb68eda","observedHarness":"mini-SWE-agent","rawScore":16.19,"zScore":-0.241841,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"spacexai:grok-4.5:high:direct:native","provider":"spacexai","modelSlug":"grok-4.5","displayName":"Grok 4.5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.310702,"shrinkageFactor":0.987947,"score":45.34,"se":3.95,"samplingSe":1.5603,"residualSe":3.6307,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.430646,"costLogSd":1.0437,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-06-30","paretoProb":0.3406,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:7HWW3sjRstKzb9JpcnvLaV","observedHarness":"Direct API","rawScore":48.3,"zScore":-0.256772,"seModel":"upstream_se","seRaw":1.581015,"ciHalfWidth":3.0988,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Grok 4.5","observedHarness":"Direct API","rawScore":1464.71,"zScore":0.130495,"seModel":"upstream_ci","ciHalfWidth":3.4732,"ciLower":1461.2374,"ciUpper":1468.1839,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:grok/grok-4.5","observedHarness":"Direct API","rawScore":37.98,"zScore":-0.071799,"seModel":"upstream_se","seRaw":3.373,"ciHalfWidth":6.6111,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.086814,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:grok/grok-4.5","observedHarness":"Direct API","rawScore":43.29,"zScore":-0.360206,"seModel":"upstream_se","seRaw":2.313,"ciHalfWidth":4.5335,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.048461,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:grok/grok-4.5","observedHarness":"Vals Tax Agent","rawScore":24.86,"zScore":-0.46078,"seModel":"upstream_se","seRaw":3.118,"ciHalfWidth":6.1113,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.473604,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-5:high:direct:native","provider":"openai","modelSlug":"gpt-5","displayName":"GPT-5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.320184,"shrinkageFactor":0.993163,"score":45.2,"se":3.8,"samplingSe":1.1752,"residualSe":3.615,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.622478,"costLogSd":1.0441,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3.4375,"inputPer1mUsd":1.25,"outputPer1mUsd":10,"contextWindow":400000,"releaseDate":"2025-08-07","paretoProb":0.003,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:gTLurbyKU63i2opsGkJGwh","observedHarness":"Direct API","rawScore":50.1,"zScore":-0.128719,"seModel":"upstream_se","seRaw":1.581927,"ciHalfWidth":3.1006,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-5 (High)","observedHarness":"Direct API","rawScore":1420.56,"zScore":-1.945885,"seModel":"upstream_ci","ciHalfWidth":6.83,"ciLower":1413.7268,"ciUpper":1427.3867,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:GPT 5 (one-pass-agent) high","observedHarness":"Scale one-pass-agent","rawScore":51.32,"zScore":0.590846,"seModel":"upstream_ci","ciHalfWidth":0.1714,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2025-11-13","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:GPT 5 (one-pass-agent) high","observedHarness":"Scale one-pass-agent","rawScore":48.96,"zScore":-0.131083,"seModel":"upstream_ci","ciHalfWidth":0.0281,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2025-11-13","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:openai/gpt-5-2025-08-07","observedHarness":"Direct API","rawScore":49.63,"zScore":0.565788,"seModel":"upstream_se","seRaw":2.098,"ciHalfWidth":4.1121,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.045858,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"}]},{"id":"meta:muse-spark-1.2:high:direct:native","provider":"meta","modelSlug":"muse-spark-1.2","displayName":"Muse Spark 1.2","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.348465,"shrinkageFactor":0.908527,"score":44.77,"se":9.66,"samplingSe":4.2985,"residualSe":8.651,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.768757,"costLogSd":1.0444,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.1053,"benchmarks":[{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recu1kljfh6r4IOhx","observedHarness":"APEX-Agents Harness","rawScore":45.4,"zScore":-0.351098,"seModel":"upstream_se","seRaw":5,"ciHalfWidth":9.8,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"}]},{"id":"spacexai:grok-4.7:low:direct:native","provider":"spacexai","modelSlug":"grok-4.7","displayName":"Grok 4.7","effort":"low","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.364468,"shrinkageFactor":0.988238,"score":44.53,"se":3.68,"samplingSe":1.5414,"residualSe":3.3406,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.922201,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-10-18","paretoProb":0.1028,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:208c16f0-8c5b-4524-ac8a-ac74c02e80a9","observedHarness":"Direct API","rawScore":45.73,"zScore":0.333751,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:208c16f0-8c5b-4524-ac8a-ac74c02e80a9","observedHarness":"Stirrup","rawScore":43.5,"zScore":0.280532,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:208c16f0-8c5b-4524-ac8a-ac74c02e80a9","observedHarness":"Direct API","rawScore":21.4,"zScore":0.272724,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:208c16f0-8c5b-4524-ac8a-ac74c02e80a9","observedHarness":"Direct API","rawScore":18.5,"zScore":-0.547651,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:208c16f0-8c5b-4524-ac8a-ac74c02e80a9","observedHarness":"Direct API","rawScore":21.11,"zScore":0.012028,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:208c16f0-8c5b-4524-ac8a-ac74c02e80a9","observedHarness":"mini-SWE-agent","rawScore":4.76,"zScore":-0.766256,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"openai:gpt-6-sol:max:direct:native","provider":"openai","modelSlug":"gpt-6-sol","displayName":"GPT-6 Sol","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.389976,"shrinkageFactor":0.992067,"score":44.15,"se":2.79,"samplingSe":1.2659,"residualSe":2.4916,"coverageCount":10,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.487251,"costLogSd":1.048,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0002,"benchmarks":[{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recLBjIuBBmwMbz6o","observedHarness":"APEX-Agents Harness","rawScore":54.3,"zScore":0.184049,"seModel":"upstream_se","seRaw":5.6,"ciHalfWidth":10.976,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:donMeEKBZZdqpQsojKWJEm","observedHarness":"Direct API","rawScore":60.7,"zScore":0.62537,"seModel":"upstream_se","seRaw":1.545282,"ciHalfWidth":3.0288,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-22","publishedDate":"2026-09-22","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-6 Sol (Max)","observedHarness":"Direct API","rawScore":1441.82,"zScore":-0.946025,"seModel":"upstream_ci","ciHalfWidth":4.4048,"ciLower":1437.4147,"ciUpper":1446.2243,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:GPT 6 Sol (Max reasoning)","observedHarness":"Direct API","rawScore":19.5,"zScore":0.10413,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:GPT 6 Sol (Max reasoning)","observedHarness":"Direct API","rawScore":7.2,"zScore":-0.529475,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:GPT 6 Sol (Max reasoning)","observedHarness":"Direct API","rawScore":17.1,"zScore":-0.545762,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:openai/gpt-6-sol","observedHarness":"Direct API","rawScore":28.85,"zScore":-0.881677,"seModel":"upstream_se","seRaw":3.149,"ciHalfWidth":6.172,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.897717,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:openai/gpt-6-sol","observedHarness":"Direct API","rawScore":47.07,"zScore":0.191885,"seModel":"upstream_se","seRaw":2.119,"ciHalfWidth":4.1532,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.085827,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:openai/gpt-6-sol","observedHarness":"Direct API","rawScore":30.18,"zScore":-0.08811,"seModel":"upstream_se","seRaw":3.088,"ciHalfWidth":6.0525,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.460931,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:openai/gpt-6-sol","observedHarness":"Vals Tax Agent","rawScore":15,"zScore":-1.329891,"seModel":"upstream_se","seRaw":2.575,"ciHalfWidth":5.047,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.277613,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-6.1-terra:none:direct:native","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.390549,"shrinkageFactor":0.988607,"score":44.14,"se":4.01,"samplingSe":1.517,"residualSe":3.7139,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.225896,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.0005,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:0d3c777f-6817-42d7-9bb9-223314bc5a57","observedHarness":"Direct API","rawScore":40.73,"zScore":-0.023156,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:0d3c777f-6817-42d7-9bb9-223314bc5a57","observedHarness":"Stirrup","rawScore":33.5,"zScore":-0.235226,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:0d3c777f-6817-42d7-9bb9-223314bc5a57","observedHarness":"Direct API","rawScore":23.2,"zScore":0.485164,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:0d3c777f-6817-42d7-9bb9-223314bc5a57","observedHarness":"Direct API","rawScore":14.5,"zScore":-0.76571,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:0d3c777f-6817-42d7-9bb9-223314bc5a57","observedHarness":"mini-SWE-agent","rawScore":24.29,"zScore":0.129791,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"google:gemini-3.1-pro:high:direct:native","provider":"google","modelSlug":"gemini-3.1-pro","displayName":"Gemini 3.1 Pro","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.410811,"shrinkageFactor":0.996385,"score":43.84,"se":1.68,"samplingSe":0.8545,"residualSe":1.4462,"coverageCount":16,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.316496,"costLogSd":1.0447,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4.5,"inputPer1mUsd":2,"outputPer1mUsd":12,"contextWindow":1048576,"releaseDate":"2026-02-19","paretoProb":0.0128,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:bbd93ebe-80da-4594-bb19-61e69d0331df","observedHarness":"AA Analyst Harness","rawScore":41.25,"zScore":-0.027834,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recPuGr6uc4AJrsxK","observedHarness":"APEX-Agents Harness","rawScore":35.1,"zScore":-0.970425,"seModel":"upstream_se","seRaw":5,"ciHalfWidth":9.8,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:VGeF8Xac4WcSAGgSXVdEBi","observedHarness":"Direct API","rawScore":73.5,"zScore":1.535969,"seModel":"upstream_se","seRaw":1.396316,"ciHalfWidth":2.7368,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-10","publishedDate":"2026-08-10","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:bbd93ebe-80da-4594-bb19-61e69d0331df","observedHarness":"Direct API","rawScore":17.8,"zScore":-0.152155,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:bbd93ebe-80da-4594-bb19-61e69d0331df","observedHarness":"Direct API","rawScore":31.4,"zScore":0.155589,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Gemini 3.1 Pro Preview","observedHarness":"Direct API","rawScore":1471.6,"zScore":0.454532,"seModel":"upstream_ci","ciHalfWidth":2.2736,"ciLower":1469.3282,"ciUpper":1473.8754,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:bbd93ebe-80da-4594-bb19-61e69d0331df","observedHarness":"Direct API","rawScore":11.11,"zScore":-1.106064,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:Gemini 3.1 Pro (one-pass-agent)","observedHarness":"Scale one-pass-agent","rawScore":41.87,"zScore":-0.663884,"seModel":"upstream_ci","ciHalfWidth":1.2163,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-03-23","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Gemini 3.1 Pro (one-pass-agent)","observedHarness":"Scale one-pass-agent","rawScore":44.02,"zScore":-1.094055,"seModel":"upstream_ci","ciHalfWidth":1.1208,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-03-23","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Gemini 3.1 Pro (High reasoning)","observedHarness":"Direct API","rawScore":0.3,"zScore":-1.360379,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Gemini 3.1 Pro (High reasoning)","observedHarness":"Direct API","rawScore":0.4,"zScore":-1.050452,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:Gemini 3.1 Pro (High reasoning)","observedHarness":"Direct API","rawScore":10.3,"zScore":-1.432662,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:google/gemini-3.1-pro-preview","observedHarness":"Direct API","rawScore":20.67,"zScore":-1.607285,"seModel":"upstream_se","seRaw":2.815,"ciHalfWidth":5.5174,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.100145,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:google/gemini-3.1-pro-preview","observedHarness":"Direct API","rawScore":59.06,"zScore":1.943094,"seModel":"upstream_se","seRaw":1.996,"ciHalfWidth":3.9122,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.024714,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:google/gemini-3.1-pro-preview","observedHarness":"Direct API","rawScore":20.72,"zScore":-1.086798,"seModel":"upstream_se","seRaw":2.726,"ciHalfWidth":5.343,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.065147,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:google/gemini-3.1-pro-preview","observedHarness":"Vals Tax Agent","rawScore":9.29,"zScore":-1.8332,"seModel":"upstream_se","seRaw":2.092,"ciHalfWidth":4.1003,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.301555,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"anthropic:claude-opus-4.5:max:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.5","displayName":"Claude Opus 4.5","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.430298,"shrinkageFactor":0.986185,"score":43.55,"se":6.55,"samplingSe":1.6705,"residualSe":6.3341,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":14.827231,"costLogSd":1.0558,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":200000,"releaseDate":"2025-11-24","paretoProb":0.0035,"benchmarks":[{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:Claude Opus 4.5 (one-pass-agent) max","observedHarness":"Scale one-pass-agent","rawScore":46.16,"zScore":-0.094277,"seModel":"upstream_ci","ciHalfWidth":0.27,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2025-11-26","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Claude Opus 4.5 (one-pass-agent) max","observedHarness":"Scale one-pass-agent","rawScore":44.21,"zScore":-1.057018,"seModel":"upstream_ci","ciHalfWidth":0.34,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2025-11-26","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"}]},{"id":"anthropic:claude-sonnet-5:max:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-5","displayName":"Claude Sonnet 5","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.448614,"shrinkageFactor":0.9873,"score":43.27,"se":3.54,"samplingSe":1.6016,"residualSe":3.1595,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.487251,"costLogSd":1.048,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-07-15","paretoProb":0.0074,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:LPo74T9L3CawnKX5LLUrSZ","observedHarness":"Direct API","rawScore":33.7,"zScore":-1.295424,"seModel":"upstream_se","seRaw":1.495509,"ciHalfWidth":2.9312,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Claude Sonnet 5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":4.5,"zScore":-1.040017,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Claude Sonnet 5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":2.8,"zScore":-0.866578,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:Claude Sonnet 5 (Adaptive/Max)","observedHarness":"Direct API","rawScore":16.9,"zScore":-0.571847,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:anthropic/claude-sonnet-5","observedHarness":"Direct API","rawScore":41.83,"zScore":0.269716,"seModel":"upstream_se","seRaw":3.428,"ciHalfWidth":6.7189,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.719446,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:anthropic/claude-sonnet-5","observedHarness":"Direct API","rawScore":47.54,"zScore":0.260531,"seModel":"upstream_se","seRaw":2.274,"ciHalfWidth":4.457,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.278799,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:anthropic/claude-sonnet-5","observedHarness":"Vals Tax Agent","rawScore":29.37,"zScore":-0.063245,"seModel":"upstream_se","seRaw":3.248,"ciHalfWidth":6.3661,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.794774,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"anthropic:claude-sonnet-5.5:low:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"low","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.476699,"shrinkageFactor":0.988836,"score":42.85,"se":3.66,"samplingSe":1.5017,"residualSe":3.3426,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.511342,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.0196,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:bbc2ffea-cf6b-43be-8c41-1769347e234d","observedHarness":"Direct API","rawScore":46,"zScore":0.353024,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:bbc2ffea-cf6b-43be-8c41-1769347e234d","observedHarness":"Stirrup","rawScore":26,"zScore":-0.622044,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:bbc2ffea-cf6b-43be-8c41-1769347e234d","observedHarness":"Direct API","rawScore":16,"zScore":-0.364594,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:bbc2ffea-cf6b-43be-8c41-1769347e234d","observedHarness":"Direct API","rawScore":20.6,"zScore":-0.43317,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:bbc2ffea-cf6b-43be-8c41-1769347e234d","observedHarness":"Direct API","rawScore":19.44,"zScore":-0.174693,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:bbc2ffea-cf6b-43be-8c41-1769347e234d","observedHarness":"mini-SWE-agent","rawScore":10.48,"zScore":-0.503819,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"openai:gpt-5.6-terra:xhigh:direct:native","provider":"openai","modelSlug":"gpt-5.6-terra","displayName":"GPT-5.6 Terra","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.484201,"shrinkageFactor":0.987754,"score":42.74,"se":4.7,"samplingSe":1.5728,"residualSe":4.4297,"coverageCount":4,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":5.29237,"costLogSd":1.0526,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4.5,"inputPer1mUsd":2,"outputPer1mUsd":12,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0004,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-5.6 Terra (xHigh)","observedHarness":"Direct API","rawScore":1469.36,"zScore":0.349185,"seModel":"upstream_ci","ciHalfWidth":3.2313,"ciLower":1466.1281,"ciUpper":1472.5906,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:GPT 5.6 Terra (xHigh reasoning)","observedHarness":"Direct API","rawScore":3.3,"zScore":-1.131549,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:GPT 5.6 Terra (xHigh reasoning)","observedHarness":"Direct API","rawScore":3.6,"zScore":-0.805286,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:openai/gpt-5.6-terra","observedHarness":"Direct API","rawScore":43.41,"zScore":-0.342679,"seModel":"upstream_se","seRaw":2.173,"ciHalfWidth":4.2591,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.046558,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"}]},{"id":"zhipu:glm-5.3-flash:max:direct:native","provider":"zhipu","modelSlug":"glm-5.3-flash","displayName":"GLM-5.3 Flash","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.530737,"shrinkageFactor":0.995048,"score":42.04,"se":1.76,"samplingSe":1.0001,"residualSe":1.4514,"coverageCount":17,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.58331,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2375,"inputPer1mUsd":0.15,"outputPer1mUsd":0.5,"contextWindow":1048576,"releaseDate":"2026-09-28","paretoProb":0.6663,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:19496b81-9f41-4214-a77a-1df803b3c5ae","observedHarness":"AA Analyst Harness","rawScore":25,"zScore":-1.2109,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_mka","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_mka:19496b81-9f41-4214-a77a-1df803b3c5ae","observedHarness":"Stirrup","rawScore":36.24,"zScore":-0.510989,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/mka"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:19496b81-9f41-4214-a77a-1df803b3c5ae","observedHarness":"Direct API","rawScore":28.2,"zScore":-0.917566,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"aa_vials","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_vials:19496b81-9f41-4214-a77a-1df803b3c5ae","observedHarness":"Stirrup","rawScore":39.75,"zScore":-0.216013,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/vials-aa"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:19496b81-9f41-4214-a77a-1df803b3c5ae","observedHarness":"Stirrup","rawScore":24.5,"zScore":-0.699407,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:19496b81-9f41-4214-a77a-1df803b3c5ae","observedHarness":"Direct API","rawScore":15.4,"zScore":-0.435407,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdpval_aa","benchmarkVersion":"2.1","sourceId":"artificial-analysis","sourceRowId":"aa:gdpval_aa:19496b81-9f41-4214-a77a-1df803b3c5ae","observedHarness":"Stirrup","rawScore":1648.28,"zScore":0.144007,"seModel":"upstream_ci","ciHalfWidth":16.41,"ciLower":1631.87,"ciUpper":1664.69,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:19496b81-9f41-4214-a77a-1df803b3c5ae","observedHarness":"Direct API","rawScore":18.2,"zScore":-0.564006,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GLM-5.3-Flash","observedHarness":"Direct API","rawScore":1461.24,"zScore":-0.0327,"seModel":"upstream_ci","ciHalfWidth":3.4333,"ciLower":1457.8117,"ciUpper":1464.6782,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:19496b81-9f41-4214-a77a-1df803b3c5ae","observedHarness":"Direct API","rawScore":11.67,"zScore":-1.043451,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:GLM 5.3 Flash (Max reasoning)","observedHarness":"Direct API","rawScore":7.3,"zScore":-0.826443,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:GLM 5.3 Flash (Max reasoning)","observedHarness":"Direct API","rawScore":1.2,"zScore":-0.98916,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:19496b81-9f41-4214-a77a-1df803b3c5ae","observedHarness":"mini-SWE-agent","rawScore":4.76,"zScore":-0.766256,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:zai/glm-5.3-flash","observedHarness":"Direct API","rawScore":66.25,"zScore":-1.313421,"seModel":"upstream_se","seRaw":2.685,"ciHalfWidth":5.2626,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.426065,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:zai/glm-5.3-flash","observedHarness":"Vals Finance Agent","rawScore":60.59,"zScore":-0.66126,"seModel":"upstream_se","seRaw":2.032,"ciHalfWidth":3.9827,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.072473,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:zai/glm-5.3-flash","observedHarness":"Direct API","rawScore":45.19,"zScore":0.567765,"seModel":"upstream_se","seRaw":3.459,"ciHalfWidth":6.7796,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.240392,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:zai/glm-5.3-flash","observedHarness":"Vals Tax Agent","rawScore":31.55,"zScore":0.128912,"seModel":"upstream_se","seRaw":3.35,"ciHalfWidth":6.566,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.138062,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-6-luna:max:direct:native","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.535159,"shrinkageFactor":0.995738,"score":41.97,"se":1.27,"samplingSe":0.9278,"residualSe":0.8633,"coverageCount":23,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.151956,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0374,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","observedHarness":"AA Analyst Harness","rawScore":43.75,"zScore":0.154177,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_briefcase:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","observedHarness":"Stirrup","rawScore":1335.97,"zScore":-1.404205,"seModel":"upstream_ci","ciHalfWidth":8.275,"ciLower":1328.22,"ciUpper":1344.77,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase"},{"benchmarkId":"aa_mka","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_mka:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","observedHarness":"Stirrup","rawScore":36.75,"zScore":-0.452273,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/mka"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","observedHarness":"Direct API","rawScore":39.63,"zScore":-0.101676,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"aa_vials","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_vials:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","observedHarness":"Stirrup","rawScore":27.33,"zScore":-1.03687,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/vials-aa"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","observedHarness":"Stirrup","rawScore":42.5,"zScore":0.228956,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:rec1U23OkfwoEOepV","observedHarness":"APEX-Agents Harness","rawScore":44.3,"zScore":-0.41724,"seModel":"upstream_se","seRaw":5.5,"ciHalfWidth":10.78,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:HX9sRVWJtZiNvneALnVzs8","observedHarness":"Direct API","rawScore":41.4,"zScore":-0.747642,"seModel":"upstream_se","seRaw":1.558354,"ciHalfWidth":3.0544,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-22","publishedDate":"2026-09-22","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","observedHarness":"Direct API","rawScore":22.8,"zScore":0.437955,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdpval_aa","benchmarkVersion":"2.1","sourceId":"artificial-analysis","sourceRowId":"aa:gdpval_aa:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","observedHarness":"Stirrup","rawScore":1443.29,"zScore":-1.160525,"seModel":"upstream_ci","ciHalfWidth":19.735,"ciLower":1423.55,"ciUpper":1463.02,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","observedHarness":"Direct API","rawScore":22.2,"zScore":-0.345947,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-6 Luna (Max)","observedHarness":"Direct API","rawScore":1466.04,"zScore":0.193045,"seModel":"upstream_ci","ciHalfWidth":2.8012,"ciLower":1463.2343,"ciUpper":1468.8368,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","observedHarness":"Direct API","rawScore":26.11,"zScore":0.571074,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:GPT 6 Luna (Max reasoning)","observedHarness":"Direct API","rawScore":6.8,"zScore":-0.864581,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:GPT 6 Luna (Max reasoning)","observedHarness":"Direct API","rawScore":1.2,"zScore":-0.98916,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:GPT 6 Luna (Max reasoning)","observedHarness":"Direct API","rawScore":14.9,"zScore":-0.8327,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:6fb13851-40ce-4bda-ad3f-6b38e0c5daa7","observedHarness":"mini-SWE-agent","rawScore":8.57,"zScore":-0.591451,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:openai/gpt-6-luna","observedHarness":"Direct API","rawScore":69.95,"zScore":-0.298157,"seModel":"upstream_se","seRaw":2.542,"ciHalfWidth":4.9823,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.141708,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:openai/gpt-6-luna","observedHarness":"Vals Finance Agent","rawScore":56.84,"zScore":-1.735509,"seModel":"upstream_se","seRaw":2.123,"ciHalfWidth":4.1611,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.120919,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:openai/gpt-6-luna","observedHarness":"Direct API","rawScore":30.29,"zScore":-0.753942,"seModel":"upstream_se","seRaw":3.194,"ciHalfWidth":6.2602,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.440296,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:openai/gpt-6-luna","observedHarness":"Direct API","rawScore":44.69,"zScore":-0.155728,"seModel":"upstream_se","seRaw":2.303,"ciHalfWidth":4.5139,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.007323,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:openai/gpt-6-luna","observedHarness":"Direct API","rawScore":19.37,"zScore":-1.229317,"seModel":"upstream_se","seRaw":2.658,"ciHalfWidth":5.2097,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.025854,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:openai/gpt-6-luna","observedHarness":"Vals Tax Agent","rawScore":19.95,"zScore":-0.893572,"seModel":"upstream_se","seRaw":2.866,"ciHalfWidth":5.6174,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.190956,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"deepseek:deepseek-v4-pro-0813:max:direct:native","provider":"deepseek","modelSlug":"deepseek-v4-pro-0813","displayName":"DeepSeek V4 Pro (0813)","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.548471,"shrinkageFactor":0.993669,"score":41.77,"se":2.3,"samplingSe":1.1308,"residualSe":1.9973,"coverageCount":11,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.769134,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.98,"inputPer1mUsd":1.32,"outputPer1mUsd":3.96,"contextWindow":1048576,"releaseDate":"2026-08-13","paretoProb":0.003,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:83173329-c09a-41f1-a028-a282a5f908d7","observedHarness":"Direct API","rawScore":45.2,"zScore":0.295919,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:83173329-c09a-41f1-a028-a282a5f908d7","observedHarness":"Stirrup","rawScore":23.5,"zScore":-0.750983,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:2xfFf6KZAp2gUWYhHGkFgs","observedHarness":"Direct API","rawScore":52.91,"zScore":0.071186,"seModel":"upstream_se","seRaw":1.580839,"ciHalfWidth":3.0984,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:83173329-c09a-41f1-a028-a282a5f908d7","observedHarness":"Direct API","rawScore":11.4,"zScore":-0.907494,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:83173329-c09a-41f1-a028-a282a5f908d7","observedHarness":"Direct API","rawScore":19.5,"zScore":-0.493136,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:83173329-c09a-41f1-a028-a282a5f908d7","observedHarness":"mini-SWE-agent","rawScore":5.71,"zScore":-0.722669,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:deepseek/deepseek-v4-pro-0813","observedHarness":"Direct API","rawScore":65.13,"zScore":-1.620744,"seModel":"upstream_se","seRaw":2.35,"ciHalfWidth":4.606,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.08141,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:deepseek/deepseek-v4-pro-0813","observedHarness":"Vals Finance Agent","rawScore":57.12,"zScore":-1.655298,"seModel":"upstream_se","seRaw":2.083,"ciHalfWidth":4.0827,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.82234,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:deepseek/deepseek-v4-pro-0813","observedHarness":"Direct API","rawScore":40.87,"zScore":0.184559,"seModel":"upstream_se","seRaw":3.417,"ciHalfWidth":6.6973,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.088017,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:deepseek/deepseek-v4-pro-0813","observedHarness":"Direct API","rawScore":42.47,"zScore":-0.479972,"seModel":"upstream_se","seRaw":2.16,"ciHalfWidth":4.2336,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.06106,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:deepseek/deepseek-v4-pro-0813","observedHarness":"Vals Tax Agent","rawScore":26.76,"zScore":-0.293304,"seModel":"upstream_se","seRaw":3.093,"ciHalfWidth":6.0623,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.729136,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-6-luna:xhigh:direct:native","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.574236,"shrinkageFactor":0.99117,"score":41.39,"se":3.19,"samplingSe":1.3355,"residualSe":2.8975,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.104093,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0728,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:19813eb2-460a-475c-af65-810bb8660fec","observedHarness":"Direct API","rawScore":38.6,"zScore":-0.175199,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:19813eb2-460a-475c-af65-810bb8660fec","observedHarness":"Stirrup","rawScore":37,"zScore":-0.05471,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:19813eb2-460a-475c-af65-810bb8660fec","observedHarness":"Direct API","rawScore":18.6,"zScore":-0.057737,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:19813eb2-460a-475c-af65-810bb8660fec","observedHarness":"Direct API","rawScore":18.9,"zScore":-0.525845,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:19813eb2-460a-475c-af65-810bb8660fec","observedHarness":"Direct API","rawScore":20.56,"zScore":-0.049467,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:GPT 6 Luna (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":42.85,"zScore":-1.322127,"seModel":"upstream_ci","ciHalfWidth":1.69,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-25","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:19813eb2-460a-475c-af65-810bb8660fec","observedHarness":"mini-SWE-agent","rawScore":5.24,"zScore":-0.744233,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"openai:gpt-5.2:xhigh:direct:native","provider":"openai","modelSlug":"gpt-5.2","displayName":"GPT-5.2","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.590306,"shrinkageFactor":0.973273,"score":41.15,"se":6.66,"samplingSe":2.3235,"residualSe":6.2368,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.006963,"costLogSd":1.0533,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4.8125,"inputPer1mUsd":1.75,"outputPer1mUsd":14,"contextWindow":400000,"releaseDate":"2025-12-11","paretoProb":0.0003,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:E3325ytkSNYP8ez53VA4fE","observedHarness":"Direct API","rawScore":37.1,"zScore":-1.053546,"seModel":"upstream_se","seRaw":1.528374,"ciHalfWidth":2.9956,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:openai/gpt-5.2-2025-12-11","observedHarness":"Direct API","rawScore":49.75,"zScore":0.583315,"seModel":"upstream_se","seRaw":2.262,"ciHalfWidth":4.4335,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.018852,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"}]},{"id":"spacexai:grok-4.20:none:direct:native","provider":"spacexai","modelSlug":"grok-4.20","displayName":"Grok 4.20","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.596158,"shrinkageFactor":0.989118,"score":41.06,"se":9.26,"samplingSe":1.4826,"residualSe":9.1419,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.409226,"costLogSd":1.0415,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.5625,"inputPer1mUsd":1.25,"outputPer1mUsd":2.5,"contextWindow":2000000,"releaseDate":"2026-03-09","paretoProb":0.7936,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Grok 4.20 Beta 1","observedHarness":"Direct API","rawScore":1452.6,"zScore":-0.43904,"seModel":"upstream_ci","ciHalfWidth":4.1417,"ciLower":1448.4618,"ciUpper":1456.7453,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-sonnet-4.6:max:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-4.6","displayName":"Claude Sonnet 4.6","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.602517,"shrinkageFactor":0.979181,"score":40.96,"se":5.41,"samplingSe":2.0506,"residualSe":5.0064,"coverageCount":3,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":9.345558,"costLogSd":1.0507,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1000000,"releaseDate":"2026-02-17","paretoProb":0.0016,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:Zs5fCvNzaz8LhqgwDSAztU","observedHarness":"Direct API","rawScore":32.8,"zScore":-1.35945,"seModel":"upstream_se","seRaw":1.485384,"ciHalfWidth":2.9114,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:anthropic/claude-sonnet-4-6","observedHarness":"Direct API","rawScore":38.46,"zScore":-0.02922,"seModel":"upstream_se","seRaw":3.381,"ciHalfWidth":6.6268,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.302948,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:anthropic/claude-sonnet-4-6","observedHarness":"Vals Tax Agent","rawScore":25.84,"zScore":-0.374397,"seModel":"upstream_se","seRaw":3.154,"ciHalfWidth":6.1818,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.193501,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"deepseek:deepseek-v4.1-flash-1017:high:direct:native","provider":"deepseek","modelSlug":"deepseek-v4.1-flash-1017","displayName":"DeepSeek V4.1 Flash (1017)","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.609985,"shrinkageFactor":0.991429,"score":40.85,"se":3.95,"samplingSe":1.3158,"residualSe":3.7245,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.456194,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.525,"inputPer1mUsd":0.3,"outputPer1mUsd":1.2,"contextWindow":1048576,"releaseDate":"2026-10-17","paretoProb":0.9995,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:c3b66080-fdcb-48e4-8ade-6baf77921c7c","observedHarness":"Direct API","rawScore":38.93,"zScore":-0.151643,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:c3b66080-fdcb-48e4-8ade-6baf77921c7c","observedHarness":"Stirrup","rawScore":27.5,"zScore":-0.54468,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:c3b66080-fdcb-48e4-8ade-6baf77921c7c","observedHarness":"Direct API","rawScore":14.2,"zScore":-0.577033,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:c3b66080-fdcb-48e4-8ade-6baf77921c7c","observedHarness":"Direct API","rawScore":20.8,"zScore":-0.422267,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:c3b66080-fdcb-48e4-8ade-6baf77921c7c","observedHarness":"mini-SWE-agent","rawScore":10.48,"zScore":-0.503819,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"zhipu:glm-5:thinking:direct:native","provider":"zhipu","modelSlug":"glm-5","displayName":"GLM-5","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.62609,"shrinkageFactor":0.990598,"score":40.61,"se":9.26,"samplingSe":1.3781,"residualSe":9.1556,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.989151,"costLogSd":1.033,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.55,"inputPer1mUsd":1,"outputPer1mUsd":3.2,"contextWindow":204800,"releaseDate":"2026-02-01","paretoProb":0.6386,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GLM-5","observedHarness":"Direct API","rawScore":1451.97,"zScore":-0.468669,"seModel":"upstream_ci","ciHalfWidth":3.847,"ciLower":1448.1181,"ciUpper":1455.812,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-6-luna:high:direct:native","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.649028,"shrinkageFactor":0.989522,"score":40.26,"se":3.65,"samplingSe":1.4548,"residualSe":3.345,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.072948,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0674,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:7d1229bc-deea-4717-ba5b-f59a35d991e3","observedHarness":"Direct API","rawScore":39,"zScore":-0.146646,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:7d1229bc-deea-4717-ba5b-f59a35d991e3","observedHarness":"Stirrup","rawScore":31,"zScore":-0.364165,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:7d1229bc-deea-4717-ba5b-f59a35d991e3","observedHarness":"Direct API","rawScore":18.2,"zScore":-0.104946,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:7d1229bc-deea-4717-ba5b-f59a35d991e3","observedHarness":"Direct API","rawScore":15.6,"zScore":-0.705744,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:7d1229bc-deea-4717-ba5b-f59a35d991e3","observedHarness":"Direct API","rawScore":19.44,"zScore":-0.174693,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:7d1229bc-deea-4717-ba5b-f59a35d991e3","observedHarness":"mini-SWE-agent","rawScore":2.86,"zScore":-0.853429,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"anthropic:claude-sonnet-5:high:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-5","displayName":"Claude Sonnet 5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.65291,"shrinkageFactor":0.993266,"score":40.21,"se":9.25,"samplingSe":1.1663,"residualSe":9.1802,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.700253,"costLogSd":1.0442,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-07-15","paretoProb":0.0315,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Claude Sonnet 5 (High)","observedHarness":"Direct API","rawScore":1451.42,"zScore":-0.494536,"seModel":"upstream_ci","ciHalfWidth":3.2514,"ciLower":1448.1705,"ciUpper":1454.6733,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"deepseek:deepseek-v4-pro-0813:high:direct:native","provider":"deepseek","modelSlug":"deepseek-v4-pro-0813","displayName":"DeepSeek V4 Pro (0813)","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.653577,"shrinkageFactor":0.994736,"score":40.2,"se":3.38,"samplingSe":1.0311,"residualSe":3.2185,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.848531,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.98,"inputPer1mUsd":1.32,"outputPer1mUsd":3.96,"contextWindow":1048576,"releaseDate":"2026-08-13","paretoProb":0.0076,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:e94c7595-a320-4c88-8f07-d9fd50fc4595","observedHarness":"Direct API","rawScore":45.13,"zScore":0.290922,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:e94c7595-a320-4c88-8f07-d9fd50fc4595","observedHarness":"Stirrup","rawScore":28.5,"zScore":-0.493104,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:e94c7595-a320-4c88-8f07-d9fd50fc4595","observedHarness":"Direct API","rawScore":10.4,"zScore":-1.025516,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:e94c7595-a320-4c88-8f07-d9fd50fc4595","observedHarness":"Direct API","rawScore":20.5,"zScore":-0.438622,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:DeepSeek-V4-Pro (High, 20260813)","observedHarness":"Direct API","rawScore":1452.87,"zScore":-0.426342,"seModel":"upstream_ci","ciHalfWidth":5.546,"ciLower":1447.3287,"ciUpper":1458.4206,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:e94c7595-a320-4c88-8f07-d9fd50fc4595","observedHarness":"mini-SWE-agent","rawScore":1.9,"zScore":-0.897474,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"google:gemini-3-flash:minimal:direct:native","provider":"google","modelSlug":"gemini-3-flash","displayName":"Gemini 3 Flash","effort":"minimal","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.666715,"shrinkageFactor":0.995849,"score":40,"se":9.25,"samplingSe":0.9156,"residualSe":9.2041,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.440275,"costLogSd":1.0412,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.125,"inputPer1mUsd":0.5,"outputPer1mUsd":3,"contextWindow":1048576,"releaseDate":"2025-12-15","paretoProb":0.4878,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Gemini 3 Flash (Minimal)","observedHarness":"Direct API","rawScore":1451.15,"zScore":-0.507234,"seModel":"upstream_ci","ciHalfWidth":2.5493,"ciLower":1448.6037,"ciUpper":1453.7022,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-sonnet-4.5:thinking:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-4.5","displayName":"Claude Sonnet 4.5","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.68125,"shrinkageFactor":0.91336,"score":39.78,"se":9.49,"samplingSe":4.1834,"residualSe":8.522,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.810458,"costLogSd":1.0326,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1000000,"releaseDate":"2025-09-29","paretoProb":0.0291,"benchmarks":[{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:anthropic/claude-sonnet-4-5-20250929-thinking","observedHarness":"Direct API","rawScore":44.13,"zScore":-0.237519,"seModel":"upstream_se","seRaw":1.998,"ciHalfWidth":3.9161,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.101495,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"}]},{"id":"anthropic:claude-haiku-5.5:high:direct:native","provider":"anthropic","modelSlug":"claude-haiku-5.5","displayName":"Claude Haiku 5.5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.704995,"shrinkageFactor":0.991491,"score":39.43,"se":3.2,"samplingSe":1.311,"residualSe":2.9215,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.15856,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1000000,"releaseDate":"2026-10-07","paretoProb":0.4802,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:20829974-af06-497e-9f42-77fb958d43b9","observedHarness":"Direct API","rawScore":31.83,"zScore":-0.658451,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:20829974-af06-497e-9f42-77fb958d43b9","observedHarness":"Stirrup","rawScore":45,"zScore":0.357896,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:20829974-af06-497e-9f42-77fb958d43b9","observedHarness":"Direct API","rawScore":17.2,"zScore":-0.222968,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:20829974-af06-497e-9f42-77fb958d43b9","observedHarness":"Direct API","rawScore":23.7,"zScore":-0.264174,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Claude Haiku 5.5 (High)","observedHarness":"Direct API","rawScore":1424.39,"zScore":-1.76576,"seModel":"upstream_ci","ciHalfWidth":4.8462,"ciLower":1419.5436,"ciUpper":1429.2361,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:20829974-af06-497e-9f42-77fb958d43b9","observedHarness":"Direct API","rawScore":15,"zScore":-0.671126,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:20829974-af06-497e-9f42-77fb958d43b9","observedHarness":"mini-SWE-agent","rawScore":10,"zScore":-0.525842,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"zhipu:glm-5.2:max:direct:native","provider":"zhipu","modelSlug":"glm-5.2","displayName":"GLM-5.2","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.718324,"shrinkageFactor":0.991763,"score":39.23,"se":5.26,"samplingSe":1.2899,"residualSe":5.0988,"coverageCount":3,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.211024,"costLogSd":1.0457,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":1048576,"releaseDate":"2026-07-02","paretoProb":0.2462,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:WgjFZzojpiv54XqUhvfuUG","observedHarness":"Direct API","rawScore":34.2,"zScore":-1.259854,"seModel":"upstream_se","seRaw":1.500871,"ciHalfWidth":2.9417,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GLM-5.2 (Max)","observedHarness":"Direct API","rawScore":1462.59,"zScore":0.030791,"seModel":"upstream_ci","ciHalfWidth":3.2378,"ciLower":1459.3562,"ciUpper":1465.8317,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:zai/glm-5.2","observedHarness":"Direct API","rawScore":31.25,"zScore":-0.668785,"seModel":"upstream_se","seRaw":3.222,"ciHalfWidth":6.3151,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.889209,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"}]},{"id":"anthropic:claude-haiku-5.5:medium:direct:native","provider":"anthropic","modelSlug":"claude-haiku-5.5","displayName":"Claude Haiku 5.5","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.738947,"shrinkageFactor":0.989652,"score":38.92,"se":3.64,"samplingSe":1.4458,"residualSe":3.3454,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.091765,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1000000,"releaseDate":"2026-10-07","paretoProb":0.8358,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:af26fff4-8120-41be-a5f0-86aaea023f20","observedHarness":"Direct API","rawScore":31.73,"zScore":-0.665589,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:af26fff4-8120-41be-a5f0-86aaea023f20","observedHarness":"Stirrup","rawScore":30.5,"zScore":-0.389953,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:af26fff4-8120-41be-a5f0-86aaea023f20","observedHarness":"Direct API","rawScore":15.2,"zScore":-0.459011,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:af26fff4-8120-41be-a5f0-86aaea023f20","observedHarness":"Direct API","rawScore":16.8,"zScore":-0.640326,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:af26fff4-8120-41be-a5f0-86aaea023f20","observedHarness":"Direct API","rawScore":16.11,"zScore":-0.547018,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:af26fff4-8120-41be-a5f0-86aaea023f20","observedHarness":"mini-SWE-agent","rawScore":7.14,"zScore":-0.65706,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"openai:gpt-5.6-luna:max:direct:native","provider":"openai","modelSlug":"gpt-5.6-luna","displayName":"GPT-5.6 Luna","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.751747,"shrinkageFactor":0.989974,"score":38.72,"se":3.43,"samplingSe":1.4231,"residualSe":3.1221,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.970791,"costLogSd":1.0515,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.45,"inputPer1mUsd":0.2,"outputPer1mUsd":1.2,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0172,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:CeDghdBk8CqtiFSbUzxNH5","observedHarness":"Direct API","rawScore":41,"zScore":-0.776098,"seModel":"upstream_se","seRaw":1.556092,"ciHalfWidth":3.0499,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-10","publishedDate":"2026-08-10","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:GPT 5.6 Luna (Max reasoning)","observedHarness":"Direct API","rawScore":4.3,"zScore":-1.055273,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:GPT 5.6 Luna (Max reasoning)","observedHarness":"Direct API","rawScore":1.6,"zScore":-0.958515,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:openai/gpt-5.6-luna","observedHarness":"Direct API","rawScore":36.54,"zScore":-0.199534,"seModel":"upstream_se","seRaw":3.347,"ciHalfWidth":6.5601,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.845008,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:openai/gpt-5.6-luna","observedHarness":"Direct API","rawScore":42.39,"zScore":-0.491656,"seModel":"upstream_se","seRaw":2.27,"ciHalfWidth":4.4492,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.01597,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:openai/gpt-5.6-luna","observedHarness":"Direct API","rawScore":14.41,"zScore":-1.752942,"seModel":"upstream_se","seRaw":2.363,"ciHalfWidth":4.6315,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.109246,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:openai/gpt-5.6-luna","observedHarness":"Vals Tax Agent","rawScore":24.82,"zScore":-0.464305,"seModel":"upstream_se","seRaw":3.113,"ciHalfWidth":6.1015,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.465359,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-6-luna:medium:direct:native","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.755001,"shrinkageFactor":0.990431,"score":38.67,"se":3.63,"samplingSe":1.3903,"residualSe":3.348,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.042202,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.8535,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:36667da0-9222-4967-adbb-f7efa15ad213","observedHarness":"Direct API","rawScore":38.7,"zScore":-0.16806,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:36667da0-9222-4967-adbb-f7efa15ad213","observedHarness":"Stirrup","rawScore":27.5,"zScore":-0.54468,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:36667da0-9222-4967-adbb-f7efa15ad213","observedHarness":"Direct API","rawScore":13.8,"zScore":-0.624242,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:36667da0-9222-4967-adbb-f7efa15ad213","observedHarness":"Direct API","rawScore":15.1,"zScore":-0.733001,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:36667da0-9222-4967-adbb-f7efa15ad213","observedHarness":"Direct API","rawScore":18.33,"zScore":-0.298801,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:36667da0-9222-4967-adbb-f7efa15ad213","observedHarness":"mini-SWE-agent","rawScore":1.43,"zScore":-0.919038,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"openai:gpt-5.4:xhigh:direct:native","provider":"openai","modelSlug":"gpt-5.4","displayName":"GPT-5.4","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.8065,"shrinkageFactor":0.975204,"score":37.9,"se":6.64,"samplingSe":2.238,"residualSe":6.2492,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.468237,"costLogSd":1.0538,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":5.625,"inputPer1mUsd":2.5,"outputPer1mUsd":15,"contextWindow":1050000,"releaseDate":"2026-03-05","paretoProb":0.0003,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:AzCkm282PLhDdLJ8DnAQQc","observedHarness":"Direct API","rawScore":45.1,"zScore":-0.484422,"seModel":"upstream_se","seRaw":1.574315,"ciHalfWidth":3.0857,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:openai/gpt-5.4-2026-03-05","observedHarness":"Direct API","rawScore":41.29,"zScore":-0.652317,"seModel":"upstream_se","seRaw":2.148,"ciHalfWidth":4.2101,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.212108,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"}]},{"id":"alibaba:qwen3.7-max:thinking:direct:native","provider":"alibaba","modelSlug":"qwen3.7-max","displayName":"Qwen3.7 Max","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.824769,"shrinkageFactor":0.984979,"score":37.63,"se":4.58,"samplingSe":1.7418,"residualSe":4.2335,"coverageCount":4,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.123992,"costLogSd":1.0308,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3.75,"inputPer1mUsd":2.5,"outputPer1mUsd":7.5,"contextWindow":1000000,"releaseDate":"2026-05-20","paretoProb":0.5572,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:DMjUrG6eZEqwcQ4SqUoViu","observedHarness":"Direct API","rawScore":55.77,"zScore":0.274648,"seModel":"upstream_se","seRaw":1.573736,"ciHalfWidth":3.0845,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:alibaba/qwen3.7-max","observedHarness":"Direct API","rawScore":25.48,"zScore":-1.180614,"seModel":"upstream_se","seRaw":3.029,"ciHalfWidth":5.9368,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.400701,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:alibaba/qwen3.7-max","observedHarness":"Direct API","rawScore":38.75,"zScore":-1.023299,"seModel":"upstream_se","seRaw":2.196,"ciHalfWidth":4.3042,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.042362,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:alibaba/qwen3.7-max","observedHarness":"Vals Tax Agent","rawScore":15.83,"zScore":-1.256731,"seModel":"upstream_se","seRaw":2.617,"ciHalfWidth":5.1293,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.435494,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"zhipu:glm-5.1:thinking:direct:native","provider":"zhipu","modelSlug":"glm-5.1","displayName":"GLM-5.1","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.911471,"shrinkageFactor":0.990444,"score":36.33,"se":4.13,"samplingSe":1.3893,"residualSe":3.8893,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.314752,"costLogSd":1.0316,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":204800,"releaseDate":"2026-04-15","paretoProb":0.3184,"benchmarks":[{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recB78P8U9128Kwmw","observedHarness":"APEX-Agents Harness","rawScore":40.9,"zScore":-0.621678,"seModel":"upstream_se","seRaw":5.4,"ciHalfWidth":10.584,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:R2fMFAtRXah864hMifdQGV","observedHarness":"Direct API","rawScore":34,"zScore":-1.274082,"seModel":"upstream_se","seRaw":1.498748,"ciHalfWidth":2.9375,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GLM-5.1","observedHarness":"Direct API","rawScore":1456.22,"zScore":-0.268791,"seModel":"upstream_ci","ciHalfWidth":2.9515,"ciLower":1453.2695,"ciUpper":1459.1724,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:zai/glm-5.1","observedHarness":"Direct API","rawScore":27.89,"zScore":-0.966834,"seModel":"upstream_se","seRaw":3.117,"ciHalfWidth":6.1093,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.385737,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:zai/glm-5.1","observedHarness":"Direct API","rawScore":41.6,"zScore":-0.60704,"seModel":"upstream_se","seRaw":2.124,"ciHalfWidth":4.163,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.024196,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"}]},{"id":"anthropic:claude-sonnet-4.5:high:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-4.5","displayName":"Claude Sonnet 4.5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.93149,"shrinkageFactor":0.982598,"score":36.03,"se":6.58,"samplingSe":1.8748,"residualSe":6.311,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":5.323777,"costLogSd":1.0457,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1000000,"releaseDate":"2025-09-29","paretoProb":0.0035,"benchmarks":[{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:Claude Sonnet 4.5 (one-pass-agent) high","observedHarness":"Scale one-pass-agent","rawScore":43.79,"zScore":-0.408955,"seModel":"upstream_ci","ciHalfWidth":0.4016,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2025-11-13","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Claude Sonnet 4.5 (one-pass-agent) high","observedHarness":"Scale one-pass-agent","rawScore":40.84,"zScore":-1.713944,"seModel":"upstream_ci","ciHalfWidth":0.398,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2025-11-13","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"}]},{"id":"zhipu:glm-5.3:low:direct:native","provider":"zhipu","modelSlug":"glm-5.3","displayName":"GLM-5.3","effort":"low","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.950669,"shrinkageFactor":0.992212,"score":35.74,"se":3.58,"samplingSe":1.2542,"residualSe":3.3541,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.57493,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":1048576,"releaseDate":"2026-09-18","paretoProb":0.0269,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:cc1e488b-8ba4-464d-875d-690166b15ad3","observedHarness":"Direct API","rawScore":31.9,"zScore":-0.653454,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:cc1e488b-8ba4-464d-875d-690166b15ad3","observedHarness":"Stirrup","rawScore":21.5,"zScore":-0.854135,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:cc1e488b-8ba4-464d-875d-690166b15ad3","observedHarness":"Direct API","rawScore":8.8,"zScore":-1.214351,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:cc1e488b-8ba4-464d-875d-690166b15ad3","observedHarness":"Direct API","rawScore":17.5,"zScore":-0.602166,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:cc1e488b-8ba4-464d-875d-690166b15ad3","observedHarness":"Direct API","rawScore":11.11,"zScore":-1.106064,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:cc1e488b-8ba4-464d-875d-690166b15ad3","observedHarness":"mini-SWE-agent","rawScore":3.81,"zScore":-0.809843,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"anthropic:claude-haiku-5.5:low:direct:native","provider":"anthropic","modelSlug":"claude-haiku-5.5","displayName":"Claude Haiku 5.5","effort":"low","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.003502,"shrinkageFactor":0.992176,"score":34.95,"se":3.58,"samplingSe":1.2571,"residualSe":3.3539,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.043758,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1000000,"releaseDate":"2026-10-07","paretoProb":0.9979,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:42908ae3-d3bd-4934-be19-fbff93d3fde8","observedHarness":"Direct API","rawScore":30.37,"zScore":-0.762668,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:42908ae3-d3bd-4934-be19-fbff93d3fde8","observedHarness":"Stirrup","rawScore":21.5,"zScore":-0.854135,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:42908ae3-d3bd-4934-be19-fbff93d3fde8","observedHarness":"Direct API","rawScore":11.2,"zScore":-0.931099,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:42908ae3-d3bd-4934-be19-fbff93d3fde8","observedHarness":"Direct API","rawScore":14.1,"zScore":-0.787516,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:42908ae3-d3bd-4934-be19-fbff93d3fde8","observedHarness":"Direct API","rawScore":12.22,"zScore":-0.981956,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:42908ae3-d3bd-4934-be19-fbff93d3fde8","observedHarness":"mini-SWE-agent","rawScore":1.9,"zScore":-0.897474,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"openai:gpt-5.1:none:direct:native","provider":"openai","modelSlug":"gpt-5.1","displayName":"GPT-5.1","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.00555,"shrinkageFactor":0.993208,"score":34.92,"se":9.25,"samplingSe":1.1713,"residualSe":9.1797,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.276104,"costLogSd":1.0418,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3.4375,"inputPer1mUsd":1.25,"outputPer1mUsd":10,"contextWindow":400000,"releaseDate":"2025-11-13","paretoProb":0.0124,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-5.1","observedHarness":"Direct API","rawScore":1443.87,"zScore":-0.849613,"seModel":"upstream_ci","ciHalfWidth":3.2653,"ciLower":1440.6019,"ciUpper":1447.1325,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-sonnet-4.6:high:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-4.6","displayName":"Claude Sonnet 4.6","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.021369,"shrinkageFactor":0.983268,"score":34.68,"se":7.11,"samplingSe":1.8384,"residualSe":6.8658,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":5.323777,"costLogSd":1.0457,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1000000,"releaseDate":"2026-02-17","paretoProb":0.0031,"benchmarks":[{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recNKXGqIBBG6Q12j","observedHarness":"APEX-Agents Harness","rawScore":43,"zScore":-0.495407,"seModel":"upstream_se","seRaw":5.5,"ciHalfWidth":10.78,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:LJyKf8Xy4dymxNHbGLwNdA","observedHarness":"Direct API","rawScore":35.5,"zScore":-1.167371,"seModel":"upstream_se","seRaw":1.513949,"ciHalfWidth":2.9673,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-10","publishedDate":"2026-08-10","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"}]},{"id":"moonshot:kimi-k2.7-code:max:direct:native","provider":"moonshot","modelSlug":"kimi-k2.7-code","displayName":"Kimi K2.7 Code","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.032856,"shrinkageFactor":0.993879,"score":34.51,"se":3.5,"samplingSe":1.112,"residualSe":3.3232,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.884912,"costLogSd":1.0458,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.7125,"inputPer1mUsd":0.95,"outputPer1mUsd":4,"contextWindow":262144,"releaseDate":"2026-06-15","paretoProb":0.4156,"benchmarks":[{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recc9OBbGet6i61HC","observedHarness":"APEX-Agents Harness","rawScore":37.6,"zScore":-0.820103,"seModel":"upstream_se","seRaw":4.8,"ciHalfWidth":9.408,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:5AijYw5bFJtSuahnbzNo6Z","observedHarness":"Direct API","rawScore":36.5,"zScore":-1.096231,"seModel":"upstream_se","seRaw":1.523178,"ciHalfWidth":2.9854,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:8d0cb231-7303-452c-9923-a9620b948475","observedHarness":"Direct API","rawScore":11.2,"zScore":-0.931099,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:8d0cb231-7303-452c-9923-a9620b948475","observedHarness":"Direct API","rawScore":14.8,"zScore":-0.749356,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:8d0cb231-7303-452c-9923-a9620b948475","observedHarness":"Direct API","rawScore":12.22,"zScore":-0.981956,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Kimi K2.7 Code (Max reasoning)","observedHarness":"Direct API","rawScore":0,"zScore":-1.383262,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Kimi K2.7 Code (Max reasoning)","observedHarness":"Direct API","rawScore":0,"zScore":-1.081097,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"}]},{"id":"openai:gpt-6-luna:low:direct:native","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"low","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.041243,"shrinkageFactor":0.992684,"score":34.38,"se":3.57,"samplingSe":1.2156,"residualSe":3.3557,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.005115,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":1,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:bf9708d8-933d-44f6-affa-b696a7a650c9","observedHarness":"Direct API","rawScore":36.1,"zScore":-0.353652,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:bf9708d8-933d-44f6-affa-b696a7a650c9","observedHarness":"Stirrup","rawScore":9.5,"zScore":-1.473044,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:bf9708d8-933d-44f6-affa-b696a7a650c9","observedHarness":"Direct API","rawScore":9,"zScore":-1.190747,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:bf9708d8-933d-44f6-affa-b696a7a650c9","observedHarness":"Direct API","rawScore":10.2,"zScore":-1.000124,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:bf9708d8-933d-44f6-affa-b696a7a650c9","observedHarness":"Direct API","rawScore":18.89,"zScore":-0.236188,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:bf9708d8-933d-44f6-affa-b696a7a650c9","observedHarness":"mini-SWE-agent","rawScore":0.95,"zScore":-0.941061,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"zhipu:glm-5.2:thinking:direct:native","provider":"zhipu","modelSlug":"glm-5.2","displayName":"GLM-5.2","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.115689,"shrinkageFactor":0.8997,"score":33.26,"se":9.53,"samplingSe":4.5011,"residualSe":8.3945,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.314752,"costLogSd":1.0316,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":1048576,"releaseDate":"2026-07-02","paretoProb":0.3527,"benchmarks":[{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:zai/glm-5.2","observedHarness":"Direct API","rawScore":40.77,"zScore":-0.728266,"seModel":"upstream_se","seRaw":2.166,"ciHalfWidth":4.2454,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.04501,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"}]},{"id":"openai:gpt-5.2:none:direct:native","provider":"openai","modelSlug":"gpt-5.2","displayName":"GPT-5.2","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.126041,"shrinkageFactor":0.995715,"score":33.11,"se":9.25,"samplingSe":0.9304,"residualSe":9.2029,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.72689,"costLogSd":1.0437,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4.8125,"inputPer1mUsd":1.75,"outputPer1mUsd":14,"contextWindow":400000,"releaseDate":"2025-12-11","paretoProb":0.0053,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-5.2","observedHarness":"Direct API","rawScore":1441.34,"zScore":-0.968599,"seModel":"upstream_ci","ciHalfWidth":2.5905,"ciLower":1438.7531,"ciUpper":1443.9341,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemini-3.5-flash-lite:minimal:direct:native","provider":"google","modelSlug":"gemini-3.5-flash-lite","displayName":"Gemini 3.5 Flash-Lite","effort":"minimal","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.220785,"shrinkageFactor":0.996257,"score":31.69,"se":2.88,"samplingSe":0.8695,"residualSe":2.7472,"coverageCount":8,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.692902,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.85,"inputPer1mUsd":0.3,"outputPer1mUsd":2.5,"contextWindow":1048576,"releaseDate":"2026-04-12","paretoProb":0,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:093b9df2-ebee-4309-943f-5a7e88360adf","observedHarness":"Direct API","rawScore":26.5,"zScore":-1.038914,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:093b9df2-ebee-4309-943f-5a7e88360adf","observedHarness":"Stirrup","rawScore":3.5,"zScore":-1.782499,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recWGYASTIVksDKsa","observedHarness":"APEX-Agents Harness","rawScore":29.3,"zScore":-1.319173,"seModel":"upstream_se","seRaw":4.9,"ciHalfWidth":9.604,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:093b9df2-ebee-4309-943f-5a7e88360adf","observedHarness":"Direct API","rawScore":13.6,"zScore":-0.647846,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:093b9df2-ebee-4309-943f-5a7e88360adf","observedHarness":"Direct API","rawScore":9.9,"zScore":-1.016478,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Gemini 3.5 Flash-Lite","observedHarness":"Direct API","rawScore":1446.62,"zScore":-0.72028,"seModel":"upstream_ci","ciHalfWidth":3.733,"ciLower":1442.8867,"ciUpper":1450.3527,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:093b9df2-ebee-4309-943f-5a7e88360adf","observedHarness":"Direct API","rawScore":10.56,"zScore":-1.167559,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:093b9df2-ebee-4309-943f-5a7e88360adf","observedHarness":"mini-SWE-agent","rawScore":0,"zScore":-0.984647,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"moonshot:kimi-k2.6:thinking:direct:native","provider":"moonshot","modelSlug":"kimi-k2.6","displayName":"Kimi K2.6","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.231591,"shrinkageFactor":0.991833,"score":31.53,"se":3.86,"samplingSe":1.2844,"residualSe":3.645,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.181727,"costLogSd":1.032,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.7125,"inputPer1mUsd":0.95,"outputPer1mUsd":4,"contextWindow":262144,"releaseDate":"2026-04-10","paretoProb":0.6915,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:hHnQVatVTbQjM4sfrjw7GH","observedHarness":"Direct API","rawScore":34.9,"zScore":-1.210055,"seModel":"upstream_se","seRaw":1.508066,"ciHalfWidth":2.9558,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-10","publishedDate":"2026-08-10","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Kimi K2.6","observedHarness":"Direct API","rawScore":1456.4,"zScore":-0.260326,"seModel":"upstream_ci","ciHalfWidth":3.6859,"ciLower":1452.7151,"ciUpper":1460.087,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:kimi/kimi-k2.6","observedHarness":"Direct API","rawScore":15.87,"zScore":-2.03307,"seModel":"upstream_se","seRaw":2.539,"ciHalfWidth":4.9764,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.520305,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:kimi/kimi-k2.6","observedHarness":"Direct API","rawScore":40.14,"zScore":-0.820282,"seModel":"upstream_se","seRaw":2.041,"ciHalfWidth":4.0004,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.041295,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:kimi/kimi-k2.6","observedHarness":"Vals Tax Agent","rawScore":12.4,"zScore":-1.559069,"seModel":"upstream_se","seRaw":2.376,"ciHalfWidth":4.657,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.565511,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"deepseek:deepseek-v4-pro-0813:none:direct:native","provider":"deepseek","modelSlug":"deepseek-v4-pro-0813","displayName":"DeepSeek V4 Pro (0813)","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.243517,"shrinkageFactor":0.997164,"score":31.35,"se":3.82,"samplingSe":0.7568,"residualSe":3.746,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.976988,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.98,"inputPer1mUsd":1.32,"outputPer1mUsd":3.96,"contextWindow":1048576,"releaseDate":"2026-08-13","paretoProb":0,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:d4f5b250-be37-4801-85e6-32fda4179337","observedHarness":"Direct API","rawScore":38.6,"zScore":-0.175199,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:d4f5b250-be37-4801-85e6-32fda4179337","observedHarness":"Stirrup","rawScore":8.5,"zScore":-1.52462,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:d4f5b250-be37-4801-85e6-32fda4179337","observedHarness":"Direct API","rawScore":3.6,"zScore":-1.828065,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:d4f5b250-be37-4801-85e6-32fda4179337","observedHarness":"Direct API","rawScore":7.2,"zScore":-1.163668,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:d4f5b250-be37-4801-85e6-32fda4179337","observedHarness":"mini-SWE-agent","rawScore":0,"zScore":-0.984647,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"openai:gpt-5.2:high:direct:native","provider":"openai","modelSlug":"gpt-5.2","displayName":"GPT-5.2","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.243498,"shrinkageFactor":0.995192,"score":31.35,"se":6.35,"samplingSe":0.9855,"residualSe":6.2716,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.898372,"costLogSd":1.0452,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4.8125,"inputPer1mUsd":1.75,"outputPer1mUsd":14,"contextWindow":400000,"releaseDate":"2025-12-11","paretoProb":0.0002,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:Xp4qkZ32zjDeSYii2eQpnm","observedHarness":"Direct API","rawScore":34.3,"zScore":-1.25274,"seModel":"upstream_se","seRaw":1.501921,"ciHalfWidth":2.9438,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-5.2 (High)","observedHarness":"Direct API","rawScore":1440.92,"zScore":-0.988352,"seModel":"upstream_ci","ciHalfWidth":3.1089,"ciLower":1437.8146,"ciUpper":1444.0323,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:gpt-6-luna:none:direct:native","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.279634,"shrinkageFactor":0.994326,"score":30.81,"se":3.53,"samplingSe":1.0706,"residualSe":3.3612,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.071417,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0001,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:63cb48ea-0b0c-4e07-8ded-6d28dec4fa28","observedHarness":"Direct API","rawScore":30.1,"zScore":-0.781941,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:63cb48ea-0b0c-4e07-8ded-6d28dec4fa28","observedHarness":"Stirrup","rawScore":5,"zScore":-1.705135,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:63cb48ea-0b0c-4e07-8ded-6d28dec4fa28","observedHarness":"Direct API","rawScore":5.8,"zScore":-1.568417,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:63cb48ea-0b0c-4e07-8ded-6d28dec4fa28","observedHarness":"Direct API","rawScore":6.3,"zScore":-1.212731,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:63cb48ea-0b0c-4e07-8ded-6d28dec4fa28","observedHarness":"Direct API","rawScore":15,"zScore":-0.671126,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:63cb48ea-0b0c-4e07-8ded-6d28dec4fa28","observedHarness":"mini-SWE-agent","rawScore":0.95,"zScore":-0.941061,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"openai:gpt-5.4-mini:high:direct:native","provider":"openai","modelSlug":"gpt-5.4-mini","displayName":"GPT-5.4 Mini","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.298202,"shrinkageFactor":0.99555,"score":30.53,"se":6.35,"samplingSe":0.948,"residualSe":6.2738,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.796841,"costLogSd":1.0444,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.6875,"inputPer1mUsd":0.75,"outputPer1mUsd":4.5,"contextWindow":400000,"releaseDate":"2026-03-17","paretoProb":0.0015,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:Z3pjKgAULY9j9wmzQpdqZG","observedHarness":"Direct API","rawScore":29.4,"zScore":-1.601328,"seModel":"upstream_se","seRaw":1.441429,"ciHalfWidth":2.8252,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-5.4 mini (High)","observedHarness":"Direct API","rawScore":1446.4,"zScore":-0.730627,"seModel":"upstream_ci","ciHalfWidth":3.0052,"ciLower":1443.3994,"ciUpper":1449.4098,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"anthropic:claude-sonnet-4.5:none:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-4.5","displayName":"Claude Sonnet 4.5","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.38658,"shrinkageFactor":0.990462,"score":29.2,"se":5.18,"samplingSe":1.388,"residualSe":4.9918,"coverageCount":3,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.87726,"costLogSd":1.0443,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1000000,"releaseDate":"2025-09-29","paretoProb":0.0042,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:MhqeuRVx4JcRfxPQDYPq7Z","observedHarness":"Direct API","rawScore":23.7,"zScore":-2.006829,"seModel":"upstream_se","seRaw":1.345407,"ciHalfWidth":2.637,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-10","publishedDate":"2026-08-10","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Claude Sonnet 4.5 (20250929)","observedHarness":"Direct API","rawScore":1448.73,"zScore":-0.621047,"seModel":"upstream_ci","ciHalfWidth":2.5569,"ciLower":1446.1718,"ciUpper":1451.2857,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:anthropic/claude-sonnet-4-5-20250929","observedHarness":"Direct API","rawScore":40.57,"zScore":-0.757478,"seModel":"upstream_se","seRaw":1.995,"ciHalfWidth":3.9102,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.042403,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"}]},{"id":"anthropic:claude-sonnet-5:xhigh:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-5","displayName":"Claude Sonnet 5","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.425897,"shrinkageFactor":0.987696,"score":28.61,"se":9.23,"samplingSe":1.5765,"residualSe":9.0973,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.535732,"costLogSd":1.0518,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-07-15","paretoProb":0.0032,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:Qw4tGw38pk4sdsEehhrLSR","observedHarness":"Direct API","rawScore":32.9,"zScore":-1.352336,"seModel":"upstream_se","seRaw":1.48654,"ciHalfWidth":2.9136,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"}]},{"id":"openai:gpt-5.2:low:direct:native","provider":"openai","modelSlug":"gpt-5.2","displayName":"GPT-5.2","effort":"low","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.432947,"shrinkageFactor":0.987714,"score":28.51,"se":9.23,"samplingSe":1.5753,"residualSe":9.0975,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.72689,"costLogSd":1.0437,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4.8125,"inputPer1mUsd":1.75,"outputPer1mUsd":14,"contextWindow":400000,"releaseDate":"2025-12-11","paretoProb":0.001,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:Cyw3GA9a64AmT4pqLAhtTF","observedHarness":"Direct API","rawScore":32.8,"zScore":-1.35945,"seModel":"upstream_se","seRaw":1.485384,"ciHalfWidth":2.9114,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"}]},{"id":"openai:gpt-5.2:medium:direct:native","provider":"openai","modelSlug":"gpt-5.2","displayName":"GPT-5.2","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.439997,"shrinkageFactor":0.987733,"score":28.4,"se":9.23,"samplingSe":1.5741,"residualSe":9.0977,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.605187,"costLogSd":1.055,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4.8125,"inputPer1mUsd":1.75,"outputPer1mUsd":14,"contextWindow":400000,"releaseDate":"2025-12-11","paretoProb":0.0007,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:esr5dcSuHiVoteo6vZfHjP","observedHarness":"Direct API","rawScore":32.7,"zScore":-1.366565,"seModel":"upstream_se","seRaw":1.484221,"ciHalfWidth":2.9091,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"}]},{"id":"deepseek:deepseek-v4.1-flash-1017:none:direct:native","provider":"deepseek","modelSlug":"deepseek-v4.1-flash-1017","displayName":"DeepSeek V4.1 Flash (1017)","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.487696,"shrinkageFactor":0.998055,"score":27.68,"se":3.8,"samplingSe":0.6268,"residualSe":3.7494,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.42542,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.525,"inputPer1mUsd":0.3,"outputPer1mUsd":1.2,"contextWindow":1048576,"releaseDate":"2026-10-17","paretoProb":1,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:5720fd2f-873a-4765-8640-a153b38bd64d","observedHarness":"Direct API","rawScore":21.87,"zScore":-1.36941,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:5720fd2f-873a-4765-8640-a153b38bd64d","observedHarness":"Stirrup","rawScore":1,"zScore":-1.911438,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:5720fd2f-873a-4765-8640-a153b38bd64d","observedHarness":"Direct API","rawScore":4.4,"zScore":-1.733648,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:5720fd2f-873a-4765-8640-a153b38bd64d","observedHarness":"Direct API","rawScore":7.1,"zScore":-1.169119,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:5720fd2f-873a-4765-8640-a153b38bd64d","observedHarness":"mini-SWE-agent","rawScore":2.38,"zScore":-0.875452,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"google:gemini-3.5-flash-lite:high:direct:native","provider":"google","modelSlug":"gemini-3.5-flash-lite","displayName":"Gemini 3.5 Flash-Lite","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.590195,"shrinkageFactor":0.982659,"score":26.15,"se":5.45,"samplingSe":1.8716,"residualSe":5.1153,"coverageCount":3,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.529863,"costLogSd":0.6488,"costMinUsd":0.692902,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.85,"inputPer1mUsd":0.3,"outputPer1mUsd":2.5,"contextWindow":1048576,"releaseDate":"2026-04-12","paretoProb":0,"benchmarks":[{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:google/gemini-3.5-flash-lite","observedHarness":"Direct API","rawScore":13.94,"zScore":-2.204271,"seModel":"upstream_se","seRaw":2.408,"ciHalfWidth":4.7197,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.124723,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:google/gemini-3.5-flash-lite","observedHarness":"Direct API","rawScore":43.49,"zScore":-0.330995,"seModel":"upstream_se","seRaw":1.951,"ciHalfWidth":3.824,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.008093,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:google/gemini-3.5-flash-lite","observedHarness":"Vals Tax Agent","rawScore":7.25,"zScore":-2.013016,"seModel":"upstream_se","seRaw":1.872,"ciHalfWidth":3.6691,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.081536,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"spacexai:grok-4.20:thinking:direct:native","provider":"spacexai","modelSlug":"grok-4.20","displayName":"Grok 4.20","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.656432,"shrinkageFactor":0.993261,"score":25.15,"se":3.83,"samplingSe":1.1667,"residualSe":3.6502,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.834409,"costLogSd":1.0341,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.5625,"inputPer1mUsd":1.25,"outputPer1mUsd":2.5,"contextWindow":2000000,"releaseDate":"2026-03-09","paretoProb":0.2044,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:GVm7khGvtdnQKGfkgAHD8q","observedHarness":"Direct API","rawScore":30.2,"zScore":-1.544416,"seModel":"upstream_se","seRaw":1.452608,"ciHalfWidth":2.8471,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Grok 4.20 Beta (Reasoning, 0309)","observedHarness":"Direct API","rawScore":1456,"zScore":-0.279138,"seModel":"upstream_ci","ciHalfWidth":3.0643,"ciLower":1452.9352,"ciUpper":1459.0637,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:grok/grok-4.20-0309-reasoning","observedHarness":"Direct API","rawScore":13.94,"zScore":-2.204271,"seModel":"upstream_se","seRaw":2.408,"ciHalfWidth":4.7197,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.261499,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:grok/grok-4.20-0309-reasoning","observedHarness":"Direct API","rawScore":32.16,"zScore":-1.985807,"seModel":"upstream_se","seRaw":2.124,"ciHalfWidth":4.163,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.036189,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:grok/grok-4.20-0309-reasoning","observedHarness":"Vals Tax Agent","rawScore":6.95,"zScore":-2.03946,"seModel":"upstream_se","seRaw":1.75,"ciHalfWidth":3.43,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.119262,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-6-sol:xhigh:direct:native","provider":"openai","modelSlug":"gpt-6-sol","displayName":"GPT-6 Sol","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.666127,"shrinkageFactor":0.97323,"score":25.01,"se":9.25,"samplingSe":2.3253,"residualSe":8.9519,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.535732,"costLogSd":1.0518,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0,"benchmarks":[{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:GPT 6 Sol (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":38.6,"zScore":-2.150595,"seModel":"upstream_ci","ciHalfWidth":1.58,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-09-25","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"}]},{"id":"spacexai:grok-4.3:low:direct:native","provider":"spacexai","modelSlug":"grok-4.3","displayName":"Grok 4.3","effort":"low","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.700359,"shrinkageFactor":0.977809,"score":24.49,"se":6.62,"samplingSe":2.1172,"residualSe":6.2703,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.409226,"costLogSd":1.0415,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.5625,"inputPer1mUsd":1.25,"outputPer1mUsd":2.5,"contextWindow":1000000,"releaseDate":"2026-05-01","paretoProb":0.4219,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Grok 4.3","observedHarness":"Direct API","rawScore":1426.33,"zScore":-1.674522,"seModel":"upstream_ci","ciHalfWidth":2.8953,"ciLower":1423.4345,"ciUpper":1429.2251,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:grok/grok-4.3","observedHarness":"Direct API","rawScore":38.07,"zScore":-1.122617,"seModel":"upstream_se","seRaw":2.081,"ciHalfWidth":4.0788,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.022202,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"}]},{"id":"spacexai:grok-4.3:high:direct:native","provider":"spacexai","modelSlug":"grok-4.3","displayName":"Grok 4.3","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.763742,"shrinkageFactor":0.991208,"score":23.54,"se":5.24,"samplingSe":1.3326,"residualSe":5.0679,"coverageCount":3,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.16504,"costLogSd":1.0466,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.5625,"inputPer1mUsd":1.25,"outputPer1mUsd":2.5,"contextWindow":1000000,"releaseDate":"2026-05-01","paretoProb":0.1301,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:Y39ZRp553ULpUDBV2XrBbc","observedHarness":"Direct API","rawScore":33.2,"zScore":-1.330994,"seModel":"upstream_se","seRaw":1.48996,"ciHalfWidth":2.9203,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:grok/grok-4.3","observedHarness":"Direct API","rawScore":15.38,"zScore":-2.076536,"seModel":"upstream_se","seRaw":2.508,"ciHalfWidth":4.9157,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.203143,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:grok/grok-4.3","observedHarness":"Vals Tax Agent","rawScore":6.18,"zScore":-2.107332,"seModel":"upstream_se","seRaw":1.732,"ciHalfWidth":3.3947,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.116616,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-5.4-mini:xhigh:direct:native","provider":"openai","modelSlug":"gpt-5.4-mini","displayName":"GPT-5.4 Mini","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.889096,"shrinkageFactor":0.979592,"score":21.66,"se":6.77,"samplingSe":2.0303,"residualSe":6.4607,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.200096,"costLogSd":1.051,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":1.6875,"inputPer1mUsd":0.75,"outputPer1mUsd":4.5,"contextWindow":400000,"releaseDate":"2026-03-17","paretoProb":0.0001,"benchmarks":[{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:openai/gpt-5.4-mini-2026-03-17","observedHarness":"Direct API","rawScore":12.5,"zScore":-2.332007,"seModel":"upstream_se","seRaw":2.299,"ciHalfWidth":4.506,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.864673,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:openai/gpt-5.4-mini-2026-03-17","observedHarness":"Vals Tax Agent","rawScore":8.81,"zScore":-1.87551,"seModel":"upstream_se","seRaw":2.045,"ciHalfWidth":4.0082,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.806171,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"anthropic:claude-haiku-4.5:thinking:direct:native","provider":"anthropic","modelSlug":"claude-haiku-4.5","displayName":"Claude Haiku 4.5","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"research","theta":-2.208727,"shrinkageFactor":0.913374,"score":16.87,"se":9.49,"samplingSe":4.183,"residualSe":8.5221,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.426413,"costLogSd":1.0313,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":200000,"releaseDate":"2025-10-15","paretoProb":0.0041,"benchmarks":[{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:anthropic/claude-haiku-4-5-20251001-thinking","observedHarness":"Direct API","rawScore":32.68,"zScore":-1.909858,"seModel":"upstream_se","seRaw":1.998,"ciHalfWidth":3.9161,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.020099,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"}]},{"id":"openai:gpt-5.4-nano:high:direct:native","provider":"openai","modelSlug":"gpt-5.4-nano","displayName":"GPT-5.4 Nano","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-2.258032,"shrinkageFactor":0.994413,"score":16.13,"se":3.81,"samplingSe":1.0623,"residualSe":3.6545,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.576808,"costLogSd":1.0536,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":0.4625,"inputPer1mUsd":0.2,"outputPer1mUsd":1.25,"contextWindow":400000,"releaseDate":"2026-03-17","paretoProb":0.0001,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:3QBsxKkbF2R2LEXTd6HDrJ","observedHarness":"Direct API","rawScore":11.7,"zScore":-2.860516,"seModel":"upstream_se","seRaw":1.016929,"ciHalfWidth":1.9932,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-5.4 nano (High)","observedHarness":"Direct API","rawScore":1416.09,"zScore":-2.15611,"seModel":"upstream_ci","ciHalfWidth":3.0301,"ciLower":1413.0619,"ciUpper":1419.1221,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:openai/gpt-5.4-nano-2026-03-17","observedHarness":"Direct API","rawScore":6.25,"zScore":-2.886414,"seModel":"upstream_se","seRaw":1.682,"ciHalfWidth":3.2967,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.141253,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:openai/gpt-5.4-nano-2026-03-17","observedHarness":"Direct API","rawScore":41.03,"zScore":-0.690292,"seModel":"upstream_se","seRaw":2.256,"ciHalfWidth":4.4218,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.000844,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:openai/gpt-5.4-nano-2026-03-17","observedHarness":"Vals Tax Agent","rawScore":4.97,"zScore":-2.213987,"seModel":"upstream_se","seRaw":1.417,"ciHalfWidth":2.7773,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.069662,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"anthropic:claude-haiku-4.5:none:direct:native","provider":"anthropic","modelSlug":"claude-haiku-4.5","displayName":"Claude Haiku 4.5","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-2.534389,"shrinkageFactor":0.997671,"score":11.98,"se":6.32,"samplingSe":0.6858,"residualSe":6.2872,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.70069,"costLogSd":1.0403,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":200000,"releaseDate":"2025-10-15","paretoProb":0.0174,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:J2kUEjR9r6Spd594qDNTTf","observedHarness":"Direct API","rawScore":13.2,"zScore":-2.753805,"seModel":"upstream_se","seRaw":1.070937,"ciHalfWidth":2.099,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-10","publishedDate":"2026-08-10","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Claude Haiku 4.5 (20251001)","observedHarness":"Direct API","rawScore":1418.2,"zScore":-2.056877,"seModel":"upstream_ci","ciHalfWidth":2.0286,"ciLower":1416.1685,"ciUpper":1420.2257,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"openai:o3:medium:direct:native","provider":"openai","modelSlug":"o3","displayName":"o3","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"research","theta":-2.792037,"shrinkageFactor":0.973948,"score":8.12,"se":9.29,"samplingSe":2.294,"residualSe":9.0017,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.634849,"costLogSd":1.0539,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":3.5,"inputPer1mUsd":2,"outputPer1mUsd":8,"contextWindow":200000,"releaseDate":"2025-04-16","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:o3 (2025-04-16)","observedHarness":"Direct API","rawScore":1404.53,"zScore":-2.699778,"seModel":"upstream_ci","ciHalfWidth":6.6304,"ciLower":1397.9017,"ciUpper":1411.1626,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]}],"provisionalConfigs":[{"id":"meta:muse-wonder-full:max:direct:native","provider":"meta","modelSlug":"muse-wonder-full","displayName":"Muse Wonder Full","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.811106,"shrinkageFactor":0.991106,"score":62.17,"se":2.11,"samplingSe":1.3404,"residualSe":1.6323,"coverageCount":15,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_briefcase:e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2","observedHarness":"Stirrup","rawScore":1729.32,"zScore":1.201925,"seModel":"upstream_ci","ciHalfWidth":10.34,"ciLower":1718.98,"ciUpper":1739.66,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2","observedHarness":"Direct API","rawScore":40.97,"zScore":-0.006024,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2","observedHarness":"Stirrup","rawScore":56.5,"zScore":0.951017,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2","observedHarness":"Direct API","rawScore":32.6,"zScore":1.594569,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdpval_aa","benchmarkVersion":"2.1","sourceId":"artificial-analysis","sourceRowId":"aa:gdpval_aa:e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2","observedHarness":"Stirrup","rawScore":1857.09,"zScore":1.472848,"seModel":"upstream_ci","ciHalfWidth":24.88,"ciLower":1832.21,"ciUpper":1881.97,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2","observedHarness":"Direct API","rawScore":48.5,"zScore":1.087791,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2","observedHarness":"Direct API","rawScore":22.22,"zScore":0.136136,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:Muse Wonder Full (Max reasoning)","observedHarness":"Direct API","rawScore":63.2,"zScore":0.382343,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:e994a6e0-d36b-4dab-8ae1-c1b469e4d9e2","observedHarness":"mini-SWE-agent","rawScore":37.14,"zScore":0.719356,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:meta/muse-wonder-full","observedHarness":"Direct API","rawScore":75,"zScore":1.087542,"seModel":"upstream_se","seRaw":2.141,"ciHalfWidth":4.1964,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.58223,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:meta/muse-wonder-full","observedHarness":"Vals Finance Agent","rawScore":67.58,"zScore":1.34114,"seModel":"upstream_se","seRaw":1.98,"ciHalfWidth":3.8808,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.680843,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:meta/muse-wonder-full","observedHarness":"Direct API","rawScore":53.37,"zScore":1.293374,"seModel":"upstream_se","seRaw":3.467,"ciHalfWidth":6.7953,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.595401,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:meta/muse-wonder-full","observedHarness":"Direct API","rawScore":49.93,"zScore":0.609605,"seModel":"upstream_se","seRaw":2.177,"ciHalfWidth":4.2669,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.247179,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:meta/muse-wonder-full","observedHarness":"Direct API","rawScore":35.13,"zScore":0.434459,"seModel":"upstream_se","seRaw":3.211,"ciHalfWidth":6.2936,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.163398,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:meta/muse-wonder-full","observedHarness":"Vals Tax Agent","rawScore":43,"zScore":1.138174,"seModel":"upstream_se","seRaw":3.572,"ciHalfWidth":7.0011,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.147235,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"meta:muse-spark:high:direct:native","provider":"meta","modelSlug":"muse-spark","displayName":"Muse Spark","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":0.518845,"shrinkageFactor":0.989805,"score":57.78,"se":4.45,"samplingSe":1.435,"residualSe":4.214,"coverageCount":4,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Muse Spark","observedHarness":"Direct API","rawScore":1468.09,"zScore":0.289457,"seModel":"upstream_ci","ciHalfWidth":5.4674,"ciLower":1462.6263,"ciUpper":1473.5611,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:Muse Spark (one-pass-agent)","observedHarness":"Scale one-pass-agent","rawScore":52.44,"zScore":0.739554,"seModel":"upstream_ci","ciHalfWidth":0.06,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-04-08","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Muse Spark (one-pass-agent)","observedHarness":"Scale one-pass-agent","rawScore":52.29,"zScore":0.518046,"seModel":"upstream_ci","ciHalfWidth":0.06,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-04-08","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:meta/muse_spark","observedHarness":"Direct API","rawScore":51.31,"zScore":0.811162,"seModel":"upstream_se","seRaw":2.244,"ciHalfWidth":4.3982,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.005341,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"}]},{"id":"deepseek:deepseek-v4.1-flash:xhigh:direct:native","provider":"deepseek","modelSlug":"deepseek-v4.1-flash","displayName":"DeepSeek V4.1 Flash","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.039552,"shrinkageFactor":0.981991,"score":49.41,"se":6.59,"samplingSe":1.9073,"residualSe":6.3071,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-10","paretoProb":0,"benchmarks":[{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:DeepSeek V4.1 Flash (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":47.15,"zScore":0.037171,"seModel":"upstream_ci","ciHalfWidth":1.61,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-27","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:DeepSeek V4.1 Flash (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":47.26,"zScore":-0.46247,"seModel":"upstream_ci","ciHalfWidth":0.97,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-27","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"}]},{"id":"meta:muse-spark-open:xhigh:direct:native","provider":"meta","modelSlug":"muse-spark-open","displayName":"Muse Spark Open","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.067792,"shrinkageFactor":0.995159,"score":48.98,"se":1.89,"samplingSe":0.9889,"residualSe":1.6114,"coverageCount":13,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":2.017668,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:3661c8fb-caf7-4118-aa18-65b84fea8e44","observedHarness":"Direct API","rawScore":42.03,"zScore":0.06964,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:3661c8fb-caf7-4118-aa18-65b84fea8e44","observedHarness":"Stirrup","rawScore":29.5,"zScore":-0.441529,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:dNuxLdV4DWqfSvtTq6NnK8","observedHarness":"Direct API","rawScore":56.6,"zScore":0.333694,"seModel":"upstream_se","seRaw":1.568088,"ciHalfWidth":3.0735,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-15","publishedDate":"2026-10-15","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:3661c8fb-caf7-4118-aa18-65b84fea8e44","observedHarness":"Direct API","rawScore":18.6,"zScore":-0.057737,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:3661c8fb-caf7-4118-aa18-65b84fea8e44","observedHarness":"Direct API","rawScore":26.3,"zScore":-0.122436,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:3661c8fb-caf7-4118-aa18-65b84fea8e44","observedHarness":"Direct API","rawScore":7.78,"zScore":-1.478389,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:Muse Spark Open (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":57.11,"zScore":1.359617,"seModel":"upstream_ci","ciHalfWidth":0.52,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-19","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Muse Spark Open (one-pass-agent) xhigh","observedHarness":"Scale one-pass-agent","rawScore":59.76,"zScore":1.974201,"seModel":"upstream_ci","ciHalfWidth":0.88,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-20","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:3661c8fb-caf7-4118-aa18-65b84fea8e44","observedHarness":"mini-SWE-agent","rawScore":1.9,"zScore":-0.897474,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:meta/muse-spark-open","observedHarness":"Direct API","rawScore":61.98,"zScore":-2.485091,"seModel":"upstream_se","seRaw":2.704,"ciHalfWidth":5.2998,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.348821,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:meta/muse-spark-open","observedHarness":"Vals Finance Agent","rawScore":60.66,"zScore":-0.641207,"seModel":"upstream_se","seRaw":2.056,"ciHalfWidth":4.0298,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.758047,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:meta/muse-spark-open","observedHarness":"Direct API","rawScore":38.46,"zScore":-0.02922,"seModel":"upstream_se","seRaw":3.381,"ciHalfWidth":6.6268,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.559385,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:meta/muse-spark-open","observedHarness":"Vals Tax Agent","rawScore":26.32,"zScore":-0.332088,"seModel":"upstream_se","seRaw":2.653,"ciHalfWidth":5.1999,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.883559,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"deepseek:deepseek-v4.1-flash:max:direct:native","provider":"deepseek","modelSlug":"deepseek-v4.1-flash","displayName":"DeepSeek V4.1 Flash","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.093692,"shrinkageFactor":0.984162,"score":48.59,"se":3.94,"samplingSe":1.7886,"residualSe":3.5125,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-10","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:3JJ4ChNxMo8CGAh3HrKNrw","observedHarness":"Direct API","rawScore":42.9,"zScore":-0.640931,"seModel":"upstream_se","seRaw":1.5659,"ciHalfWidth":3.0692,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-10-13","publishedDate":"2026-10-13","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:DeepSeek-V4.1-Flash-20260910 (Max)","observedHarness":"Direct API","rawScore":1467.6,"zScore":0.266412,"seModel":"upstream_ci","ciHalfWidth":5.2116,"ciLower":1462.3856,"ciUpper":1472.8088,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:deepseek/deepseek-v4.1-flash","observedHarness":"Direct API","rawScore":67.76,"zScore":-0.899083,"seModel":"upstream_se","seRaw":2.565,"ciHalfWidth":5.0274,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.234152,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","sourceId":"vals","sourceRowId":"vals:fabv2-1:deepseek/deepseek-v4.1-flash","observedHarness":"Vals Finance Agent","rawScore":65.43,"zScore":0.725237,"seModel":"upstream_se","seRaw":2.024,"ciHalfWidth":3.967,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.366574,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/fabv2-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:deepseek/deepseek-v4.1-flash","observedHarness":"Direct API","rawScore":41.35,"zScore":0.227138,"seModel":"upstream_se","seRaw":3.423,"ciHalfWidth":6.7091,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.24793,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:deepseek/deepseek-v4.1-flash","observedHarness":"Vals Tax Agent","rawScore":26.63,"zScore":-0.304763,"seModel":"upstream_se","seRaw":3.141,"ciHalfWidth":6.1564,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.134662,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"alibaba:qwen3.8-max:xhigh:direct:native","provider":"alibaba","modelSlug":"qwen3.8-max","displayName":"Qwen3.8 Max","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.20929,"shrinkageFactor":0.990939,"score":46.86,"se":2.57,"samplingSe":1.3529,"residualSe":2.1861,"coverageCount":12,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-08-03","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:bfftd5EzUK9yAekafEofvK","observedHarness":"Direct API","rawScore":45.8,"zScore":-0.434624,"seModel":"upstream_se","seRaw":1.576339,"ciHalfWidth":3.0896,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Qwen3.8-Max","observedHarness":"Direct API","rawScore":1473.3,"zScore":0.534484,"seModel":"upstream_ci","ciHalfWidth":3.6576,"ciLower":1469.638,"ciUpper":1476.9532,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Qwen 3.8 Max (xHigh reasoning)","observedHarness":"Direct API","rawScore":8.8,"zScore":-0.712028,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Qwen 3.8 Max (xHigh reasoning)","observedHarness":"Direct API","rawScore":4.8,"zScore":-0.713349,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-law:Qwen 3.8 Max (xHigh reasoning)","observedHarness":"Direct API","rawScore":5.6,"zScore":-0.018987,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-law"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:Qwen 3.8 Max (xHigh reasoning)","observedHarness":"Direct API","rawScore":20.3,"zScore":-0.128398,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:riemann-bench:Qwen 3.8 Max (xHigh reasoning)","observedHarness":"Direct API","rawScore":48,"zScore":-0.407755,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/riemann-bench"},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","sourceId":"vals","sourceRowId":"vals:emb-v1-1:alibaba/qwen3.8-max","observedHarness":"Direct API","rawScore":65.99,"zScore":-1.384764,"seModel":"upstream_se","seRaw":2.694,"ciHalfWidth":5.2802,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.655924,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/emb-v1-1"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:alibaba/qwen3.8-max","observedHarness":"Direct API","rawScore":47.6,"zScore":0.781545,"seModel":"upstream_se","seRaw":3.471,"ciHalfWidth":6.8032,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.486351,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:alibaba/qwen3.8-max","observedHarness":"Direct API","rawScore":40.67,"zScore":-0.742872,"seModel":"upstream_se","seRaw":2.029,"ciHalfWidth":3.9768,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.120827,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:alibaba/qwen3.8-max","observedHarness":"Direct API","rawScore":23.87,"zScore":-0.754254,"seModel":"upstream_se","seRaw":2.868,"ciHalfWidth":5.6213,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.309463,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:alibaba/qwen3.8-max","observedHarness":"Vals Tax Agent","rawScore":32.09,"zScore":0.17651,"seModel":"upstream_se","seRaw":3.367,"ciHalfWidth":6.5993,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.552558,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"google:gemini-3-pro:high:direct:native","provider":"google","modelSlug":"gemini-3-pro","displayName":"Gemini 3 Pro","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.32117,"shrinkageFactor":0.987183,"score":45.18,"se":4.5,"samplingSe":1.609,"residualSe":4.2028,"coverageCount":4,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-11-18","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Gemini 3 Pro","observedHarness":"Direct API","rawScore":1481.54,"zScore":0.922012,"seModel":"upstream_ci","ciHalfWidth":3.4196,"ciLower":1478.1175,"ciUpper":1484.9567,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:Gemini 3 Pro (one-pass-agent)","observedHarness":"Scale one-pass-agent","rawScore":39.18,"zScore":-1.021051,"seModel":"upstream_ci","ciHalfWidth":0.51,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2025-11-20","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Gemini 3 Pro (one-pass-agent)","observedHarness":"Scale one-pass-agent","rawScore":40.6,"zScore":-1.760728,"seModel":"upstream_ci","ciHalfWidth":0.28,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2025-11-20","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:google/gemini-3-pro-preview","observedHarness":"Direct API","rawScore":52.2,"zScore":0.941151,"seModel":"upstream_se","seRaw":2.073,"ciHalfWidth":4.0631,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.028248,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"}]},{"id":"alibaba:qwen3.8-2.4t-a95b:xhigh:direct:native","provider":"alibaba","modelSlug":"qwen3.8-2.4t-a95b","displayName":"Qwen3.8 2.4T A95B","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.39387,"shrinkageFactor":0.980514,"score":44.09,"se":5.02,"samplingSe":1.9839,"residualSe":4.6075,"coverageCount":4,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-12","paretoProb":0,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:8df710d3-9dae-4498-9b4e-9818238e6f31","observedHarness":"Direct API","rawScore":31.4,"zScore":-0.689145,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:8df710d3-9dae-4498-9b4e-9818238e6f31","observedHarness":"Direct API","rawScore":15.6,"zScore":-0.411803,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:8df710d3-9dae-4498-9b4e-9818238e6f31","observedHarness":"Direct API","rawScore":20.1,"zScore":-0.460428,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:8df710d3-9dae-4498-9b4e-9818238e6f31","observedHarness":"Direct API","rawScore":30.56,"zScore":1.068625,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"openai:gpt-5.5-instant:none:direct:native","provider":"openai","modelSlug":"gpt-5.5-instant","displayName":"GPT-5.5 Instant","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.518213,"shrinkageFactor":0.980561,"score":42.23,"se":4.96,"samplingSe":1.9815,"residualSe":4.5515,"coverageCount":4,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-06-01","paretoProb":0,"benchmarks":[{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:c4169d75-9d24-4794-a9c3-69e1fe364de5","observedHarness":"Direct API","rawScore":20.2,"zScore":0.131098,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:c4169d75-9d24-4794-a9c3-69e1fe364de5","observedHarness":"Direct API","rawScore":14.6,"zScore":-0.760259,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:GPT-5.5 Instant","observedHarness":"Direct API","rawScore":1442.86,"zScore":-0.897114,"seModel":"upstream_ci","ciHalfWidth":4.1802,"ciLower":1438.682,"ciUpper":1447.0424,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:c4169d75-9d24-4794-a9c3-69e1fe364de5","observedHarness":"Direct API","rawScore":31.67,"zScore":1.192734,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"deepseek:deepseek-v4-flash:high:direct:native","provider":"deepseek","modelSlug":"deepseek-v4-flash","displayName":"DeepSeek V4 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.712639,"shrinkageFactor":0.982289,"score":39.31,"se":4.64,"samplingSe":1.8914,"residualSe":4.2362,"coverageCount":4,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-31","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:DeepSeek-V4-Flash","observedHarness":"Direct API","rawScore":1439.11,"zScore":-1.073477,"seModel":"upstream_ci","ciHalfWidth":3.3225,"ciLower":1435.7887,"ciUpper":1442.4337,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:deepseek/deepseek-v4-flash-0731","observedHarness":"Direct API","rawScore":30.29,"zScore":-0.753942,"seModel":"upstream_se","seRaw":3.194,"ciHalfWidth":6.2602,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.275172,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:deepseek/deepseek-v4-flash-0731","observedHarness":"Direct API","rawScore":41.41,"zScore":-0.634791,"seModel":"upstream_se","seRaw":2.15,"ciHalfWidth":4.214,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.019659,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:deepseek/deepseek-v4-flash-0731","observedHarness":"Vals Tax Agent","rawScore":28.07,"zScore":-0.177834,"seModel":"upstream_se","seRaw":3.158,"ciHalfWidth":6.1897,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.156388,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"deepseek:deepseek-v4-flash-vision:max:direct:native","provider":"deepseek","modelSlug":"deepseek-v4-flash-vision","displayName":"DeepSeek V4 Flash Vision","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.722875,"shrinkageFactor":0.988926,"score":39.16,"se":7.28,"samplingSe":1.4956,"residualSe":7.1223,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-10","paretoProb":0,"benchmarks":[{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:7707a46f-895f-4247-b7a8-ddab87c9a11d","observedHarness":"Direct API","rawScore":12,"zScore":-0.836681,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:7707a46f-895f-4247-b7a8-ddab87c9a11d","observedHarness":"Direct API","rawScore":21.1,"zScore":-0.405913,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"google:gemma-4-31b:none:direct:native","provider":"google","modelSlug":"gemma-4-31b","displayName":"Gemma 4 31B","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.750028,"shrinkageFactor":0.970621,"score":38.75,"se":9.3,"samplingSe":2.436,"residualSe":8.9709,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-01","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Gemma 4 31B","observedHarness":"Direct API","rawScore":1449.07,"zScore":-0.605056,"seModel":"upstream_ci","ciHalfWidth":6.8698,"ciLower":1442.2011,"ciUpper":1455.9408,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"deepseek:deepseek-v4-pro:max:direct:native","provider":"deepseek","modelSlug":"deepseek-v4-pro","displayName":"DeepSeek V4 Pro (0424)","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.820184,"shrinkageFactor":0.99036,"score":37.7,"se":4.87,"samplingSe":1.3954,"residualSe":4.6638,"coverageCount":4,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-04-24","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:Sq9xqkVTmFvELpQy9KH5TM","observedHarness":"Direct API","rawScore":46.99,"zScore":-0.349966,"seModel":"upstream_se","seRaw":1.580652,"ciHalfWidth":3.0981,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:DeepSeek V4 Pro (Max reasoning)","observedHarness":"Direct API","rawScore":3.5,"zScore":-1.116294,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:DeepSeek V4 Pro (Max reasoning)","observedHarness":"Direct API","rawScore":1.2,"zScore":-0.98916,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:gdp-xlsx:DeepSeek V4 Pro (Max reasoning)","observedHarness":"Direct API","rawScore":6.9,"zScore":-1.876112,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/gdp-xlsx"}]},{"id":"alibaba:qwen3.8-flash-next:xhigh:direct:native","provider":"alibaba","modelSlug":"qwen3.8-flash-next","displayName":"Qwen3.8 Flash Next","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.827034,"shrinkageFactor":0.989351,"score":37.59,"se":3.65,"samplingSe":1.4666,"residualSe":3.3444,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":0.569381,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-15","paretoProb":0,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:9ac3908c-9eb2-44db-8c01-3ec7da6fc2ae","observedHarness":"Direct API","rawScore":23.53,"zScore":-1.250917,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:9ac3908c-9eb2-44db-8c01-3ec7da6fc2ae","observedHarness":"Stirrup","rawScore":24.5,"zScore":-0.699407,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:9ac3908c-9eb2-44db-8c01-3ec7da6fc2ae","observedHarness":"Direct API","rawScore":15.6,"zScore":-0.411803,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:9ac3908c-9eb2-44db-8c01-3ec7da6fc2ae","observedHarness":"Direct API","rawScore":16.3,"zScore":-0.667584,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:9ac3908c-9eb2-44db-8c01-3ec7da6fc2ae","observedHarness":"Direct API","rawScore":20.56,"zScore":-0.049467,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:9ac3908c-9eb2-44db-8c01-3ec7da6fc2ae","observedHarness":"mini-SWE-agent","rawScore":7.14,"zScore":-0.65706,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"deepseek:deepseek-v4-pro:high:direct:native","provider":"deepseek","modelSlug":"deepseek-v4-pro","displayName":"DeepSeek V4 Pro (0424)","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.829585,"shrinkageFactor":0.993423,"score":37.56,"se":9.25,"samplingSe":1.1526,"residualSe":9.1817,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-04-24","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:DeepSeek-V4-Pro Preview (High)","observedHarness":"Direct API","rawScore":1447.64,"zScore":-0.67231,"seModel":"upstream_ci","ciHalfWidth":3.213,"ciLower":1444.4261,"ciUpper":1450.8522,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"deepseek:deepseek-v3.2-exp:thinking:direct:native","provider":"deepseek","modelSlug":"deepseek-v3.2-exp","displayName":"DeepSeek V3.2-Exp","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.83331,"shrinkageFactor":0.931516,"score":37.5,"se":9.38,"samplingSe":3.7193,"residualSe":8.6095,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-09-29","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:DeepSeek-V3.2-Exp (Thinking)","observedHarness":"Direct API","rawScore":1446.67,"zScore":-0.717929,"seModel":"upstream_ci","ciHalfWidth":10.7066,"ciLower":1435.9592,"ciUpper":1457.3725,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemma-4-26b-a4b:none:direct:native","provider":"google","modelSlug":"gemma-4-26b-a4b","displayName":"Gemma 4 26B A4B","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-0.957569,"shrinkageFactor":0.970415,"score":35.64,"se":9.3,"samplingSe":2.4446,"residualSe":8.969,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-01","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Gemma 4 26B A4B","observedHarness":"Direct API","rawScore":1444.52,"zScore":-0.819044,"seModel":"upstream_ci","ciHalfWidth":6.8947,"ciLower":1437.6285,"ciUpper":1451.4179,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"alibaba:qwen3.8-27b:xhigh:direct:native","provider":"alibaba","modelSlug":"qwen3.8-27b","displayName":"Qwen3.8 27B","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.032014,"shrinkageFactor":0.995231,"score":34.52,"se":1.99,"samplingSe":0.9815,"residualSe":1.7289,"coverageCount":13,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":3.529549,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-05","paretoProb":0,"benchmarks":[{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_briefcase:b01dee41-c62b-48ed-8d16-984adc405e5c","observedHarness":"Stirrup","rawScore":1391.36,"zScore":-1.03722,"seModel":"upstream_ci","ciHalfWidth":7.19,"ciLower":1384.36,"ciUpper":1398.74,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-briefcase"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:b01dee41-c62b-48ed-8d16-984adc405e5c","observedHarness":"Direct API","rawScore":17.3,"zScore":-1.695624,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:b01dee41-c62b-48ed-8d16-984adc405e5c","observedHarness":"Stirrup","rawScore":4,"zScore":-1.756711,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recfGdLMX8buaQjoL","observedHarness":"APEX-Agents Harness","rawScore":47.5,"zScore":-0.224827,"seModel":"upstream_se","seRaw":4.9,"ciHalfWidth":9.604,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:b01dee41-c62b-48ed-8d16-984adc405e5c","observedHarness":"Direct API","rawScore":16.4,"zScore":-0.317385,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdpval_aa","benchmarkVersion":"2.1","sourceId":"artificial-analysis","sourceRowId":"aa:gdpval_aa:b01dee41-c62b-48ed-8d16-984adc405e5c","observedHarness":"Stirrup","rawScore":1426.15,"zScore":-1.269601,"seModel":"upstream_ci","ciHalfWidth":19.125,"ciLower":1407.03,"ciUpper":1445.28,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:b01dee41-c62b-48ed-8d16-984adc405e5c","observedHarness":"Direct API","rawScore":15.3,"zScore":-0.722098,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Qwen3.8-27B","observedHarness":"Direct API","rawScore":1436.16,"zScore":-1.212215,"seModel":"upstream_ci","ciHalfWidth":3.8568,"ciLower":1432.3019,"ciUpper":1440.0154,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:b01dee41-c62b-48ed-8d16-984adc405e5c","observedHarness":"Direct API","rawScore":22.78,"zScore":0.19875,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:b01dee41-c62b-48ed-8d16-984adc405e5c","observedHarness":"mini-SWE-agent","rawScore":0.48,"zScore":-0.962625,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:alibaba/qwen3.8-27b","observedHarness":"Direct API","rawScore":36.06,"zScore":-0.242113,"seModel":"upstream_se","seRaw":3.337,"ciHalfWidth":6.5405,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.159719,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:alibaba/qwen3.8-27b","observedHarness":"Direct API","rawScore":28.7,"zScore":-2.49116,"seModel":"upstream_se","seRaw":1.971,"ciHalfWidth":3.8632,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.050145,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:alibaba/qwen3.8-27b","observedHarness":"Vals Tax Agent","rawScore":30.89,"zScore":0.070736,"seModel":"upstream_se","seRaw":3.323,"ciHalfWidth":6.5131,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.81533,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"meta:muse-glimmer:xhigh:direct:native","provider":"meta","modelSlug":"muse-glimmer","displayName":"Muse Glimmer","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.062773,"shrinkageFactor":0.995611,"score":34.06,"se":6.64,"samplingSe":0.9415,"residualSe":6.5707,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:Muse Glimmer 30B (xHigh reasoning)","observedHarness":"Direct API","rawScore":0,"zScore":-1.383262,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:Muse Glimmer 30B (xHigh reasoning)","observedHarness":"Direct API","rawScore":0,"zScore":-1.081097,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"}]},{"id":"deepseek:deepseek-v4.1-flash:high:direct:native","provider":"deepseek","modelSlug":"deepseek-v4.1-flash","displayName":"DeepSeek V4.1 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.065655,"shrinkageFactor":0.951629,"score":34.02,"se":7.08,"samplingSe":3.1258,"residualSe":6.3567,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-10","paretoProb":0,"benchmarks":[{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:deepseek/deepseek-v4.1-flash","observedHarness":"Direct API","rawScore":41.17,"zScore":-0.669844,"seModel":"upstream_se","seRaw":2.042,"ciHalfWidth":4.0023,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.01245,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:mysterymechanism:deepseek/deepseek-v4.1-flash","observedHarness":"Direct API","rawScore":21.17,"zScore":-1.039292,"seModel":"upstream_se","seRaw":2.748,"ciHalfWidth":5.3861,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.164089,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism"}]},{"id":"alibaba:qwen3.5-397b-a17b:thinking:direct:native","provider":"alibaba","modelSlug":"qwen3.5-397b-a17b","displayName":"Qwen3.5 397B A17B","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.14805,"shrinkageFactor":0.991712,"score":32.78,"se":4.35,"samplingSe":1.2939,"residualSe":4.1487,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-03-20","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:recooiGyFYUY9K58B","observedHarness":"APEX-Agents Harness","rawScore":24.9,"zScore":-1.58374,"seModel":"upstream_se","seRaw":4.7,"ciHalfWidth":9.212,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:0e66bae9-41f1-42fc-9276-ce8cb6f72919","observedHarness":"Direct API","rawScore":12.2,"zScore":-0.813077,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:0e66bae9-41f1-42fc-9276-ce8cb6f72919","observedHarness":"Direct API","rawScore":12.4,"zScore":-0.880191,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Qwen3.5-397B-A17B","observedHarness":"Direct API","rawScore":1444.4,"zScore":-0.824687,"seModel":"upstream_ci","ciHalfWidth":2.4766,"ciLower":1441.9196,"ciUpper":1446.8728,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:0e66bae9-41f1-42fc-9276-ce8cb6f72919","observedHarness":"Direct API","rawScore":8.89,"zScore":-1.35428,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"deepseek:deepseek-v4-flash:max:direct:native","provider":"deepseek","modelSlug":"deepseek-v4-flash","displayName":"DeepSeek V4 Flash","effort":"max","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.205011,"shrinkageFactor":0.995883,"score":31.92,"se":5.41,"samplingSe":0.9119,"residualSe":5.3362,"coverageCount":3,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-31","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_simpleqa_verified:TqFmiGvqozt7TSZYmhgF8m","observedHarness":"Direct API","rawScore":33.63,"zScore":-1.300404,"seModel":"upstream_se","seRaw":1.495532,"ciHalfWidth":2.9312,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-08-27","publishedDate":"2026-08-27","sourceUrl":"https://epoch.ai/benchmarks/simpleqa-verified"},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-finance:DeepSeek V4 Flash (Max reasoning)","observedHarness":"Direct API","rawScore":0.3,"zScore":-1.360379,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-finance"},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","sourceId":"surge","sourceRowId":"surge:dayjob-healthcare:DeepSeek V4 Flash (Max reasoning)","observedHarness":"Direct API","rawScore":0,"zScore":-1.081097,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.surgehq.ai/benchmarks/dayjob-healthcare"}]},{"id":"alibaba:qwen3.7-plus:thinking:direct:native","provider":"alibaba","modelSlug":"qwen3.7-plus","displayName":"Qwen3.7 Plus","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.211755,"shrinkageFactor":0.993374,"score":31.82,"se":3.59,"samplingSe":1.1569,"residualSe":3.401,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-10","paretoProb":0,"benchmarks":[{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:a87fce42-eea3-4e45-a96d-fe057814f371","observedHarness":"Direct API","rawScore":12.2,"zScore":-0.813077,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:a87fce42-eea3-4e45-a96d-fe057814f371","observedHarness":"Direct API","rawScore":13.9,"zScore":-0.798419,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Qwen3.7-Plus","observedHarness":"Direct API","rawScore":1452.26,"zScore":-0.45503,"seModel":"upstream_ci","ciHalfWidth":3.511,"ciLower":1448.751,"ciUpper":1455.773,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:a87fce42-eea3-4e45-a96d-fe057814f371","observedHarness":"Direct API","rawScore":8.89,"zScore":-1.35428,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:alibaba/qwen3.7-plus","observedHarness":"Direct API","rawScore":16.35,"zScore":-1.990492,"seModel":"upstream_se","seRaw":2.57,"ciHalfWidth":5.0372,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.295949,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:alibaba/qwen3.7-plus","observedHarness":"Vals Tax Agent","rawScore":10.47,"zScore":-1.729189,"seModel":"upstream_se","seRaw":1.993,"ciHalfWidth":3.9063,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.093885,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"deepseek:deepseek-v3.2-exp:none:direct:native","provider":"deepseek","modelSlug":"deepseek-v3.2-exp","displayName":"DeepSeek V3.2-Exp","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.342968,"shrinkageFactor":0.978063,"score":29.86,"se":9.28,"samplingSe":2.105,"residualSe":9.0397,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-09-29","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:DeepSeek-V3.2-Exp","observedHarness":"Direct API","rawScore":1436.27,"zScore":-1.207042,"seModel":"upstream_ci","ciHalfWidth":5.9137,"ciLower":1430.3583,"ciUpper":1442.1857,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"alibaba:qwen3.8-27b:medium:direct:native","provider":"alibaba","modelSlug":"qwen3.8-27b","displayName":"Qwen3.8 27B","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.392895,"shrinkageFactor":0.994762,"score":29.11,"se":3.52,"samplingSe":1.0286,"residualSe":3.3627,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":2.160005,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-05","paretoProb":0,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:6657d7de-a2a9-40bc-a32f-86e80ad63698","observedHarness":"Direct API","rawScore":17.87,"zScore":-1.654936,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:6657d7de-a2a9-40bc-a32f-86e80ad63698","observedHarness":"Stirrup","rawScore":16.5,"zScore":-1.112014,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:6657d7de-a2a9-40bc-a32f-86e80ad63698","observedHarness":"Direct API","rawScore":12.8,"zScore":-0.742264,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:6657d7de-a2a9-40bc-a32f-86e80ad63698","observedHarness":"Direct API","rawScore":6.7,"zScore":-1.190925,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:6657d7de-a2a9-40bc-a32f-86e80ad63698","observedHarness":"Direct API","rawScore":3.89,"zScore":-1.913327,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:6657d7de-a2a9-40bc-a32f-86e80ad63698","observedHarness":"mini-SWE-agent","rawScore":0.95,"zScore":-0.941061,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"spacexai:grok-4.1:thinking:direct:native","provider":"spacexai","modelSlug":"grok-4.1","displayName":"Grok 4.1","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.440183,"shrinkageFactor":0.994519,"score":28.4,"se":9.25,"samplingSe":1.0522,"residualSe":9.1918,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-11-20","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Grok 4.1 (Thinking)","observedHarness":"Direct API","rawScore":1434.6,"zScore":-1.285582,"seModel":"upstream_ci","ciHalfWidth":2.9315,"ciLower":1431.6663,"ciUpper":1437.5293,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"alibaba:qwen3.8-27b:low:direct:native","provider":"alibaba","modelSlug":"qwen3.8-27b","displayName":"Qwen3.8 27B","effort":"low","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.453113,"shrinkageFactor":0.995494,"score":28.2,"se":3.5,"samplingSe":0.954,"residualSe":3.3652,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":1.905905,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-05","paretoProb":0,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:7e30585f-fad7-40df-a8cb-03c1d96df38a","observedHarness":"Direct API","rawScore":18.3,"zScore":-1.624242,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:7e30585f-fad7-40df-a8cb-03c1d96df38a","observedHarness":"Stirrup","rawScore":5.5,"zScore":-1.679347,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:7e30585f-fad7-40df-a8cb-03c1d96df38a","observedHarness":"Direct API","rawScore":10.4,"zScore":-1.025516,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:7e30585f-fad7-40df-a8cb-03c1d96df38a","observedHarness":"Direct API","rawScore":6.8,"zScore":-1.185474,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:7e30585f-fad7-40df-a8cb-03c1d96df38a","observedHarness":"Direct API","rawScore":7.78,"zScore":-1.478389,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:7e30585f-fad7-40df-a8cb-03c1d96df38a","observedHarness":"mini-SWE-agent","rawScore":0.48,"zScore":-0.962625,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"openai:gpt-oss-120b:high:direct:native","provider":"openai","modelSlug":"gpt-oss-120b","displayName":"gpt-oss-120b","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.456671,"shrinkageFactor":0.996454,"score":28.15,"se":2.74,"samplingSe":0.8464,"residualSe":2.61,"coverageCount":8,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":0.345342,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-08-05","paretoProb":0,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:f0083258-8646-45b8-8082-7aaf6c2ea82a","observedHarness":"Direct API","rawScore":21.67,"zScore":-1.383687,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:f0083258-8646-45b8-8082-7aaf6c2ea82a","observedHarness":"Stirrup","rawScore":4,"zScore":-1.756711,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:f0083258-8646-45b8-8082-7aaf6c2ea82a","observedHarness":"Direct API","rawScore":4.2,"zScore":-1.757252,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:f0083258-8646-45b8-8082-7aaf6c2ea82a","observedHarness":"Direct API","rawScore":3.8,"zScore":-1.349018,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:f0083258-8646-45b8-8082-7aaf6c2ea82a","observedHarness":"Direct API","rawScore":1.67,"zScore":-2.161543,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:gpt-oss-120b (one-pass-agent) high","observedHarness":"Scale one-pass-agent","rawScore":43.8,"zScore":-0.407627,"seModel":"upstream_ci","ciHalfWidth":1.9609,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2025-11-13","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:gpt-oss-120b (one-pass-agent) high","observedHarness":"Scale one-pass-agent","rawScore":40.21,"zScore":-1.836752,"seModel":"upstream_ci","ciHalfWidth":2.0687,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2025-11-13","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:f0083258-8646-45b8-8082-7aaf6c2ea82a","observedHarness":"mini-SWE-agent","rawScore":0,"zScore":-0.984647,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"deepseek:deepseek-v3.2:none:direct:native","provider":"deepseek","modelSlug":"deepseek-v3.2","displayName":"DeepSeek V3.2","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.474488,"shrinkageFactor":0.99057,"score":27.88,"se":7.02,"samplingSe":1.3802,"residualSe":6.8857,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-12-01","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:rec38Wcf41rxdUK44","observedHarness":"APEX-Agents Harness","rawScore":21.3,"zScore":-1.800204,"seModel":"upstream_se","seRaw":4.1,"ciHalfWidth":8.036,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:DeepSeek-V3.2","observedHarness":"Direct API","rawScore":1436.83,"zScore":-1.180705,"seModel":"upstream_ci","ciHalfWidth":3.1834,"ciLower":1433.6448,"ciUpper":1440.0116,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemma-4-31b:thinking:direct:native","provider":"google","modelSlug":"gemma-4-31b","displayName":"Gemma 4 31B","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.49504,"shrinkageFactor":0.997791,"score":27.57,"se":3.81,"samplingSe":0.6679,"residualSe":3.7484,"coverageCount":5,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-01","paretoProb":0,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:cd26a386-4873-46ff-b853-d239050025a2","observedHarness":"Direct API","rawScore":19.93,"zScore":-1.50789,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:cd26a386-4873-46ff-b853-d239050025a2","observedHarness":"Stirrup","rawScore":4,"zScore":-1.756711,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:cd26a386-4873-46ff-b853-d239050025a2","observedHarness":"Direct API","rawScore":6,"zScore":-1.544813,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:cd26a386-4873-46ff-b853-d239050025a2","observedHarness":"Direct API","rawScore":7.2,"zScore":-1.163668,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:cd26a386-4873-46ff-b853-d239050025a2","observedHarness":"mini-SWE-agent","rawScore":0,"zScore":-0.984647,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"google:gemma-4-26b-a4b:thinking:direct:native","provider":"google","modelSlug":"gemma-4-26b-a4b","displayName":"Gemma 4 26B A4B","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.505675,"shrinkageFactor":0.998885,"score":27.41,"se":4.4,"samplingSe":0.4747,"residualSe":4.3789,"coverageCount":4,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-01","paretoProb":0,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:6e1b44ff-c227-496b-aef4-19b70cd18c76","observedHarness":"Direct API","rawScore":18.4,"zScore":-1.617104,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:6e1b44ff-c227-496b-aef4-19b70cd18c76","observedHarness":"Direct API","rawScore":2.6,"zScore":-1.946087,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:6e1b44ff-c227-496b-aef4-19b70cd18c76","observedHarness":"Direct API","rawScore":6.2,"zScore":-1.218183,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:6e1b44ff-c227-496b-aef4-19b70cd18c76","observedHarness":"mini-SWE-agent","rawScore":0,"zScore":-0.984647,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"google:gemini-3.1-flash-lite-preview:minimal:direct:native","provider":"google","modelSlug":"gemini-3.1-flash-lite-preview","displayName":"Gemini 3.1 Flash-Lite Preview","effort":"minimal","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.51198,"shrinkageFactor":0.993985,"score":27.32,"se":9.25,"samplingSe":1.1022,"residualSe":9.1869,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-03-03","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Gemini 3.1 Flash-Lite Preview","observedHarness":"Direct API","rawScore":1433.05,"zScore":-1.358479,"seModel":"upstream_ci","ciHalfWidth":3.0716,"ciLower":1429.9808,"ciUpper":1436.1241,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"alibaba:qwen3.6-35b-a3b:thinking:direct:native","provider":"alibaba","modelSlug":"qwen3.6-35b-a3b","displayName":"Qwen3.6 35B A3B","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.523358,"shrinkageFactor":0.989583,"score":27.15,"se":5.9,"samplingSe":1.4505,"residualSe":5.7193,"coverageCount":3,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-05-20","paretoProb":0,"benchmarks":[{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:8665ca00-c687-44c7-875c-22618cb31c4f","observedHarness":"Direct API","rawScore":6.6,"zScore":-1.473999,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:8665ca00-c687-44c7-875c-22618cb31c4f","observedHarness":"Direct API","rawScore":6.9,"zScore":-1.180022,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:8665ca00-c687-44c7-875c-22618cb31c4f","observedHarness":"Direct API","rawScore":5,"zScore":-1.789218,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"deepseek:deepseek-v3.2:thinking:direct:native","provider":"deepseek","modelSlug":"deepseek-v3.2","displayName":"DeepSeek V3.2","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.532,"shrinkageFactor":0.993223,"score":27.02,"se":9.25,"samplingSe":1.17,"residualSe":9.1798,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-12-01","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:DeepSeek-V3.2 (Thinking)","observedHarness":"Direct API","rawScore":1432.6,"zScore":-1.379643,"seModel":"upstream_ci","ciHalfWidth":3.2618,"ciLower":1429.3403,"ciUpper":1435.8639,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"alibaba:qwen3.5-9b:thinking:direct:native","provider":"alibaba","modelSlug":"qwen3.5-9b","displayName":"Qwen3.5 9B","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.555525,"shrinkageFactor":0.99439,"score":26.67,"se":7.24,"samplingSe":1.0645,"residualSe":7.1617,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-03-20","paretoProb":0,"benchmarks":[{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:2d28a13a-096e-475a-beb8-26bbd1c7d51c","observedHarness":"Direct API","rawScore":6.6,"zScore":-1.473999,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:2d28a13a-096e-475a-beb8-26bbd1c7d51c","observedHarness":"Direct API","rawScore":4.8,"zScore":-1.294503,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"meta:muse-glimmer:high:direct:native","provider":"meta","modelSlug":"muse-glimmer","displayName":"Muse Glimmer","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.568764,"shrinkageFactor":0.996378,"score":26.47,"se":3.12,"samplingSe":0.8553,"residualSe":2.999,"coverageCount":7,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":0.178208,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_analyst_agent:583f98fb-c4b8-4df3-8d40-60ac0ed69882","observedHarness":"AA Analyst Harness","rawScore":20,"zScore":-1.57492,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent"},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:583f98fb-c4b8-4df3-8d40-60ac0ed69882","observedHarness":"Direct API","rawScore":27.97,"zScore":-0.933983,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:583f98fb-c4b8-4df3-8d40-60ac0ed69882","observedHarness":"Stirrup","rawScore":7,"zScore":-1.601983,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:583f98fb-c4b8-4df3-8d40-60ac0ed69882","observedHarness":"Direct API","rawScore":10,"zScore":-1.072725,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:583f98fb-c4b8-4df3-8d40-60ac0ed69882","observedHarness":"Direct API","rawScore":9.1,"zScore":-1.06009,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Muse Glimmer","observedHarness":"Direct API","rawScore":1402.33,"zScore":-2.803245,"seModel":"upstream_ci","ciHalfWidth":8.4685,"ciLower":1393.8577,"ciUpper":1410.7948,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:583f98fb-c4b8-4df3-8d40-60ac0ed69882","observedHarness":"mini-SWE-agent","rawScore":0,"zScore":-0.984647,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"alibaba:qwen3.5-122b-a10b:thinking:direct:native","provider":"alibaba","modelSlug":"qwen3.5-122b-a10b","displayName":"Qwen3.5 122B A10B","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.574149,"shrinkageFactor":0.992855,"score":26.39,"se":4.76,"samplingSe":1.2013,"residualSe":4.6086,"coverageCount":4,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-03-20","paretoProb":0,"benchmarks":[{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:2236df45-0699-40d1-b5cc-69ee345d2257","observedHarness":"Direct API","rawScore":3.4,"zScore":-1.851669,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:2236df45-0699-40d1-b5cc-69ee345d2257","observedHarness":"Direct API","rawScore":7.3,"zScore":-1.158216,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Qwen3.5-122B-A10B","observedHarness":"Direct API","rawScore":1427.79,"zScore":-1.605858,"seModel":"upstream_ci","ciHalfWidth":3.8701,"ciLower":1423.917,"ciUpper":1431.6572,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:2236df45-0699-40d1-b5cc-69ee345d2257","observedHarness":"Direct API","rawScore":7.78,"zScore":-1.478389,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"spacexai:grok-4.1:none:direct:native","provider":"spacexai","modelSlug":"grok-4.1","displayName":"Grok 4.1","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.577937,"shrinkageFactor":0.994695,"score":26.33,"se":9.25,"samplingSe":1.0352,"residualSe":9.1934,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-11-20","paretoProb":0,"benchmarks":[{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:Grok 4.1","observedHarness":"Direct API","rawScore":1431.66,"zScore":-1.423851,"seModel":"upstream_ci","ciHalfWidth":2.8837,"ciLower":1428.7777,"ciUpper":1434.5452,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]},{"id":"google:gemma-4-12b:thinking:direct:native","provider":"google","modelSlug":"gemma-4-12b","displayName":"Gemma 4 12B","effort":"thinking","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.601445,"shrinkageFactor":0.998197,"score":25.98,"se":5.48,"samplingSe":0.6035,"residualSe":5.4482,"coverageCount":3,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-07-01","paretoProb":0,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:feb02d3b-ff8d-4ed4-b165-13f8d4a7192c","observedHarness":"Direct API","rawScore":15.67,"zScore":-1.811976,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:feb02d3b-ff8d-4ed4-b165-13f8d4a7192c","observedHarness":"Direct API","rawScore":1.8,"zScore":-2.040504,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:feb02d3b-ff8d-4ed4-b165-13f8d4a7192c","observedHarness":"mini-SWE-agent","rawScore":0,"zScore":-0.984647,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"alibaba:qwen3.8-27b:none:direct:native","provider":"alibaba","modelSlug":"qwen3.8-27b","displayName":"Qwen3.8 27B","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.648043,"shrinkageFactor":0.996093,"score":25.28,"se":3.48,"samplingSe":0.8884,"residualSe":3.3672,"coverageCount":6,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":3.772944,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-05","paretoProb":0,"benchmarks":[{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","sourceId":"artificial-analysis","sourceRowId":"aa:aa_omniscience:353c92f3-2148-4c2d-9231-aa7e1322a1fd","observedHarness":"Direct API","rawScore":9.1,"zScore":-2.280952,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:critpt_v2_hard:353c92f3-2148-4c2d-9231-aa7e1322a1fd","observedHarness":"Stirrup","rawScore":2,"zScore":-1.859862,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:353c92f3-2148-4c2d-9231-aa7e1322a1fd","observedHarness":"Direct API","rawScore":5.2,"zScore":-1.63923,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:353c92f3-2148-4c2d-9231-aa7e1322a1fd","observedHarness":"Direct API","rawScore":4.2,"zScore":-1.327212,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:353c92f3-2148-4c2d-9231-aa7e1322a1fd","observedHarness":"Direct API","rawScore":10,"zScore":-1.230172,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","sourceId":"artificial-analysis","sourceRowId":"aa:terminalbench_science_aa:353c92f3-2148-4c2d-9231-aa7e1322a1fd","observedHarness":"mini-SWE-agent","rawScore":0,"zScore":-0.984647,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/evaluations/terminal-bench-science"}]},{"id":"alibaba:qwen3-coder-next:none:direct:native","provider":"alibaba","modelSlug":"qwen3-coder-next","displayName":"Qwen3 Coder Next","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.681769,"shrinkageFactor":0.997021,"score":24.77,"se":7.22,"samplingSe":0.7757,"residualSe":7.1806,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-02-03","paretoProb":0,"benchmarks":[{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:fc92f822-04b7-420d-9c07-a21af5e9aac7","observedHarness":"Direct API","rawScore":2.6,"zScore":-1.946087,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:fc92f822-04b7-420d-9c07-a21af5e9aac7","observedHarness":"Direct API","rawScore":4.3,"zScore":-1.321761,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"google:gemini-3.1-flash-lite-preview:high:direct:native","provider":"google","modelSlug":"gemini-3.1-flash-lite-preview","displayName":"Gemini 3.1 Flash-Lite Preview","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.771463,"shrinkageFactor":0.987845,"score":23.43,"se":5.38,"samplingSe":1.5669,"residualSe":5.1423,"coverageCount":3,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-03-03","paretoProb":0,"benchmarks":[{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:legal_research:google/gemini-3.1-flash-lite-preview","observedHarness":"Direct API","rawScore":3.37,"zScore":-3,"seModel":"upstream_se","seRaw":1.253,"ciHalfWidth":2.4559,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.013607,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/legal_research"},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:medcode:google/gemini-3.1-flash-lite-preview","observedHarness":"Direct API","rawScore":47.6,"zScore":0.269295,"seModel":"upstream_se","seRaw":2.071,"ciHalfWidth":4.0592,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.002029,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/medcode"},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","sourceId":"vals","sourceRowId":"vals:tax_agent_bench:google/gemini-3.1-flash-lite-preview","observedHarness":"Vals Tax Agent","rawScore":3.11,"zScore":-2.377937,"seModel":"upstream_se","seRaw":1.253,"ciHalfWidth":2.4559,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.014308,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench"}]},{"id":"openai:gpt-oss-20b:high:direct:native","provider":"openai","modelSlug":"gpt-oss-20b","displayName":"gpt-oss-20b","effort":"high","harness":"direct","harnessRegime":"native","domainId":"research","theta":-1.791573,"shrinkageFactor":0.996023,"score":23.13,"se":5.83,"samplingSe":0.8963,"residualSe":5.7565,"coverageCount":3,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-08-05","paretoProb":0,"benchmarks":[{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:36f73aaf-d38a-4b56-a2b3-d04d17186910","observedHarness":"Direct API","rawScore":2,"zScore":-2.0169,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:hle_diamond:36f73aaf-d38a-4b56-a2b3-d04d17186910","observedHarness":"Direct API","rawScore":4.8,"zScore":-1.294503,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:mlcr_aa:36f73aaf-d38a-4b56-a2b3-d04d17186910","observedHarness":"Direct API","rawScore":1.11,"zScore":-2.224156,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"meta:llama-4-scout:none:direct:native","provider":"meta","modelSlug":"llama-4-scout","displayName":"Llama 4 Scout","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-2.339152,"shrinkageFactor":0.992415,"score":14.91,"se":9.54,"samplingSe":1.2378,"residualSe":9.4601,"coverageCount":1,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-04-05","paretoProb":0,"benchmarks":[{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:adf9a85e-abc3-4f28-937b-db6655cc5238","observedHarness":"Direct API","rawScore":0,"zScore":-2.252944,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"}]},{"id":"meta:llama-4-maverick:none:direct:native","provider":"meta","modelSlug":"llama-4-maverick","displayName":"Llama 4 Maverick","effort":"none","harness":"direct","harnessRegime":"native","domainId":"research","theta":-2.689568,"shrinkageFactor":0.999344,"score":9.66,"se":5.47,"samplingSe":0.364,"residualSe":5.4573,"coverageCount":3,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-04-05","paretoProb":0,"benchmarks":[{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:gdp_pdf:922c69c7-9037-43c6-8bcf-a1c555e7f3eb","observedHarness":"Direct API","rawScore":0.8,"zScore":-2.158526,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/leaderboards/models"},{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_finance:finance:Llama 4 Maverick (one-pass-agent)","observedHarness":"Scale one-pass-agent","rawScore":22.36,"zScore":-3,"seModel":"upstream_ci","ciHalfWidth":0.4218,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2025-11-13","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","sourceId":"scale-seal","sourceRowId":"seal:scale_prbench_legal:legal:Llama 4 Maverick (one-pass-agent)","observedHarness":"Scale one-pass-agent","rawScore":24.84,"zScore":-3,"seModel":"upstream_ci","ciHalfWidth":0.8958,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2025-11-13","publishedDate":null,"sourceUrl":"https://scale.com/leaderboard/prbench"}]},{"id":"openai:gpt-oss-120b:medium:direct:native","provider":"openai","modelSlug":"gpt-oss-120b","displayName":"gpt-oss-120b","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"research","theta":-3.054963,"shrinkageFactor":0.99864,"score":4.18,"se":6.96,"samplingSe":0.5241,"residualSe":6.9418,"coverageCount":2,"activeBenchmarkCount":27,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"suite","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2025-08-05","paretoProb":0,"benchmarks":[{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","sourceId":"epoch","sourceRowId":"epoch:epoch_apex_agents:rec9zkde8jhxtGWhs","observedHarness":"APEX-Agents Harness","rawScore":4.4,"zScore":-2.816382,"seModel":"upstream_se","seRaw":2,"ciHalfWidth":3.92,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.mercor.com/apex-agents"},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","sourceId":"lmarena","sourceRowId":"lmarena:text_factuality:overall:gpt-oss-120b","observedHarness":"Direct API","rawScore":1384.48,"zScore":-3,"seModel":"upstream_ci","ciHalfWidth":6.655,"ciLower":1377.8216,"ciUpper":1391.1315,"measuredCostUsd":null,"evalDate":null,"publishedDate":"2026-11-19","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"}]}],"frontiers":[{"provider":"anthropic","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"anthropic:claude-haiku-5.5:low:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9979,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"anthropic:claude-haiku-5.5:medium:direct:native","zStepFromPrev":1.029,"pBetterThanPrev":0.8482,"statisticallyTied":false,"imputationSensitive":true,"paretoProb":0.8358,"marginalScorePerDollar":82.696,"costMultipleFromPrev":2.1},{"configId":"anthropic:claude-haiku-5.5:high:direct:native","zStepFromPrev":0.132,"pBetterThanPrev":0.5525,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.4802,"marginalScorePerDollar":7.635,"costMultipleFromPrev":1.73},{"configId":"anthropic:claude-haiku-5.5:xhigh:direct:native","zStepFromPrev":1.639,"pBetterThanPrev":0.9494,"statisticallyTied":false,"imputationSensitive":true,"paretoProb":0.7781,"marginalScorePerDollar":71.38,"costMultipleFromPrev":1.57},{"configId":"anthropic:claude-haiku-5.5:max:direct:native","zStepFromPrev":1.049,"pBetterThanPrev":0.853,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.5926,"marginalScorePerDollar":24.593,"costMultipleFromPrev":1.65},{"configId":"anthropic:claude-opus-5.5:low:direct:native","zStepFromPrev":1.281,"pBetterThanPrev":0.8999,"statisticallyTied":false,"imputationSensitive":true,"paretoProb":0.3425,"marginalScorePerDollar":9.194,"costMultipleFromPrev":2.37},{"configId":"anthropic:claude-opus-5.5:medium:direct:native","zStepFromPrev":1.712,"pBetterThanPrev":0.9565,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.2222,"marginalScorePerDollar":4.683,"costMultipleFromPrev":2.51},{"configId":"anthropic:claude-opus-4.6:none:direct:native","zStepFromPrev":0.328,"pBetterThanPrev":0.6284,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.4783,"marginalScorePerDollar":3.936,"costMultipleFromPrev":1.21},{"configId":"anthropic:claude-fable-5.2:low:direct:native","zStepFromPrev":1.133,"pBetterThanPrev":0.8713,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.6262,"marginalScorePerDollar":62.557,"costMultipleFromPrev":1.04},{"configId":"anthropic:claude-fable-5.2:medium:direct:native","zStepFromPrev":0.553,"pBetterThanPrev":0.71,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.5228,"marginalScorePerDollar":3.431,"costMultipleFromPrev":1.23},{"configId":"anthropic:claude-fable-5.2:xhigh:direct:native","zStepFromPrev":1.175,"pBetterThanPrev":0.8801,"statisticallyTied":false,"imputationSensitive":true,"paretoProb":0.6335,"marginalScorePerDollar":0.462,"costMultipleFromPrev":3.9},{"configId":"anthropic:claude-fable-5.2:max:direct:native","zStepFromPrev":0.268,"pBetterThanPrev":0.6056,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.4655,"marginalScorePerDollar":0.094,"costMultipleFromPrev":1.79}],"pruned":[{"configId":"anthropic:claude-sonnet-5.5:low:direct:native","dominatedById":"anthropic:claude-haiku-5.5:max:direct:native","scoreDeficit":6.99,"costRatio":1.24,"costComparisonNote":null},{"configId":"anthropic:claude-haiku-4.5:none:direct:native","dominatedById":"anthropic:claude-haiku-5.5:max:direct:native","scoreDeficit":37.86,"costRatio":1.7,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5.5:medium:direct:native","dominatedById":"anthropic:claude-haiku-5.5:max:direct:native","scoreDeficit":4.42,"costRatio":1.9,"costComparisonNote":null},{"configId":"anthropic:claude-haiku-4.5:thinking:direct:native","dominatedById":"anthropic:claude-opus-5.5:low:direct:native","scoreDeficit":38.16,"costRatio":1.46,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5.5:high:direct:native","dominatedById":"anthropic:claude-opus-5.5:low:direct:native","scoreDeficit":0.74,"costRatio":1.68,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-4.6:none:direct:native","dominatedById":"anthropic:claude-opus-5.5:low:direct:native","scoreDeficit":8.29,"costRatio":1.92,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-4.5:none:direct:native","dominatedById":"anthropic:claude-opus-5.5:low:direct:native","scoreDeficit":25.83,"costRatio":1.92,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.7:none:direct:native","dominatedById":"anthropic:claude-opus-4.6:none:direct:native","scoreDeficit":6.5,"costRatio":1,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.8:none:direct:native","dominatedById":"anthropic:claude-opus-4.6:none:direct:native","scoreDeficit":14.55,"costRatio":1,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.5:none:direct:native","dominatedById":"anthropic:claude-opus-4.6:none:direct:native","scoreDeficit":18.32,"costRatio":1,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5.5:high:direct:native","dominatedById":"anthropic:claude-fable-5.2:low:direct:native","scoreDeficit":4.95,"costRatio":1.12,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5:high:direct:native","dominatedById":"anthropic:claude-fable-5.2:low:direct:native","scoreDeficit":31.32,"costRatio":1.19,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-4.5:thinking:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":34.17,"costRatio":1,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5.5:xhigh:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":14.28,"costRatio":1.08,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5:xhigh:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":45.34,"costRatio":1.19,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-4.5:high:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":37.92,"costRatio":1.4,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-4.6:high:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":39.27,"costRatio":1.4,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5.5:xhigh:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":6.27,"costRatio":1.59,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5:max:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":30.68,"costRatio":1.71,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.6:high:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":4.77,"costRatio":2.22,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.7:high:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":15.02,"costRatio":2.22,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5:high:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":15.36,"costRatio":2.22,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.8:high:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":20.54,"costRatio":2.22,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.5:high:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":24.11,"costRatio":2.22,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.2:high:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":2.93,"costRatio":2.39,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-4.6:max:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":32.99,"costRatio":2.46,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5.5:max:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":6.74,"costRatio":2.64,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5.5:max:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":2.46,"costRatio":2.64,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.7:xhigh:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":25.53,"costRatio":2.72,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5:max:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":7.68,"costRatio":3.9,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.8:max:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":20.77,"costRatio":3.9,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.7:max:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":23.28,"costRatio":3.9,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.6:max:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":27.79,"costRatio":3.9,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.5:max:direct:native","dominatedById":"anthropic:claude-fable-5.2:medium:direct:native","scoreDeficit":30.4,"costRatio":3.9,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5:high:direct:native","dominatedById":"anthropic:claude-fable-5.2:xhigh:direct:native","scoreDeficit":15.94,"costRatio":1.06,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.1:high:direct:native","dominatedById":"anthropic:claude-fable-5.2:xhigh:direct:native","scoreDeficit":22.77,"costRatio":1.06,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5:xhigh:direct:native","dominatedById":"anthropic:claude-fable-5.2:xhigh:direct:native","scoreDeficit":10.57,"costRatio":1.31,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.1:xhigh:direct:native","dominatedById":"anthropic:claude-fable-5.2:xhigh:direct:native","scoreDeficit":19.35,"costRatio":1.31,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.1:max:direct:native","dominatedById":"anthropic:claude-fable-5.2:max:direct:native","scoreDeficit":10.25,"costRatio":1.05,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5:max:direct:native","dominatedById":"anthropic:claude-fable-5.2:max:direct:native","scoreDeficit":21.5,"costRatio":1.05,"costComparisonNote":null}],"verdicts":{"budgetId":"anthropic:claude-haiku-5.5:low:direct:native","defaultId":"anthropic:claude-fable-5.2:medium:direct:native","maxId":"anthropic:claude-fable-5.2:max:direct:native"}},{"provider":"openai","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"openai:gpt-6-luna:low:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"openai:gpt-6-luna:medium:direct:native","zStepFromPrev":1.121,"pBetterThanPrev":0.8689,"statisticallyTied":false,"imputationSensitive":true,"paretoProb":0.8535,"marginalScorePerDollar":115.674,"costMultipleFromPrev":8.25},{"configId":"openai:gpt-6.1-terra:low:direct:native","zStepFromPrev":1.69,"pBetterThanPrev":0.9545,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9254,"marginalScorePerDollar":299.286,"costMultipleFromPrev":1.69},{"configId":"openai:gpt-6.1-terra:medium:direct:native","zStepFromPrev":1.43,"pBetterThanPrev":0.9236,"statisticallyTied":false,"imputationSensitive":true,"paretoProb":0.84,"marginalScorePerDollar":62.79,"costMultipleFromPrev":2.37},{"configId":"openai:gpt-6.2-sol:low:direct:native","zStepFromPrev":0.935,"pBetterThanPrev":0.8251,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.7797,"marginalScorePerDollar":327.47,"costMultipleFromPrev":1.09},{"configId":"openai:gpt-6.1-terra:xhigh:direct:native","zStepFromPrev":0.239,"pBetterThanPrev":0.5943,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.4413,"marginalScorePerDollar":5.712,"costMultipleFromPrev":2.18},{"configId":"openai:gpt-6.2-sol:medium:direct:native","zStepFromPrev":1.473,"pBetterThanPrev":0.9296,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.8727,"marginalScorePerDollar":1425.377,"costMultipleFromPrev":1.01},{"configId":"openai:gpt-6.2-sol:high:direct:native","zStepFromPrev":0.803,"pBetterThanPrev":0.7891,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.7532,"marginalScorePerDollar":20.179,"costMultipleFromPrev":1.39},{"configId":"openai:gpt-6.2-astra:medium:direct:native","zStepFromPrev":0.289,"pBetterThanPrev":0.6136,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.5098,"marginalScorePerDollar":1.319,"costMultipleFromPrev":2.87},{"configId":"openai:gpt-6.2-astra:high:direct:native","zStepFromPrev":1.041,"pBetterThanPrev":0.8511,"statisticallyTied":false,"imputationSensitive":true,"paretoProb":0.7133,"marginalScorePerDollar":8.944,"costMultipleFromPrev":1.28}],"pruned":[{"configId":"openai:gpt-6-luna:none:direct:native","dominatedById":"openai:gpt-6.1-terra:low:direct:native","scoreDeficit":16.58,"costRatio":1,"costComparisonNote":null},{"configId":"openai:gpt-6-luna:high:direct:native","dominatedById":"openai:gpt-6.1-terra:low:direct:native","scoreDeficit":7.13,"costRatio":1.02,"costComparisonNote":null},{"configId":"openai:gpt-6-luna:xhigh:direct:native","dominatedById":"openai:gpt-6.1-terra:low:direct:native","scoreDeficit":6,"costRatio":1.46,"costComparisonNote":null},{"configId":"openai:gpt-6-luna:max:direct:native","dominatedById":"openai:gpt-6.1-terra:low:direct:native","scoreDeficit":5.42,"costRatio":2.13,"costComparisonNote":null},{"configId":"openai:gpt-6.1-terra:none:direct:native","dominatedById":"openai:gpt-6.2-sol:low:direct:native","scoreDeficit":14.21,"costRatio":1.23,"costComparisonNote":null},{"configId":"openai:gpt-6.1-terra:high:direct:native","dominatedById":"openai:gpt-6.2-sol:low:direct:native","scoreDeficit":1.77,"costRatio":1.41,"costComparisonNote":null},{"configId":"openai:gpt-5.4-nano:high:direct:native","dominatedById":"openai:gpt-6.2-sol:high:direct:native","scoreDeficit":54.29,"costRatio":1.02,"costComparisonNote":null},{"configId":"openai:gpt-6.2-sol:xhigh:direct:native","dominatedById":"openai:gpt-6.2-sol:high:direct:native","scoreDeficit":4.13,"costRatio":1.15,"costComparisonNote":null},{"configId":"openai:gpt-6.2-astra:low:direct:native","dominatedById":"openai:gpt-6.2-sol:high:direct:native","scoreDeficit":5.09,"costRatio":1.17,"costComparisonNote":null},{"configId":"openai:gpt-5.6-luna:xhigh:direct:native","dominatedById":"openai:gpt-6.2-sol:high:direct:native","scoreDeficit":24.19,"costRatio":1.21,"costComparisonNote":null},{"configId":"openai:gpt-6.1-terra:max:direct:native","dominatedById":"openai:gpt-6.2-sol:high:direct:native","scoreDeficit":16.4,"costRatio":1.71,"costComparisonNote":null},{"configId":"openai:gpt-5.6-luna:max:direct:native","dominatedById":"openai:gpt-6.2-sol:high:direct:native","scoreDeficit":31.7,"costRatio":1.72,"costComparisonNote":null},{"configId":"openai:gpt-6.2-sol:max:direct:native","dominatedById":"openai:gpt-6.2-sol:high:direct:native","scoreDeficit":7.48,"costRatio":1.8,"costComparisonNote":null},{"configId":"openai:gpt-5.1:none:direct:native","dominatedById":"openai:gpt-6.2-sol:high:direct:native","scoreDeficit":35.5,"costRatio":2.26,"costComparisonNote":null},{"configId":"openai:o3:medium:direct:native","dominatedById":"openai:gpt-6.2-astra:medium:direct:native","scoreDeficit":63.69,"costRatio":1.01,"costComparisonNote":null},{"configId":"openai:gpt-5.2:none:direct:native","dominatedById":"openai:gpt-6.2-astra:medium:direct:native","scoreDeficit":38.7,"costRatio":1.07,"costComparisonNote":null},{"configId":"openai:gpt-5.2:low:direct:native","dominatedById":"openai:gpt-6.2-astra:medium:direct:native","scoreDeficit":43.3,"costRatio":1.07,"costComparisonNote":null},{"configId":"openai:gpt-5.4-mini:high:direct:native","dominatedById":"openai:gpt-6.2-astra:medium:direct:native","scoreDeficit":41.28,"costRatio":1.11,"costComparisonNote":null},{"configId":"openai:gpt-5.4:none:direct:native","dominatedById":"openai:gpt-6.2-astra:medium:direct:native","scoreDeficit":15.15,"costRatio":1.15,"costComparisonNote":null},{"configId":"openai:gpt-5.4-mini:xhigh:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":54.23,"costRatio":1.06,"costComparisonNote":null},{"configId":"openai:gpt-6.2-astra:xhigh:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":4.77,"costRatio":1.22,"costComparisonNote":null},{"configId":"openai:gpt-5.2:medium:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":47.49,"costRatio":1.26,"costComparisonNote":null},{"configId":"openai:o3:high:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":26.85,"costRatio":1.48,"costComparisonNote":null},{"configId":"openai:gpt-5.1:high:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":27.44,"costRatio":1.75,"costComparisonNote":null},{"configId":"openai:gpt-5:high:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":30.69,"costRatio":1.75,"costComparisonNote":null},{"configId":"openai:gpt-6.2-astra:max:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":6.11,"costRatio":1.93,"costComparisonNote":null},{"configId":"openai:gpt-6.1-sol:xhigh:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":26.29,"costRatio":2.19,"costComparisonNote":null},{"configId":"openai:gpt-6-sol:xhigh:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":50.88,"costRatio":2.19,"costComparisonNote":null},{"configId":"openai:gpt-5.2:high:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":44.54,"costRatio":2.36,"costComparisonNote":null},{"configId":"openai:gpt-5.5:medium:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":14.67,"costRatio":2.52,"costComparisonNote":null},{"configId":"openai:gpt-5.4:high:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":25.67,"costRatio":2.54,"costComparisonNote":null},{"configId":"openai:gpt-5.6-terra:xhigh:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":33.15,"costRatio":2.55,"costComparisonNote":null},{"configId":"openai:gpt-5.2:xhigh:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":34.74,"costRatio":2.9,"costComparisonNote":null},{"configId":"openai:gpt-5.1:max:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":22.32,"costRatio":3.06,"costComparisonNote":null},{"configId":"openai:gpt-5.4:xhigh:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":37.99,"costRatio":3.12,"costComparisonNote":null},{"configId":"openai:gpt-6.1-sol:max:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":16.8,"costRatio":3.13,"costComparisonNote":null},{"configId":"openai:gpt-6-sol:max:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":31.74,"costRatio":3.13,"costComparisonNote":null},{"configId":"openai:gpt-5.6-terra:max:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":26.86,"costRatio":3.65,"costComparisonNote":null},{"configId":"openai:gpt-5.6-sol:xhigh:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":18.34,"costRatio":4.08,"costComparisonNote":null},{"configId":"openai:gpt-5.5:high:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":11.99,"costRatio":4.75,"costComparisonNote":null},{"configId":"openai:gpt-5.5:xhigh:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":25.26,"costRatio":5.83,"costComparisonNote":null},{"configId":"openai:gpt-5.6-sol:max:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":20.24,"costRatio":5.84,"costComparisonNote":null},{"configId":"openai:gpt-6-astra:xhigh:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":24.38,"costRatio":9.35,"costComparisonNote":null},{"configId":"openai:gpt-6-astra:max:direct:native","dominatedById":"openai:gpt-6.2-astra:high:direct:native","scoreDeficit":15.12,"costRatio":13.41,"costComparisonNote":null}],"verdicts":{"budgetId":"openai:gpt-6-luna:low:direct:native","defaultId":"openai:gpt-6.2-sol:high:direct:native","maxId":"openai:gpt-6.2-astra:high:direct:native"}},{"provider":"google","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"google:gemini-4-hydrogen:low:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.5764,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"google:gemini-4-neon:low:direct:native","zStepFromPrev":2.586,"pBetterThanPrev":0.9951,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9014,"marginalScorePerDollar":28.013,"costMultipleFromPrev":2.55},{"configId":"google:gemini-4-hydrogen:high:direct:native","zStepFromPrev":0.549,"pBetterThanPrev":0.7086,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.6529,"marginalScorePerDollar":2.092,"costMultipleFromPrev":2.51},{"configId":"google:gemini-4-neon:medium:direct:native","zStepFromPrev":1.268,"pBetterThanPrev":0.8976,"statisticallyTied":false,"imputationSensitive":true,"paretoProb":0.8652,"marginalScorePerDollar":175.405,"costMultipleFromPrev":1.02},{"configId":"google:gemini-4-neon:high:direct:native","zStepFromPrev":1.857,"pBetterThanPrev":0.9683,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9609,"marginalScorePerDollar":3.786,"costMultipleFromPrev":1.97}],"pruned":[{"configId":"google:gemini-3-flash:minimal:direct:native","dominatedById":"google:gemini-4-hydrogen:low:direct:native","scoreDeficit":8.95,"costRatio":1.51,"costComparisonNote":null},{"configId":"google:gemini-3.5-flash-lite:minimal:direct:native","dominatedById":"google:gemini-4-hydrogen:low:direct:native","scoreDeficit":17.26,"costRatio":2.37,"costComparisonNote":null},{"configId":"google:gemini-4-hydrogen:medium:direct:native","dominatedById":"google:gemini-4-neon:low:direct:native","scoreDeficit":5.16,"costRatio":1.07,"costComparisonNote":null},{"configId":"google:gemini-3-flash:high:direct:native","dominatedById":"google:gemini-4-neon:low:direct:native","scoreDeficit":14.65,"costRatio":1.68,"costComparisonNote":null},{"configId":"google:gemini-3.8-flash:high:direct:native","dominatedById":"google:gemini-4-neon:low:direct:native","scoreDeficit":8.38,"costRatio":2.07,"costComparisonNote":null},{"configId":"google:gemini-3.7-flash:high:direct:native","dominatedById":"google:gemini-4-neon:low:direct:native","scoreDeficit":10.49,"costRatio":2.07,"costComparisonNote":null},{"configId":"google:gemini-3.6-flash:high:direct:native","dominatedById":"google:gemini-4-neon:low:direct:native","scoreDeficit":11.94,"costRatio":2.07,"costComparisonNote":null},{"configId":"google:gemini-3.5-flash:medium:direct:native","dominatedById":"google:gemini-4-neon:low:direct:native","scoreDeficit":10.48,"costRatio":2.38,"costComparisonNote":null},{"configId":"google:gemini-3.5-flash-lite:high:direct:native","dominatedById":"google:gemini-4-neon:medium:direct:native","scoreDeficit":43.33,"costRatio":1.33,"costComparisonNote":null},{"configId":"google:gemini-4-hydrogen:xhigh:direct:native","dominatedById":"google:gemini-4-neon:medium:direct:native","scoreDeficit":1.52,"costRatio":1.66,"costComparisonNote":null},{"configId":"google:gemini-3.5-flash:high:direct:native","dominatedById":"google:gemini-4-neon:medium:direct:native","scoreDeficit":15.6,"costRatio":1.75,"costComparisonNote":null},{"configId":"google:gemini-3.1-pro:high:direct:native","dominatedById":"google:gemini-4-neon:high:direct:native","scoreDeficit":32.6,"costRatio":1.15,"costComparisonNote":null},{"configId":"google:gemini-4-neon:xhigh:direct:native","dominatedById":"google:gemini-4-neon:high:direct:native","scoreDeficit":2.73,"costRatio":1.72,"costComparisonNote":null},{"configId":"google:gemini-4-carbon:medium:direct:native","dominatedById":"google:gemini-4-neon:high:direct:native","scoreDeficit":8.03,"costRatio":1.76,"costComparisonNote":null},{"configId":"google:gemini-4-carbon:low:direct:native","dominatedById":"google:gemini-4-neon:high:direct:native","scoreDeficit":9.79,"costRatio":1.92,"costComparisonNote":null},{"configId":"google:gemini-4-carbon:high:direct:native","dominatedById":"google:gemini-4-neon:high:direct:native","scoreDeficit":4.07,"costRatio":2.07,"costComparisonNote":null},{"configId":"google:gemini-4-argon:medium:direct:native","dominatedById":"google:gemini-4-neon:high:direct:native","scoreDeficit":16.55,"costRatio":2.31,"costComparisonNote":null},{"configId":"google:gemini-4-argon:high:direct:native","dominatedById":"google:gemini-4-neon:high:direct:native","scoreDeficit":10.89,"costRatio":2.66,"costComparisonNote":null}],"verdicts":{"budgetId":"google:gemini-4-hydrogen:low:direct:native","defaultId":"google:gemini-4-neon:low:direct:native","maxId":"google:gemini-4-neon:high:direct:native"}},{"provider":"spacexai","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"spacexai:grok-4.20:none:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.7936,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"spacexai:grok-4.6:high:direct:native","zStepFromPrev":0.803,"pBetterThanPrev":0.789,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.5326,"marginalScorePerDollar":3.908,"costMultipleFromPrev":5.94},{"configId":"spacexai:grok-4.6:xhigh:direct:native","zStepFromPrev":0.313,"pBetterThanPrev":0.623,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.6354,"marginalScorePerDollar":3.383,"costMultipleFromPrev":1.22}],"pruned":[{"configId":"spacexai:grok-4.3:low:direct:native","dominatedById":"spacexai:grok-4.20:none:direct:native","scoreDeficit":16.57,"costRatio":1,"costComparisonNote":null},{"configId":"spacexai:grok-4.20:thinking:direct:native","dominatedById":"spacexai:grok-4.20:none:direct:native","scoreDeficit":15.91,"costRatio":2.04,"costComparisonNote":null},{"configId":"spacexai:grok-4.3:high:direct:native","dominatedById":"spacexai:grok-4.20:none:direct:native","scoreDeficit":17.52,"costRatio":2.85,"costComparisonNote":null},{"configId":"spacexai:grok-4.5:high:direct:native","dominatedById":"spacexai:grok-4.6:high:direct:native","scoreDeficit":3.62,"costRatio":1,"costComparisonNote":null},{"configId":"spacexai:grok-4.7:low:direct:native","dominatedById":"spacexai:grok-4.6:high:direct:native","scoreDeficit":4.43,"costRatio":1.2,"costComparisonNote":null},{"configId":"spacexai:grok-4.7:high:direct:native","dominatedById":"spacexai:grok-4.6:xhigh:direct:native","scoreDeficit":4.48,"costRatio":2.08,"costComparisonNote":null},{"configId":"spacexai:grok-4.7:xhigh:direct:native","dominatedById":"spacexai:grok-4.6:xhigh:direct:native","scoreDeficit":0.3,"costRatio":2.61,"costComparisonNote":null}],"verdicts":{"budgetId":"spacexai:grok-4.20:none:direct:native","defaultId":null,"maxId":"spacexai:grok-4.6:xhigh:direct:native"}},{"provider":"meta","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"meta:muse-ember:xhigh:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9488,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"meta:muse-ember:max:direct:native","zStepFromPrev":0.07,"pBetterThanPrev":0.528,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.4887,"marginalScorePerDollar":12.685,"costMultipleFromPrev":1.17},{"configId":"meta:muse-spark-1.1:high:direct:native","zStepFromPrev":1.844,"pBetterThanPrev":0.9674,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.4441,"marginalScorePerDollar":5.169,"costMultipleFromPrev":13.01},{"configId":"meta:muse-spark-1.3:xhigh:direct:native","zStepFromPrev":1.727,"pBetterThanPrev":0.9579,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.8838,"marginalScorePerDollar":24.566,"costMultipleFromPrev":1.22},{"configId":"meta:muse-wonder:xhigh:direct:native","zStepFromPrev":1.106,"pBetterThanPrev":0.8656,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.872,"marginalScorePerDollar":2.402,"costMultipleFromPrev":2.61}],"pruned":[{"configId":"meta:muse-spark-1.2:high:direct:native","dominatedById":"meta:muse-spark-1.1:high:direct:native","scoreDeficit":14.01,"costRatio":1,"costComparisonNote":null},{"configId":"meta:muse-spark-1.2:xhigh:direct:native","dominatedById":"meta:muse-spark-1.3:xhigh:direct:native","scoreDeficit":15.04,"costRatio":1,"costComparisonNote":null},{"configId":"meta:muse-spark-1.3:max:direct:native","dominatedById":"meta:muse-spark-1.3:xhigh:direct:native","scoreDeficit":7.31,"costRatio":1.43,"costComparisonNote":null},{"configId":"meta:muse-wonder:high:direct:native","dominatedById":"meta:muse-spark-1.3:xhigh:direct:native","scoreDeficit":17.04,"costRatio":2.11,"costComparisonNote":null},{"configId":"meta:muse-wonder:max:direct:native","dominatedById":"meta:muse-wonder:xhigh:direct:native","scoreDeficit":17.26,"costRatio":1.23,"costComparisonNote":null}],"verdicts":{"budgetId":"meta:muse-ember:xhigh:direct:native","defaultId":"meta:muse-spark-1.3:xhigh:direct:native","maxId":"meta:muse-wonder:xhigh:direct:native"}},{"provider":"deepseek","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"deepseek:deepseek-v4.1-flash-1017:none:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"deepseek:deepseek-v4.1-flash-1017:high:direct:native","zStepFromPrev":3.283,"pBetterThanPrev":0.9995,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9995,"marginalScorePerDollar":427.959,"costMultipleFromPrev":1.07},{"configId":"deepseek:deepseek-v4.1-flash-1017:max:direct:native","zStepFromPrev":2.09,"pBetterThanPrev":0.9817,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9817,"marginalScorePerDollar":57.706,"costMultipleFromPrev":1.32}],"pruned":[{"configId":"deepseek:deepseek-v4-pro-0813:high:direct:native","dominatedById":"deepseek:deepseek-v4.1-flash-1017:max:direct:native","scoreDeficit":9.07,"costRatio":1.41,"costComparisonNote":null},{"configId":"deepseek:deepseek-v4-pro-0813:none:direct:native","dominatedById":"deepseek:deepseek-v4.1-flash-1017:max:direct:native","scoreDeficit":17.92,"costRatio":1.62,"costComparisonNote":null},{"configId":"deepseek:deepseek-v4-pro-0813:max:direct:native","dominatedById":"deepseek:deepseek-v4.1-flash-1017:max:direct:native","scoreDeficit":7.5,"costRatio":2.94,"costComparisonNote":null}],"verdicts":{"budgetId":"deepseek:deepseek-v4.1-flash-1017:none:direct:native","defaultId":"deepseek:deepseek-v4.1-flash-1017:high:direct:native","maxId":"deepseek:deepseek-v4.1-flash-1017:max:direct:native"}},{"provider":"moonshot","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"moonshot:kimi-k2.6:thinking:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.6915,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"moonshot:kimi-k3.1-pro:xhigh:direct:native","zStepFromPrev":2.509,"pBetterThanPrev":0.9939,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9192,"marginalScorePerDollar":21.84,"costMultipleFromPrev":1.75},{"configId":"moonshot:kimi-k3.1-pro:max:direct:native","zStepFromPrev":0.6,"pBetterThanPrev":0.7258,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.7241,"marginalScorePerDollar":2.607,"costMultipleFromPrev":1.79}],"pruned":[{"configId":"moonshot:kimi-k2.7-code:max:direct:native","dominatedById":"moonshot:kimi-k3.1-pro:xhigh:direct:native","scoreDeficit":16.25,"costRatio":1.4,"costComparisonNote":null},{"configId":"moonshot:kimi-k3:max:direct:native","dominatedById":"moonshot:kimi-k3.1-pro:max:direct:native","scoreDeficit":6.21,"costRatio":2.54,"costComparisonNote":null}],"verdicts":{"budgetId":"moonshot:kimi-k2.6:thinking:direct:native","defaultId":"moonshot:kimi-k3.1-pro:xhigh:direct:native","maxId":"moonshot:kimi-k3.1-pro:max:direct:native"}},{"provider":"alibaba","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"alibaba:qwen3.7-max:thinking:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.5572,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"alibaba:qwen3.8-max-0902:xhigh:direct:native","zStepFromPrev":3.387,"pBetterThanPrev":0.9996,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.8628,"marginalScorePerDollar":23.818,"costMultipleFromPrev":1.4},{"configId":"alibaba:qwen4-max:xhigh:direct:native","zStepFromPrev":0.015,"pBetterThanPrev":0.5061,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.6304,"marginalScorePerDollar":0.241,"costMultipleFromPrev":1.1}],"pruned":[],"verdicts":{"budgetId":"alibaba:qwen3.7-max:thinking:direct:native","defaultId":"alibaba:qwen3.8-max-0902:xhigh:direct:native","maxId":"alibaba:qwen4-max:xhigh:direct:native"}},{"provider":"zhipu","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"zhipu:glm-5.3-flash:max:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.6663,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"zhipu:glm-5.3:max:direct:native","zStepFromPrev":1.649,"pBetterThanPrev":0.9504,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.5467,"marginalScorePerDollar":0.795,"costMultipleFromPrev":8.59}],"pruned":[{"configId":"zhipu:glm-5:thinking:direct:native","dominatedById":"zhipu:glm-5.3-flash:max:direct:native","scoreDeficit":1.43,"costRatio":1.7,"costComparisonNote":null},{"configId":"zhipu:glm-5.1:thinking:direct:native","dominatedById":"zhipu:glm-5.3-flash:max:direct:native","scoreDeficit":5.71,"costRatio":2.25,"costComparisonNote":null},{"configId":"zhipu:glm-5.2:thinking:direct:native","dominatedById":"zhipu:glm-5.3-flash:max:direct:native","scoreDeficit":8.78,"costRatio":2.25,"costComparisonNote":null},{"configId":"zhipu:glm-5.3:low:direct:native","dominatedById":"zhipu:glm-5.3-flash:max:direct:native","scoreDeficit":6.3,"costRatio":2.7,"costComparisonNote":null},{"configId":"zhipu:glm-5.2:max:direct:native","dominatedById":"zhipu:glm-5.3-flash:max:direct:native","scoreDeficit":2.81,"costRatio":5.5,"costComparisonNote":null}],"verdicts":{"budgetId":"zhipu:glm-5.3-flash:max:direct:native","defaultId":null,"maxId":"zhipu:glm-5.3:max:direct:native"}}]},"cyber":{"domainId":"cyber","epochVersion":"cyber-v2.0.0","costUnit":"usd_per_task","activeBenchmarkIds":["cwe_bench_v2","cyberarena_mc_v1","aikido_arena"],"candidateBenchmarkIds":[],"benchmarkVersions":{"cwe_bench_v2":"2.0","cyberarena_mc_v1":"mc-v1","aikido_arena":"1.0"},"coverageFloor":1,"tauResid":0.5521,"regimeMeta":{"native":{"activeBenchmarkIds":["cwe_bench_v2","cyberarena_mc_v1","aikido_arena"],"costSource":"measured","measuredCostBenchmarkIds":["cwe_bench_v2","aikido_arena"],"coverageFloor":1,"tauResid":0.5521,"calibrationTauResidSq":0.30482807,"priorMean":0.288505,"tauModelSq":1.164027,"anchorLabs":9,"withheldBenchmarkIds":[],"epochLink":{"previousEpoch":"cyber-v1.0.0","linked":true,"overlapCount":33,"overlapLabs":9,"slope":0.885339,"intercept":0.071897,"pearsonR":0.999,"reason":"Verified Mean-Sigma link (n=33, labs=9, r=0.999)"}}},"anchorStats":{"cwe_bench_v2":{"benchmarkId":"cwe_bench_v2","mean":56.773148,"sd":10.069873,"anchorCount":17,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":-0.377375,"reliability":0.9329,"discrimination":0.8333,"redundancyFactor":1,"effectiveWeight":0.7774,"taskFamily":"cve_patching","datasetLineageId":"cwe-bench-2.0","introducedAt":"2026-11-02","invertScore":false,"popoviciuSe":3.162278,"boundedMax":100,"sampleSize":250},"cyberarena_mc_v1":{"benchmarkId":"cyberarena_mc_v1","mean":64.92125,"sd":6.041686,"anchorCount":12,"anchorLabs":8,"linkedViaMeanSigma":false,"raschDelta":0.605723,"reliability":0.9545,"discrimination":0.7321,"redundancyFactor":0.8402,"effectiveWeight":0.5871,"taskFamily":"attack_defense_ctf","datasetLineageId":"cyberarena-mc-v1","introducedAt":"2026-10-15","invertScore":false,"popoviciuSe":7.905694,"boundedMax":100,"sampleSize":40},"aikido_arena":{"benchmarkId":"aikido_arena","mean":80.377333,"sd":4.926233,"anchorCount":10,"anchorLabs":6,"linkedViaMeanSigma":false,"raschDelta":-0.622488,"reliability":0.1,"discrimination":0.1353,"redundancyFactor":0.767,"effectiveWeight":0.1,"taskFamily":"cve_detection","datasetLineageId":"aikido-arena-1.0","introducedAt":"2026-10-01","invertScore":false,"popoviciuSe":8.838835,"boundedMax":100,"sampleSize":32}},"configs":[{"id":"anthropic:claude-fable-5.2:high:direct:native","provider":"anthropic","modelSlug":"claude-fable-5.2","displayName":"Claude Fable 5.2","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":2.095593,"shrinkageFactor":0.951646,"score":81.43,"se":6.05,"samplingSe":3.5587,"residualSe":4.8933,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":67.006135,"costFixedFloorUsd":3.925,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1000000,"releaseDate":"2026-11-09","paretoProb":0.887,"benchmarks":[{"benchmarkId":"cwe_bench_v2","benchmarkVersion":"2.0","sourceId":"cwe-bench","sourceRowId":"cwe-bench:v2:claude-fable-5-2","observedHarness":"Claude Code","rawScore":75,"zScore":1.810038,"seModel":"upstream_se","seRaw":2.44898,"ciHalfWidth":4.8,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.85,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://cwe-bench.com/"}]},{"id":"google:gemini-4-neon:high:direct:native","provider":"google","modelSlug":"gemini-4-neon","displayName":"Gemini 4 Neon","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":1.674535,"shrinkageFactor":0.941666,"score":75.12,"se":6.22,"samplingSe":3.9087,"residualSe":4.842,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":55.653503,"costFixedFloorUsd":3.26,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1048576,"releaseDate":"2026-11-18","paretoProb":0.5934,"benchmarks":[{"benchmarkId":"cwe_bench_v2","benchmarkVersion":"2.0","sourceId":"cwe-bench","sourceRowId":"cwe-bench:v2:gemini-4-neon","observedHarness":"Antigravity","rawScore":70.7,"zScore":1.383022,"seModel":"upstream_se","seRaw":2.704082,"ciHalfWidth":5.3,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.52,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://cwe-bench.com/"}]},{"id":"google:gemini-4-hydrogen:high:direct:native","provider":"google","modelSlug":"gemini-4-hydrogen","displayName":"Gemini 4 Hydrogen","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":1.4314,"shrinkageFactor":0.941666,"score":71.47,"se":6.22,"samplingSe":3.9087,"residualSe":4.842,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":22.449189,"costFixedFloorUsd":1.315,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":1.5,"outputPer1mUsd":7.5,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.727,"benchmarks":[{"benchmarkId":"cwe_bench_v2","benchmarkVersion":"2.0","sourceId":"cwe-bench","sourceRowId":"cwe-bench:v2:gemini-4-hydrogen","observedHarness":"Antigravity","rawScore":68.1,"zScore":1.124826,"seModel":"upstream_se","seRaw":2.704082,"ciHalfWidth":5.3,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.63,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://cwe-bench.com/"}]},{"id":"google:gemini-4-carbon:high:direct:native","provider":"google","modelSlug":"gemini-4-carbon","displayName":"Gemini 4 Carbon","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":1.35659,"shrinkageFactor":0.941666,"score":70.35,"se":6.22,"samplingSe":3.9087,"residualSe":4.842,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":36.447923,"costFixedFloorUsd":2.135,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-10-12","paretoProb":0.4692,"benchmarks":[{"benchmarkId":"cwe_bench_v2","benchmarkVersion":"2.0","sourceId":"cwe-bench","sourceRowId":"cwe-bench:v2:gemini-4-carbon","observedHarness":"Antigravity","rawScore":67.3,"zScore":1.045381,"seModel":"upstream_se","seRaw":2.704082,"ciHalfWidth":5.3,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.27,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://cwe-bench.com/"}]},{"id":"anthropic:claude-opus-5.5:high:direct:native","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":1.34244,"shrinkageFactor":0.945751,"score":70.14,"se":6.15,"samplingSe":3.7694,"residualSe":4.863,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":15.791255,"costFixedFloorUsd":0.925,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.7044,"benchmarks":[{"benchmarkId":"cwe_bench_v2","benchmarkVersion":"2.0","sourceId":"cwe-bench","sourceRowId":"cwe-bench:v2:claude-opus-5-5","observedHarness":"Claude Code","rawScore":67.1,"zScore":1.02552,"seModel":"upstream_se","seRaw":2.602041,"ciHalfWidth":5.1,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.85,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://cwe-bench.com/"}]},{"id":"openai:gpt-6.2-astra:high:direct:native","provider":"openai","modelSlug":"gpt-6.2-astra","displayName":"GPT-6.2 Astra","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":1.305272,"shrinkageFactor":0.93746,"score":69.58,"se":6.29,"samplingSe":4.0472,"residualSe":4.8204,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":36.874714,"costFixedFloorUsd":2.16,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.2926,"benchmarks":[{"benchmarkId":"cwe_bench_v2","benchmarkVersion":"2.0","sourceId":"cwe-bench","sourceRowId":"cwe-bench:v2:gpt-6-2-astra","observedHarness":"Codex","rawScore":66.8,"zScore":0.995728,"seModel":"upstream_se","seRaw":2.806122,"ciHalfWidth":5.5,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.32,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://cwe-bench.com/"}]},{"id":"openai:gpt-5.6-sol:medium:direct:native","provider":"openai","modelSlug":"gpt-5.6-sol","displayName":"GPT-5.6 Sol","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":1.263008,"shrinkageFactor":0.942071,"score":68.95,"se":6.72,"samplingSe":3.8951,"residualSe":5.4807,"coverageCount":2,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.529642,"costFixedFloorUsd":7.307019,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.3995,"benchmarks":[{"benchmarkId":"aikido_arena","benchmarkVersion":"1.0","sourceId":"aikido","sourceRowId":"aikido:arena:1run:gpt-5.6-sol","observedHarness":"Aikido Security Harness","rawScore":86.43,"zScore":1.22866,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":14.614038,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.aikido.dev/aikido-arena"},{"benchmarkId":"cyberarena_mc_v1","benchmarkVersion":"mc-v1","sourceId":"cyber-arena","sourceRowId":"cyber-arena:mc-v1:gpt-5.6-sol:codex","observedHarness":"Codex","rawScore":76.03,"zScore":1.838684,"seModel":"upstream_se","seRaw":1.367903,"ciHalfWidth":2.6811,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-21","publishedDate":"2026-11-21","sourceUrl":"https://cyber-arena.ai/"}]},{"id":"meta:muse-wonder:high:direct:native","provider":"meta","modelSlug":"muse-wonder","displayName":"Muse Wonder","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":1.250548,"shrinkageFactor":0.947747,"score":68.76,"se":6.12,"samplingSe":3.6994,"residualSe":4.8733,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":23.302771,"costFixedFloorUsd":1.365,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1048576,"releaseDate":null,"paretoProb":1,"benchmarks":[{"benchmarkId":"cwe_bench_v2","benchmarkVersion":"2.0","sourceId":"cwe-bench","sourceRowId":"cwe-bench:v2:muse-wonder","observedHarness":"opencode","rawScore":66.1,"zScore":0.926213,"seModel":"upstream_se","seRaw":2.55102,"ciHalfWidth":5,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.73,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://cwe-bench.com/"}]},{"id":"openai:gpt-6.2-sol:high:direct:native","provider":"openai","modelSlug":"gpt-6.2-sol","displayName":"GPT-6.2 Sol","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":1.232924,"shrinkageFactor":0.939578,"score":68.49,"se":6.26,"samplingSe":3.9781,"residualSe":4.8313,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":6.914009,"costFixedFloorUsd":0.405,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.6612,"benchmarks":[{"benchmarkId":"cwe_bench_v2","benchmarkVersion":"2.0","sourceId":"cwe-bench","sourceRowId":"cwe-bench:v2:gpt-6-2-sol","observedHarness":"Codex","rawScore":66,"zScore":0.916283,"seModel":"upstream_se","seRaw":2.755102,"ciHalfWidth":5.4,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.81,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://cwe-bench.com/"}]},{"id":"google:gemini-4-argon:high:direct:native","provider":"google","modelSlug":"gemini-4-argon","displayName":"Gemini 4 Argon","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":1.14521,"shrinkageFactor":0.945751,"score":67.18,"se":6.15,"samplingSe":3.7694,"residualSe":4.863,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":31.58251,"costFixedFloorUsd":1.85,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.3974,"benchmarks":[{"benchmarkId":"cwe_bench_v2","benchmarkVersion":"2.0","sourceId":"cwe-bench","sourceRowId":"cwe-bench:v2:gemini-4-argon","observedHarness":"Antigravity","rawScore":65,"zScore":0.816977,"seModel":"upstream_se","seRaw":2.602041,"ciHalfWidth":5.1,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.7,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://cwe-bench.com/"}]},{"id":"openai:gpt-6-sol:medium:direct:native","provider":"openai","modelSlug":"gpt-6-sol","displayName":"GPT-6 Sol","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":1.10924,"shrinkageFactor":0.925694,"score":66.64,"se":6.96,"samplingSe":4.4115,"residualSe":5.3854,"coverageCount":2,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.568259,"costFixedFloorUsd":3.913453,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.6018,"benchmarks":[{"benchmarkId":"aikido_arena","benchmarkVersion":"1.0","sourceId":"aikido","sourceRowId":"aikido:arena:1run:gpt-6-sol","observedHarness":"Aikido Security Harness","rawScore":77.31,"zScore":-0.622653,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.826906,"evalDate":"2026-09-28","publishedDate":"2026-09-28","sourceUrl":"https://www.aikido.dev/aikido-arena"},{"benchmarkId":"cyberarena_mc_v1","benchmarkVersion":"mc-v1","sourceId":"cyber-arena","sourceRowId":"cyber-arena:mc-v1:gpt-6-sol:codex","observedHarness":"Codex","rawScore":76.89,"zScore":1.981028,"seModel":"upstream_se","seRaw":1.489494,"ciHalfWidth":2.9194,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-21","publishedDate":"2026-11-21","sourceUrl":"https://cyber-arena.ai/"}]},{"id":"openai:gpt-6-astra:high:direct:native","provider":"openai","modelSlug":"gpt-6-astra","displayName":"GPT-6 Astra","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":1.064974,"shrinkageFactor":0.939578,"score":65.97,"se":6.26,"samplingSe":3.9781,"residualSe":4.8313,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":33.033598,"costFixedFloorUsd":1.935,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.1578,"benchmarks":[{"benchmarkId":"cwe_bench_v2","benchmarkVersion":"2.0","sourceId":"cwe-bench","sourceRowId":"cwe-bench:v2:gpt-6-astra","observedHarness":"Codex","rawScore":64.2,"zScore":0.737532,"seModel":"upstream_se","seRaw":2.755102,"ciHalfWidth":5.4,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.87,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://cwe-bench.com/"}]},{"id":"anthropic:claude-opus-5:high:direct:native","provider":"anthropic","modelSlug":"claude-opus-5","displayName":"Claude Opus 5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":0.988322,"shrinkageFactor":0.938948,"score":64.82,"se":6.77,"samplingSe":3.9988,"residualSe":5.4625,"coverageCount":2,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":10.731282,"costFixedFloorUsd":9.193081,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-07-28","paretoProb":0.449,"benchmarks":[{"benchmarkId":"aikido_arena","benchmarkVersion":"1.0","sourceId":"aikido","sourceRowId":"aikido:arena:1run:claude-opus-5","observedHarness":"Aikido Security Harness","rawScore":84.16,"zScore":0.767862,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":18.386161,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.aikido.dev/aikido-arena"},{"benchmarkId":"cyberarena_mc_v1","benchmarkVersion":"mc-v1","sourceId":"cyber-arena","sourceRowId":"cyber-arena:mc-v1:claude-opus-5:claude","observedHarness":"Claude Code","rawScore":74.46,"zScore":1.578823,"seModel":"upstream_se","seRaw":1.367903,"ciHalfWidth":2.6811,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-21","publishedDate":"2026-11-21","sourceUrl":"https://cyber-arena.ai/"}]},{"id":"anthropic:claude-sonnet-5.5:high:direct:native","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":0.804862,"shrinkageFactor":0.941666,"score":62.07,"se":6.22,"samplingSe":3.9087,"residualSe":4.842,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.950772,"costFixedFloorUsd":0.29,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.862,"benchmarks":[{"benchmarkId":"cwe_bench_v2","benchmarkVersion":"2.0","sourceId":"cwe-bench","sourceRowId":"cwe-bench:v2:claude-sonnet-5-5","observedHarness":"Claude Code","rawScore":61.4,"zScore":0.459475,"seModel":"upstream_se","seRaw":2.704082,"ciHalfWidth":5.3,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.58,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://cwe-bench.com/"}]},{"id":"openai:gpt-6.1-terra:high:direct:native","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":0.671344,"shrinkageFactor":0.935314,"score":60.07,"se":6.33,"samplingSe":4.116,"residualSe":4.8093,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.755758,"costFixedFloorUsd":0.22,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.6742,"benchmarks":[{"benchmarkId":"cwe_bench_v2","benchmarkVersion":"2.0","sourceId":"cwe-bench","sourceRowId":"cwe-bench:v2:gpt-6-1-terra","observedHarness":"Codex","rawScore":60,"zScore":0.320446,"seModel":"upstream_se","seRaw":2.857143,"ciHalfWidth":5.6,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.44,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://cwe-bench.com/"}]},{"id":"deepseek:deepseek-v4.1-flash-1017:high:direct:native","provider":"deepseek","modelSlug":"deepseek-v4.1-flash-1017","displayName":"DeepSeek V4.1 Flash (1017)","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":0.422901,"shrinkageFactor":0.934529,"score":56.34,"se":5.87,"samplingSe":4.1409,"residualSe":4.1604,"coverageCount":2,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.368071,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.525,"inputPer1mUsd":0.3,"outputPer1mUsd":1.2,"contextWindow":1048576,"releaseDate":"2026-10-17","paretoProb":1,"benchmarks":[{"benchmarkId":"aikido_arena","benchmarkVersion":"1.0","sourceId":"aikido","sourceRowId":"aikido:arena:1run:deepseek/deepseek-v4.1-flash-1017","observedHarness":"Aikido Security Harness","rawScore":78.4,"zScore":-0.401388,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.696143,"evalDate":"2026-10-27","publishedDate":"2026-10-27","sourceUrl":"https://www.aikido.dev/aikido-arena"},{"benchmarkId":"cwe_bench_v2","benchmarkVersion":"2.0","sourceId":"cwe-bench","sourceRowId":"cwe-bench:v2:deepseek-v4-1-flash","observedHarness":"opencode","rawScore":57.6,"zScore":0.082111,"seModel":"upstream_se","seRaw":2.602041,"ciHalfWidth":5.1,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.04,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://cwe-bench.com/"}]},{"id":"spacexai:grok-4.7:high:direct:native","provider":"spacexai","modelSlug":"grok-4.7","displayName":"Grok 4.7","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":0.204633,"shrinkageFactor":0.933606,"score":53.07,"se":5.89,"samplingSe":4.17,"residualSe":4.1563,"coverageCount":2,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":9.680879,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-10-18","paretoProb":0.878,"benchmarks":[{"benchmarkId":"aikido_arena","benchmarkVersion":"1.0","sourceId":"aikido","sourceRowId":"aikido:arena:1run:grok-4.7","observedHarness":"Aikido Security Harness","rawScore":79.33,"zScore":-0.212603,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":16.391758,"evalDate":"2026-09-28","publishedDate":"2026-09-28","sourceUrl":"https://www.aikido.dev/aikido-arena"},{"benchmarkId":"cwe_bench_v2","benchmarkVersion":"2.0","sourceId":"cwe-bench","sourceRowId":"cwe-bench:v2:grok-4-7","observedHarness":"opencode","rawScore":54.7,"zScore":-0.205876,"seModel":"upstream_se","seRaw":2.653061,"ciHalfWidth":5.2,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.97,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://cwe-bench.com/"}]},{"id":"zhipu:glm-5.3:max:direct:native","provider":"zhipu","modelSlug":"glm-5.3","displayName":"GLM-5.3","effort":"max","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":-0.352155,"shrinkageFactor":0.938365,"score":44.72,"se":6.78,"samplingSe":4.0178,"residualSe":5.4591,"coverageCount":2,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.986985,"costFixedFloorUsd":2.558836,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":1048576,"releaseDate":"2026-09-18","paretoProb":0.9193,"benchmarks":[{"benchmarkId":"aikido_arena","benchmarkVersion":"1.0","sourceId":"aikido","sourceRowId":"aikido:arena:1run:zai/glm-5.3","observedHarness":"Aikido Security Harness","rawScore":76.8,"zScore":-0.72618,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.117673,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.aikido.dev/aikido-arena"},{"benchmarkId":"cyberarena_mc_v1","benchmarkVersion":"mc-v1","sourceId":"cyber-arena","sourceRowId":"cyber-arena:mc-v1:glm-5.3:opencode","observedHarness":"OpenCode","rawScore":65.9,"zScore":0.161999,"seModel":"upstream_se","seRaw":1.155118,"ciHalfWidth":2.264,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-21","publishedDate":"2026-11-21","sourceUrl":"https://cyber-arena.ai/"}]},{"id":"google:gemini-3.7-flash:high:direct:native","provider":"google","modelSlug":"gemini-3.7-flash","displayName":"Gemini 3.7 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":-0.445056,"shrinkageFactor":0.971089,"score":43.32,"se":7.02,"samplingSe":2.7517,"residualSe":6.4608,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.49288,"costLogSd":0.687,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.5,"inputPer1mUsd":0.75,"outputPer1mUsd":3.75,"contextWindow":1048576,"releaseDate":"2026-06-18","paretoProb":0.9061,"benchmarks":[{"benchmarkId":"cyberarena_mc_v1","benchmarkVersion":"mc-v1","sourceId":"cyber-arena","sourceRowId":"cyber-arena:mc-v1:gemini-3.7-flash:gemini","observedHarness":"gemini","rawScore":65.76,"zScore":0.138827,"seModel":"upstream_se","seRaw":1.12472,"ciHalfWidth":2.2045,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-21","publishedDate":"2026-11-21","sourceUrl":"https://cyber-arena.ai/"}]},{"id":"spacexai:grok-4.6:high:direct:native","provider":"spacexai","modelSlug":"grok-4.6","displayName":"Grok 4.6","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":-0.478577,"shrinkageFactor":0.964732,"score":42.82,"se":7.1,"samplingSe":3.0392,"residualSe":6.4185,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":5.974269,"costLogSd":0.6784,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-08-25","paretoProb":0.8743,"benchmarks":[{"benchmarkId":"cyberarena_mc_v1","benchmarkVersion":"mc-v1","sourceId":"cyber-arena","sourceRowId":"cyber-arena:mc-v1:grok-4.6:grok","observedHarness":"grok","rawScore":65.52,"zScore":0.099103,"seModel":"upstream_se","seRaw":1.246312,"ciHalfWidth":2.4428,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-21","publishedDate":"2026-11-21","sourceUrl":"https://cyber-arena.ai/"}]},{"id":"anthropic:claude-opus-4.8:none:direct:native","provider":"anthropic","modelSlug":"claude-opus-4.8","displayName":"Claude Opus 4.8","effort":"none","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":-0.487599,"shrinkageFactor":0.974051,"score":42.69,"se":6.99,"samplingSe":2.607,"residualSe":6.4805,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":18.512271,"costLogSd":0.73,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-06-12","paretoProb":0.0755,"benchmarks":[{"benchmarkId":"cyberarena_mc_v1","benchmarkVersion":"mc-v1","sourceId":"cyber-arena","sourceRowId":"cyber-arena:mc-v1:claude-opus-4-8:claude","observedHarness":"Claude Code","rawScore":65.51,"zScore":0.097448,"seModel":"upstream_se","seRaw":1.063924,"ciHalfWidth":2.0853,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-21","publishedDate":"2026-11-21","sourceUrl":"https://cyber-arena.ai/"}]},{"id":"alibaba:qwen3.8-max-0902:high:direct:native","provider":"alibaba","modelSlug":"qwen3.8-max-0902","displayName":"Qwen3.8 Max (0902)","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":-0.699721,"shrinkageFactor":0.949712,"score":39.5,"se":6.08,"samplingSe":3.6291,"residualSe":4.8834,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":17.583776,"costFixedFloorUsd":1.03,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":1000000,"releaseDate":"2026-09-02","paretoProb":1,"benchmarks":[{"benchmarkId":"cwe_bench_v2","benchmarkVersion":"2.0","sourceId":"cwe-bench","sourceRowId":"cwe-bench:v2:qwen-3-8-max","observedHarness":"opencode","rawScore":45.4,"zScore":-1.129423,"seModel":"upstream_se","seRaw":2.5,"ciHalfWidth":4.9,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.06,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://cwe-bench.com/"}]},{"id":"moonshot:kimi-k3.1-pro:high:direct:native","provider":"moonshot","modelSlug":"kimi-k3.1-pro","displayName":"Kimi K3.1 Pro","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":-0.704991,"shrinkageFactor":0.945751,"score":39.43,"se":6.15,"samplingSe":3.7694,"residualSe":4.863,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.194381,"costFixedFloorUsd":0.48,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-10-10","paretoProb":0.7788,"benchmarks":[{"benchmarkId":"cwe_bench_v2","benchmarkVersion":"2.0","sourceId":"cwe-bench","sourceRowId":"cwe-bench:v2:kimi-k3-1-pro","observedHarness":"opencode","rawScore":45.3,"zScore":-1.139354,"seModel":"upstream_se","seRaw":2.602041,"ciHalfWidth":5.1,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.96,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://cwe-bench.com/"}]},{"id":"moonshot:kimi-k3.1-pro:max:direct:native","provider":"moonshot","modelSlug":"kimi-k3.1-pro","displayName":"Kimi K3.1 Pro","effort":"max","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":-0.844556,"shrinkageFactor":0.911059,"score":37.33,"se":7.75,"samplingSe":4.8264,"residualSe":6.0614,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":9.433314,"costLogSd":1.0939,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-10-10","paretoProb":0.7112,"benchmarks":[{"benchmarkId":"cyberarena_mc_v1","benchmarkVersion":"mc-v1","sourceId":"cyber-arena","sourceRowId":"cyber-arena:mc-v1:kimi-k3.1-pro:kimi","observedHarness":"kimi","rawScore":62.81,"zScore":-0.349447,"seModel":"upstream_se","seRaw":2.036655,"ciHalfWidth":3.9918,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-21","publishedDate":"2026-11-21","sourceUrl":"https://cyber-arena.ai/"}]},{"id":"zhipu:glm-5.3:high:direct:native","provider":"zhipu","modelSlug":"glm-5.3","displayName":"GLM-5.3","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":-1.020383,"shrinkageFactor":0.949712,"score":34.69,"se":6.08,"samplingSe":3.6291,"residualSe":4.8834,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.950772,"costFixedFloorUsd":0.29,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":1048576,"releaseDate":"2026-09-18","paretoProb":0.5593,"benchmarks":[{"benchmarkId":"cwe_bench_v2","benchmarkVersion":"2.0","sourceId":"cwe-bench","sourceRowId":"cwe-bench:v2:glm-5-3","observedHarness":"opencode","rawScore":42,"zScore":-1.467064,"seModel":"upstream_se","seRaw":2.5,"ciHalfWidth":4.9,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.58,"evalDate":null,"publishedDate":"2026-11-18","sourceUrl":"https://cwe-bench.com/"}]},{"id":"google:gemini-3.5-flash:high:direct:native","provider":"google","modelSlug":"gemini-3.5-flash","displayName":"Gemini 3.5 Flash","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":-1.376775,"shrinkageFactor":0.967981,"score":29.35,"se":7.06,"samplingSe":2.8958,"residualSe":6.4401,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.686706,"costLogSd":0.6756,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3.375,"inputPer1mUsd":1.5,"outputPer1mUsd":9,"contextWindow":1048576,"releaseDate":"2026-04-12","paretoProb":0.1855,"benchmarks":[{"benchmarkId":"cyberarena_mc_v1","benchmarkVersion":"mc-v1","sourceId":"cyber-arena","sourceRowId":"cyber-arena:mc-v1:gemini-3.5-flash:gemini","observedHarness":"gemini","rawScore":59.93,"zScore":-0.826135,"seModel":"upstream_se","seRaw":1.185516,"ciHalfWidth":2.3236,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-21","publishedDate":"2026-11-21","sourceUrl":"https://cyber-arena.ai/"}]},{"id":"deepseek:deepseek-v4-pro-0813:high:direct:native","provider":"deepseek","modelSlug":"deepseek-v4-pro-0813","displayName":"DeepSeek V4 Pro (0813)","effort":"high","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":-1.655223,"shrinkageFactor":0.963055,"score":25.17,"se":7.12,"samplingSe":3.1106,"residualSe":6.4073,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.859691,"costLogSd":0.685,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.98,"inputPer1mUsd":1.32,"outputPer1mUsd":3.96,"contextWindow":1048576,"releaseDate":"2026-08-13","paretoProb":0.0014,"benchmarks":[{"benchmarkId":"cyberarena_mc_v1","benchmarkVersion":"mc-v1","sourceId":"cyber-arena","sourceRowId":"cyber-arena:mc-v1:deepseek-v4-pro-0813:deepseek","observedHarness":"deepseek","rawScore":58.13,"zScore":-1.124065,"seModel":"upstream_se","seRaw":1.276709,"ciHalfWidth":2.5024,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-21","publishedDate":"2026-11-21","sourceUrl":"https://cyber-arena.ai/"}]},{"id":"moonshot:kimi-k3:max:direct:native","provider":"moonshot","modelSlug":"kimi-k3","displayName":"Kimi K3","effort":"max","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":-1.873057,"shrinkageFactor":0.966374,"score":21.9,"se":7.08,"samplingSe":2.9676,"residualSe":6.4294,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":15.120273,"costLogSd":0.6763,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1048576,"releaseDate":"2026-08-14","paretoProb":0.1966,"benchmarks":[{"benchmarkId":"cyberarena_mc_v1","benchmarkVersion":"mc-v1","sourceId":"cyber-arena","sourceRowId":"cyber-arena:mc-v1:kimi-k3:kimi","observedHarness":"kimi","rawScore":56.81,"zScore":-1.342547,"seModel":"upstream_se","seRaw":1.215914,"ciHalfWidth":2.3832,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-21","publishedDate":"2026-11-21","sourceUrl":"https://cyber-arena.ai/"}]}],"provisionalConfigs":[{"id":"google:gemini-4-argon:medium:direct:native","provider":"google","modelSlug":"gemini-4-argon","displayName":"Gemini 4 Argon","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":0.993186,"shrinkageFactor":0.425527,"score":64.9,"se":12.91,"samplingSe":12.2661,"residualSe":4.0344,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":10.311911,"costFixedFloorUsd":8.833821,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0,"benchmarks":[{"benchmarkId":"aikido_arena","benchmarkVersion":"1.0","sourceId":"aikido","sourceRowId":"aikido:arena:1run:gemini-4-argon","observedHarness":"Aikido Security Harness","rawScore":86.89,"zScore":1.322038,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":17.667642,"evalDate":"2026-11-12","publishedDate":"2026-11-12","sourceUrl":"https://www.aikido.dev/aikido-arena"}]},{"id":"openai:gpt-5.6-luna:medium:direct:native","provider":"openai","modelSlug":"gpt-5.6-luna","displayName":"GPT-5.6 Luna","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":0.286966,"shrinkageFactor":0.343852,"score":54.3,"se":13.51,"samplingSe":13.1091,"residualSe":3.26,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":0.449846,"costFixedFloorUsd":0.385366,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.45,"inputPer1mUsd":0.2,"outputPer1mUsd":1.2,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0,"benchmarks":[{"benchmarkId":"aikido_arena","benchmarkVersion":"1.0","sourceId":"aikido","sourceRowId":"aikido:arena:1run:gpt-5.6-luna","observedHarness":"Aikido Security Harness","rawScore":78.71,"zScore":-0.33846,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.770731,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.aikido.dev/aikido-arena"}]},{"id":"openai:gpt-6-astra:medium:direct:native","provider":"openai","modelSlug":"gpt-6-astra","displayName":"GPT-6 Astra","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":0.233392,"shrinkageFactor":0.338503,"score":53.5,"se":13.55,"samplingSe":13.1624,"residualSe":3.2093,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":34.644971,"costFixedFloorUsd":29.679026,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0,"benchmarks":[{"benchmarkId":"aikido_arena","benchmarkVersion":"1.0","sourceId":"aikido","sourceRowId":"aikido:arena:1run:gpt-6-astra","observedHarness":"Aikido Security Harness","rawScore":77.93,"zScore":-0.496796,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":59.358052,"evalDate":"2026-09-10","publishedDate":"2026-09-10","sourceUrl":"https://www.aikido.dev/aikido-arena"}]},{"id":"openai:gpt-6-luna:medium:direct:native","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"medium","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":-0.454373,"shrinkageFactor":0.278647,"score":43.18,"se":14,"samplingSe":13.7451,"residualSe":2.6418,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":0.208543,"costFixedFloorUsd":0.17865,"costLogSd":0.9946,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0,"benchmarks":[{"benchmarkId":"aikido_arena","benchmarkVersion":"1.0","sourceId":"aikido","sourceRowId":"aikido:arena:1run:gpt-6-luna","observedHarness":"Aikido Security Harness","rawScore":63.04,"zScore":-3,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.357301,"evalDate":"2026-09-28","publishedDate":"2026-09-28","sourceUrl":"https://www.aikido.dev/aikido-arena"}]},{"id":"alibaba:qwen3.8-max:xhigh:direct:native","provider":"alibaba","modelSlug":"qwen3.8-max","displayName":"Qwen3.8 Max","effort":"xhigh","harness":"direct","harnessRegime":"native","domainId":"cyber","theta":-1.262118,"shrinkageFactor":0.975477,"score":31.07,"se":6.97,"samplingSe":2.5343,"residualSe":6.49,"coverageCount":1,"activeBenchmarkCount":3,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-08-03","paretoProb":0,"benchmarks":[{"benchmarkId":"cyberarena_mc_v1","benchmarkVersion":"mc-v1","sourceId":"cyber-arena","sourceRowId":"cyber-arena:mc-v1:qwen3.8-max:qwen","observedHarness":"qwen","rawScore":60.72,"zScore":-0.695377,"seModel":"upstream_se","seRaw":1.033527,"ciHalfWidth":2.0257,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":"2026-11-21","publishedDate":"2026-11-21","sourceUrl":"https://cyber-arena.ai/"}]}],"frontiers":[{"provider":"anthropic","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"anthropic:claude-sonnet-5.5:high:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.862,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"anthropic:claude-opus-5:high:direct:native","zStepFromPrev":0.29,"pBetterThanPrev":0.6141,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.449,"marginalScorePerDollar":0.476,"costMultipleFromPrev":2.17},{"configId":"anthropic:claude-opus-5.5:high:direct:native","zStepFromPrev":0.564,"pBetterThanPrev":0.7136,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.7044,"marginalScorePerDollar":1.051,"costMultipleFromPrev":1.47},{"configId":"anthropic:claude-fable-5.2:high:direct:native","zStepFromPrev":1.352,"pBetterThanPrev":0.9119,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.887,"marginalScorePerDollar":0.22,"costMultipleFromPrev":4.24}],"pruned":[{"configId":"anthropic:claude-opus-4.8:none:direct:native","dominatedById":"anthropic:claude-opus-5.5:high:direct:native","scoreDeficit":27.45,"costRatio":1.17,"costComparisonNote":null}],"verdicts":{"budgetId":"anthropic:claude-sonnet-5.5:high:direct:native","defaultId":null,"maxId":"anthropic:claude-fable-5.2:high:direct:native"}},{"provider":"openai","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"openai:gpt-6.1-terra:high:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.6742,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"openai:gpt-6-sol:medium:direct:native","zStepFromPrev":0.679,"pBetterThanPrev":0.7513,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.6018,"marginalScorePerDollar":8.086,"costMultipleFromPrev":1.22},{"configId":"openai:gpt-6.2-sol:high:direct:native","zStepFromPrev":0.192,"pBetterThanPrev":0.5761,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.6612,"marginalScorePerDollar":0.789,"costMultipleFromPrev":1.51},{"configId":"openai:gpt-5.6-sol:medium:direct:native","zStepFromPrev":0.049,"pBetterThanPrev":0.5194,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.3995,"marginalScorePerDollar":0.285,"costMultipleFromPrev":1.23},{"configId":"openai:gpt-6.2-astra:high:direct:native","zStepFromPrev":0.066,"pBetterThanPrev":0.5265,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.2926,"marginalScorePerDollar":0.022,"costMultipleFromPrev":4.32}],"pruned":[{"configId":"openai:gpt-6-astra:high:direct:native","dominatedById":"openai:gpt-5.6-sol:medium:direct:native","scoreDeficit":2.98,"costRatio":3.87,"costComparisonNote":null}],"verdicts":{"budgetId":"openai:gpt-6.1-terra:high:direct:native","defaultId":null,"maxId":"openai:gpt-6.2-astra:high:direct:native"}},{"provider":"google","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"google:gemini-3.7-flash:high:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9061,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"google:gemini-4-hydrogen:high:direct:native","zStepFromPrev":2.902,"pBetterThanPrev":0.9981,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.727,"marginalScorePerDollar":1.485,"costMultipleFromPrev":6.43},{"configId":"google:gemini-4-neon:high:direct:native","zStepFromPrev":0.428,"pBetterThanPrev":0.6656,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.5934,"marginalScorePerDollar":0.11,"costMultipleFromPrev":2.48}],"pruned":[{"configId":"google:gemini-3.5-flash:high:direct:native","dominatedById":"google:gemini-3.7-flash:high:direct:native","scoreDeficit":13.97,"costRatio":2.49,"costComparisonNote":null},{"configId":"google:gemini-4-argon:high:direct:native","dominatedById":"google:gemini-4-hydrogen:high:direct:native","scoreDeficit":4.29,"costRatio":1.41,"costComparisonNote":null},{"configId":"google:gemini-4-carbon:high:direct:native","dominatedById":"google:gemini-4-hydrogen:high:direct:native","scoreDeficit":1.12,"costRatio":1.62,"costComparisonNote":null}],"verdicts":{"budgetId":"google:gemini-3.7-flash:high:direct:native","defaultId":"google:gemini-4-hydrogen:high:direct:native","maxId":"google:gemini-4-neon:high:direct:native"}},{"provider":"spacexai","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"spacexai:grok-4.6:high:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.8743,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"spacexai:grok-4.7:high:direct:native","zStepFromPrev":1.078,"pBetterThanPrev":0.8595,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.878,"marginalScorePerDollar":2.765,"costMultipleFromPrev":1.62}],"pruned":[],"verdicts":{"budgetId":"spacexai:grok-4.6:high:direct:native","defaultId":null,"maxId":"spacexai:grok-4.7:high:direct:native"}},{"provider":"meta","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"meta:muse-wonder:high:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null}],"pruned":[],"verdicts":{"budgetId":"meta:muse-wonder:high:direct:native","defaultId":null,"maxId":"meta:muse-wonder:high:direct:native"}},{"provider":"deepseek","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"deepseek:deepseek-v4.1-flash-1017:high:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null}],"pruned":[{"configId":"deepseek:deepseek-v4-pro-0813:high:direct:native","dominatedById":"deepseek:deepseek-v4.1-flash-1017:high:direct:native","scoreDeficit":31.17,"costRatio":10.49,"costComparisonNote":null}],"verdicts":{"budgetId":"deepseek:deepseek-v4.1-flash-1017:high:direct:native","defaultId":null,"maxId":"deepseek:deepseek-v4.1-flash-1017:high:direct:native"}},{"provider":"moonshot","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"moonshot:kimi-k3.1-pro:high:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.7788,"marginalScorePerDollar":null,"costMultipleFromPrev":null}],"pruned":[{"configId":"moonshot:kimi-k3.1-pro:max:direct:native","dominatedById":"moonshot:kimi-k3.1-pro:high:direct:native","scoreDeficit":2.1,"costRatio":1.15,"costComparisonNote":null},{"configId":"moonshot:kimi-k3:max:direct:native","dominatedById":"moonshot:kimi-k3.1-pro:high:direct:native","scoreDeficit":17.53,"costRatio":1.85,"costComparisonNote":null}],"verdicts":{"budgetId":"moonshot:kimi-k3.1-pro:high:direct:native","defaultId":null,"maxId":"moonshot:kimi-k3.1-pro:high:direct:native"}},{"provider":"alibaba","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"alibaba:qwen3.8-max-0902:high:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null}],"pruned":[],"verdicts":{"budgetId":"alibaba:qwen3.8-max-0902:high:direct:native","defaultId":null,"maxId":"alibaba:qwen3.8-max-0902:high:direct:native"}},{"provider":"zhipu","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"zhipu:glm-5.3:max:direct:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9193,"marginalScorePerDollar":null,"costMultipleFromPrev":null}],"pruned":[{"configId":"zhipu:glm-5.3:high:direct:native","dominatedById":"zhipu:glm-5.3:max:direct:native","scoreDeficit":10.03,"costRatio":1.66,"costComparisonNote":null}],"verdicts":{"budgetId":"zhipu:glm-5.3:max:direct:native","defaultId":null,"maxId":"zhipu:glm-5.3:max:direct:native"}}]},"harnesses":{"domainId":"harnesses","epochVersion":"harnesses-v2.0.0","costUnit":"usd_per_task","activeBenchmarkIds":["tbench_4_1","vulcanbench_swe_v4","aa_agent_frontiercode","aa_agent_swe_atlas"],"candidateBenchmarkIds":[],"benchmarkVersions":{"tbench_4_1":"4.1","vulcanbench_swe_v4":"4.0","aa_agent_frontiercode":"2.0","aa_agent_swe_atlas":"1.0"},"coverageFloor":1,"tauResid":0.6359,"regimeMeta":{"native":{"activeBenchmarkIds":["tbench_4_1","vulcanbench_swe_v4","aa_agent_frontiercode","aa_agent_swe_atlas"],"costSource":"measured","measuredCostBenchmarkIds":["tbench_4_1","vulcanbench_swe_v4"],"coverageFloor":1,"tauResid":0.6359,"calibrationTauResidSq":0.40433903,"priorMean":-0.023968,"tauModelSq":0.655414,"anchorLabs":9,"withheldBenchmarkIds":[],"epochLink":{"previousEpoch":"harnesses-v1.0.0","linked":true,"overlapCount":124,"overlapLabs":9,"slope":0.723458,"intercept":0.298891,"pearsonR":0.957,"reason":"Verified Mean-Sigma link (n=124, labs=9, r=0.957)"}}},"anchorStats":{"tbench_4_1":{"benchmarkId":"tbench_4_1","mean":36.887573,"sd":17.351927,"anchorCount":50,"anchorLabs":9,"linkedViaMeanSigma":false,"raschDelta":0.335919,"reliability":0.8973,"discrimination":0.6411,"redundancyFactor":0.7625,"effectiveWeight":0.4386,"taskFamily":"terminal_agent","datasetLineageId":"terminal-bench-4.1","introducedAt":"2026-09-01","invertScore":false,"popoviciuSe":6.154575,"boundedMax":100,"sampleSize":66},"vulcanbench_swe_v4":{"benchmarkId":"vulcanbench_swe_v4","mean":87.153364,"sd":7.907176,"anchorCount":86,"anchorLabs":5,"linkedViaMeanSigma":false,"raschDelta":-0.236657,"reliability":0.9721,"discrimination":0.7,"redundancyFactor":0.7481,"effectiveWeight":0.5091,"taskFamily":"swe_agent","datasetLineageId":"vulcanbench-swe-4.0","introducedAt":"2026-10-01","invertScore":false,"popoviciuSe":10.425721,"boundedMax":100,"sampleSize":23},"aa_agent_frontiercode":{"benchmarkId":"aa_agent_frontiercode","mean":29.295278,"sd":8.443617,"anchorCount":29,"anchorLabs":6,"linkedViaMeanSigma":false,"raschDelta":0.161934,"reliability":0.7867,"discrimination":0.7,"redundancyFactor":0.5781,"effectiveWeight":0.3184,"taskFamily":"coding_agent_eval","datasetLineageId":"aa-agent-frontiercode-2.0","introducedAt":"2026-10-15","invertScore":false,"popoviciuSe":4.351941,"boundedMax":100,"sampleSize":132},"aa_agent_swe_atlas":{"benchmarkId":"aa_agent_swe_atlas","mean":57.55977,"sd":6.475508,"anchorCount":29,"anchorLabs":6,"linkedViaMeanSigma":false,"raschDelta":-0.784119,"reliability":0.5379,"discrimination":0.1678,"redundancyFactor":0.433,"effectiveWeight":0.1,"taskFamily":"swe_agent","datasetLineageId":"aa-agent-swe-atlas-1.0","introducedAt":"2026-10-15","invertScore":false,"popoviciuSe":4.490133,"boundedMax":100,"sampleSize":124}},"configs":[{"id":"anthropic:claude-fable-5.2:high:claude-code:native","provider":"anthropic","modelSlug":"claude-fable-5.2","displayName":"Claude Fable 5.2","effort":"high","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":1.332165,"shrinkageFactor":0.945917,"score":69.98,"se":4.47,"samplingSe":2.8241,"residualSe":3.4649,"coverageCount":3,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":5.610851,"costFixedFloorUsd":1.428206,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1000000,"releaseDate":"2026-11-09","paretoProb":0.6613,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:37d7db5622db33bcc5c6824d564246e7","observedHarness":"Claude Code","rawScore":48.74,"zScore":2.30289,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:37d7db5622db33bcc5c6824d564246e7","observedHarness":"Claude Code","rawScore":61.83,"zScore":0.659443,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Fable 5.2:Claude Code:high","observedHarness":"Claude Code","rawScore":92.76,"zScore":0.709057,"seModel":"upstream_se","seRaw":0.4336,"ciHalfWidth":0.8499,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.856412,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-fable-5.2:max:claude-code:native","provider":"anthropic","modelSlug":"claude-fable-5.2","displayName":"Claude Fable 5.2","effort":"max","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":1.222972,"shrinkageFactor":0.957952,"score":68.34,"se":2.49,"samplingSe":2.4901,"residualSe":0,"coverageCount":4,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":30.428847,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1000000,"releaseDate":"2026-11-09","paretoProb":0.1211,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:78442b1eea1f0b696ab08b6d12ec9354","observedHarness":"Claude Code","rawScore":43.6,"zScore":1.694146,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:78442b1eea1f0b696ab08b6d12ec9354","observedHarness":"Claude Code","rawScore":60.48,"zScore":0.450965,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Fable 5.2:max:Claude Code","observedHarness":"Claude Code","rawScore":65.76,"zScore":1.663932,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":49.978061,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Fable 5.2:Claude Code:max","observedHarness":"Claude Code","rawScore":93.85,"zScore":0.846906,"seModel":"upstream_se","seRaw":0.418,"ciHalfWidth":0.8193,"ciLower":null,"ciUpper":null,"measuredCostUsd":10.879634,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-opus-5.5:max:claude-code:native","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"max","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":1.124066,"shrinkageFactor":0.955405,"score":66.86,"se":2.56,"samplingSe":2.5644,"residualSe":0,"coverageCount":4,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":11.523477,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.1068,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:e594775f1aa1026d8fa9385fa35f68b4","observedHarness":"Claude Code","rawScore":43.97,"zScore":1.737966,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:e594775f1aa1026d8fa9385fa35f68b4","observedHarness":"Claude Code","rawScore":65.86,"zScore":1.281788,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Opus 5.5:max:Claude Code","observedHarness":"Claude Code","rawScore":65.76,"zScore":1.663932,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":14.298939,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Opus 5.5:Claude Code:max","observedHarness":"Claude Code","rawScore":90.22,"zScore":0.387829,"seModel":"upstream_se","seRaw":1.0241,"ciHalfWidth":2.0072,"ciLower":null,"ciUpper":null,"measuredCostUsd":8.748014,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"google:gemini-4-carbon:high:antigravity:native","provider":"google","modelSlug":"gemini-4-carbon","displayName":"Gemini 4 Carbon","effort":"high","harness":"antigravity","harnessRegime":"native","domainId":"harnesses","theta":1.054254,"shrinkageFactor":0.902374,"score":65.81,"se":5.38,"samplingSe":3.7943,"residualSe":3.8136,"coverageCount":3,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.04633,"costFixedFloorUsd":2.057955,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-10-12","paretoProb":0.6061,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:7814cb87040ea234c69eca79cd85486d","observedHarness":"Antigravity CLI","rawScore":38.04,"zScore":1.035661,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:7814cb87040ea234c69eca79cd85486d","observedHarness":"Antigravity CLI","rawScore":63.71,"zScore":0.949768,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Gemini 4 Carbon:high:Antigravity CLI","observedHarness":"Antigravity CLI","rawScore":64.55,"zScore":1.594199,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.115909,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"anthropic:claude-sonnet-5.5:max:claude-code:native","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"max","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":1.030238,"shrinkageFactor":0.958477,"score":65.45,"se":2.47,"samplingSe":2.4745,"residualSe":0,"coverageCount":4,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":13.871391,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.0333,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:4d278d53a18fbdcb5138d11466ea0b1b","observedHarness":"Claude Code","rawScore":39.09,"zScore":1.160015,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:4d278d53a18fbdcb5138d11466ea0b1b","observedHarness":"Claude Code","rawScore":64.25,"zScore":1.033159,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Sonnet 5.5:max:Claude Code","observedHarness":"Claude Code","rawScore":63.94,"zScore":1.559045,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":22.223152,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Sonnet 5.5:Claude Code:max","observedHarness":"Claude Code","rawScore":92.02,"zScore":0.615471,"seModel":"upstream_se","seRaw":0.3546,"ciHalfWidth":0.695,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.519631,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-fable-5.2:xhigh:claude-code:native","provider":"anthropic","modelSlug":"claude-fable-5.2","displayName":"Claude Fable 5.2","effort":"xhigh","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":1.002652,"shrinkageFactor":0.995124,"score":65.04,"se":7.13,"samplingSe":0.848,"residualSe":7.0794,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":9.8009,"costFixedFloorUsd":2.494756,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1000000,"releaseDate":"2026-11-09","paretoProb":0.2028,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Fable 5.2:Claude Code:xhigh","observedHarness":"Claude Code","rawScore":93.25,"zScore":0.771026,"seModel":"upstream_se","seRaw":0.4481,"ciHalfWidth":0.8783,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.989512,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6.2-astra:max:codex:native","provider":"openai","modelSlug":"gpt-6.2-astra","displayName":"GPT-6.2 Astra","effort":"max","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.94318,"shrinkageFactor":0.846737,"score":64.15,"se":8.1,"samplingSe":4.7541,"residualSe":6.5588,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":3.160468,"costFixedFloorUsd":2.13506,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.3192,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:GPT-6.2 Astra:max:Codex","observedHarness":"Codex","rawScore":62.12,"zScore":1.454157,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.270121,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"openai:gpt-6.2-astra:xhigh:codex:native","provider":"openai","modelSlug":"gpt-6.2-astra","displayName":"GPT-6.2 Astra","effort":"xhigh","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.939768,"shrinkageFactor":0.785877,"score":64.1,"se":7.9,"samplingSe":5.6193,"residualSe":5.5497,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.44279,"costLogSd":0.6227,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.3064,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:d2d75a8abf0ca8d8888f99cff7bee543","observedHarness":"Codex","rawScore":43.48,"zScore":1.679934,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:d2d75a8abf0ca8d8888f99cff7bee543","observedHarness":"Codex","rawScore":53.76,"zScore":-0.586791,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"google:gemini-4-neon:xhigh:antigravity:native","provider":"google","modelSlug":"gemini-4-neon","displayName":"Gemini 4 Neon","effort":"xhigh","harness":"antigravity","harnessRegime":"native","domainId":"harnesses","theta":0.89754,"shrinkageFactor":0.845604,"score":63.46,"se":8.1,"samplingSe":4.7716,"residualSe":6.5501,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.311267,"costFixedFloorUsd":2.912485,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1048576,"releaseDate":"2026-11-18","paretoProb":0.319,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Gemini 4 Neon:xhigh:Antigravity CLI","observedHarness":"Antigravity CLI","rawScore":61.21,"zScore":1.401713,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.82497,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"openai:gpt-6-astra:max:codex:native","provider":"openai","modelSlug":"gpt-6-astra","displayName":"GPT-6 Astra","effort":"max","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.878531,"shrinkageFactor":0.957151,"score":63.18,"se":2.51,"samplingSe":2.5137,"residualSe":0,"coverageCount":4,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.234336,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0413,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:97f24b3f300fe06034101d053f3a894b","observedHarness":"Codex","rawScore":42.58,"zScore":1.573345,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:97f24b3f300fe06034101d053f3a894b","observedHarness":"Codex","rawScore":59.14,"zScore":0.244032,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:GPT-6 Astra:max:Codex","observedHarness":"Codex","rawScore":60.3,"zScore":1.34927,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":9.900545,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6 Astra:Codex:max","observedHarness":"Codex","rawScore":89.3,"zScore":0.271479,"seModel":"upstream_se","seRaw":0.3663,"ciHalfWidth":0.7179,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.568128,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6.2-astra:high:codex:native","provider":"openai","modelSlug":"gpt-6.2-astra","displayName":"GPT-6.2 Astra","effort":"high","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.871081,"shrinkageFactor":0.78703,"score":63.07,"se":7.89,"samplingSe":5.6041,"residualSe":5.5578,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":2.6644,"costLogSd":0.6227,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.316,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:f60010d6306ed860deef16152e949af3","observedHarness":"Codex","rawScore":42.16,"zScore":1.523603,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:f60010d6306ed860deef16152e949af3","observedHarness":"Codex","rawScore":54.57,"zScore":-0.461704,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"openai:gpt-6.1-sol:max:codex:native","provider":"openai","modelSlug":"gpt-6.1-sol","displayName":"GPT-6.1 Sol","effort":"max","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.853049,"shrinkageFactor":0.957351,"score":62.8,"se":2.51,"samplingSe":2.5079,"residualSe":0,"coverageCount":4,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.190139,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.3467,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:5146f276cbceb4e9fdeb5834609f805b","observedHarness":"Codex","rawScore":42.49,"zScore":1.562686,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:5146f276cbceb4e9fdeb5834609f805b","observedHarness":"Codex","rawScore":59.95,"zScore":0.369119,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:GPT-6.1 Sol:max:Codex","observedHarness":"Codex","rawScore":60.91,"zScore":1.384424,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.921939,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6.1 Sol:Codex:max","observedHarness":"Codex","rawScore":88.35,"zScore":0.151335,"seModel":"upstream_se","seRaw":0.331,"ciHalfWidth":0.6488,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.458339,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-fable-5.2:medium:claude-code:native","provider":"anthropic","modelSlug":"claude-fable-5.2","displayName":"Claude Fable 5.2","effort":"medium","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.834563,"shrinkageFactor":0.949401,"score":62.52,"se":4.42,"samplingSe":2.7316,"residualSe":3.4776,"coverageCount":3,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":3.374438,"costFixedFloorUsd":0.858942,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1000000,"releaseDate":"2026-11-09","paretoProb":0.386,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:b02d0ed1020688cc426aebb910ee1cd8","observedHarness":"Claude Code","rawScore":35.82,"zScore":0.77274,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:b02d0ed1020688cc426aebb910ee1cd8","observedHarness":"Claude Code","rawScore":63.71,"zScore":0.949768,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Fable 5.2:Claude Code:medium","observedHarness":"Claude Code","rawScore":92.25,"zScore":0.644558,"seModel":"upstream_se","seRaw":0.4392,"ciHalfWidth":0.8608,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.717883,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-haiku-5.5:xhigh:claude-code:native","provider":"anthropic","modelSlug":"claude-haiku-5.5","displayName":"Claude Haiku 5.5","effort":"xhigh","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.824893,"shrinkageFactor":0.996238,"score":62.37,"se":7.13,"samplingSe":0.7449,"residualSe":7.0874,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.648838,"costFixedFloorUsd":0.419701,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1000000,"releaseDate":"2026-10-07","paretoProb":0.8741,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Haiku 5.5:Claude Code:xhigh","observedHarness":"Claude Code","rawScore":91.83,"zScore":0.591442,"seModel":"upstream_se","seRaw":0.3934,"ciHalfWidth":0.7711,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.839402,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"moonshot:kimi-k3.1-pro:max:claude-code:native","provider":"moonshot","modelSlug":"kimi-k3.1-pro","displayName":"Kimi K3.1 Pro","effort":"max","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.82227,"shrinkageFactor":0.843926,"score":62.33,"se":8.11,"samplingSe":4.7975,"residualSe":6.5371,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.437199,"costFixedFloorUsd":1.646454,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-10-10","paretoProb":1,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Kimi K3.1 Pro:max:Claude Code","observedHarness":"Claude Code","rawScore":59.7,"zScore":1.314691,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.292909,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"anthropic:claude-fable-5.1:max:claude-code:native","provider":"anthropic","modelSlug":"claude-fable-5.1","displayName":"Claude Fable 5.1","effort":"max","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.765864,"shrinkageFactor":0.958648,"score":61.49,"se":2.47,"samplingSe":2.4694,"residualSe":0,"coverageCount":4,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":13.98923,"costImputed":false,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-09-12","paretoProb":0.001,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:98e53b1f02eee089dc8221078e300b3e","observedHarness":"Claude Code","rawScore":32.99,"zScore":0.437576,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:98e53b1f02eee089dc8221078e300b3e","observedHarness":"Claude Code","rawScore":66.4,"zScore":1.365179,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Fable 5.1:max:Claude Code","observedHarness":"Claude Code","rawScore":57.27,"zScore":1.174649,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":18.919697,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Fable 5.1:Claude Code:max","observedHarness":"Claude Code","rawScore":91.84,"zScore":0.592707,"seModel":"upstream_se","seRaw":0.4651,"ciHalfWidth":0.9116,"ciLower":null,"ciUpper":null,"measuredCostUsd":9.058763,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-fable-5.1:xhigh:claude-code:native","provider":"anthropic","modelSlug":"claude-fable-5.1","displayName":"Claude Fable 5.1","effort":"xhigh","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.733856,"shrinkageFactor":0.95699,"score":61.01,"se":3.9,"samplingSe":2.5185,"residualSe":2.9836,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":14.624696,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-09-12","paretoProb":0.0072,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Fable 5.1:xhigh:Claude Code","observedHarness":"Claude Code","rawScore":57.58,"zScore":1.192515,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":14.763758,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Fable 5.1:Claude Code:xhigh","observedHarness":"Claude Code","rawScore":90.75,"zScore":0.454857,"seModel":"upstream_se","seRaw":0.821,"ciHalfWidth":1.6092,"ciLower":null,"ciUpper":null,"measuredCostUsd":14.485635,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-opus-5.5:high:claude-code:native","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"high","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.731143,"shrinkageFactor":0.992248,"score":60.97,"se":7.14,"samplingSe":1.0692,"residualSe":7.059,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.42217,"costFixedFloorUsd":1.634722,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.1403,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Opus 5.5:Claude Code:high","observedHarness":"Claude Code","rawScore":91.11,"zScore":0.500385,"seModel":"upstream_se","seRaw":0.5658,"ciHalfWidth":1.109,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.269444,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6.2-sol:max:codex:native","provider":"openai","modelSlug":"gpt-6.2-sol","displayName":"GPT-6.2 Sol","effort":"max","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.70766,"shrinkageFactor":0.957285,"score":60.61,"se":2.51,"samplingSe":2.5098,"residualSe":0,"coverageCount":4,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.803682,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.3789,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:eb55a750b7784b3f03c16efc3c39e1ff","observedHarness":"Codex","rawScore":41.53,"zScore":1.448991,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:eb55a750b7784b3f03c16efc3c39e1ff","observedHarness":"Codex","rawScore":46.24,"zScore":-1.74809,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:GPT-6.2 Sol:max:Codex","observedHarness":"Codex","rawScore":61.52,"zScore":1.419579,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.058061,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6.2 Sol:Codex:max","observedHarness":"Codex","rawScore":88.74,"zScore":0.200658,"seModel":"upstream_se","seRaw":0.4027,"ciHalfWidth":0.7893,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.549303,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6.1-terra:max:codex:native","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"max","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.699956,"shrinkageFactor":0.956474,"score":60.5,"se":2.53,"samplingSe":2.5335,"residualSe":0,"coverageCount":4,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.50309,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.0757,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:1fece3eef716cd9668b972cced54606c","observedHarness":"Codex","rawScore":41.41,"zScore":1.434779,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:1fece3eef716cd9668b972cced54606c","observedHarness":"Codex","rawScore":54.03,"zScore":-0.545095,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:GPT-6.1 Terra:max:Codex","observedHarness":"Codex","rawScore":53.33,"zScore":0.947585,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.649818,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6.1 Terra:Codex:max","observedHarness":"Codex","rawScore":90,"zScore":0.360007,"seModel":"upstream_se","seRaw":0.3927,"ciHalfWidth":0.7697,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.356363,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-opus-5.5:medium:claude-code:native","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"medium","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.698171,"shrinkageFactor":0.990054,"score":60.47,"se":7.15,"samplingSe":1.2111,"residualSe":7.0434,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":5.582936,"costFixedFloorUsd":1.421101,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.1523,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Opus 5.5:Claude Code:medium","observedHarness":"Claude Code","rawScore":90.86,"zScore":0.468769,"seModel":"upstream_se","seRaw":0.6416,"ciHalfWidth":1.2575,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.842201,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6-astra:xhigh:codex:native","provider":"openai","modelSlug":"gpt-6-astra","displayName":"GPT-6 Astra","effort":"xhigh","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.687801,"shrinkageFactor":0.961323,"score":60.32,"se":3.83,"samplingSe":2.3882,"residualSe":2.9971,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.71239,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0309,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:GPT-6 Astra:xhigh:Codex","observedHarness":"Codex","rawScore":59.7,"zScore":1.314691,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.122758,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6 Astra:Codex:xhigh","observedHarness":"Codex","rawScore":89.16,"zScore":0.253774,"seModel":"upstream_se","seRaw":0.2971,"ciHalfWidth":0.5823,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.302022,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6.2-astra:medium:codex:native","provider":"openai","modelSlug":"gpt-6.2-astra","displayName":"GPT-6.2 Astra","effort":"medium","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.687185,"shrinkageFactor":0.789325,"score":60.31,"se":7.88,"samplingSe":5.5739,"residualSe":5.574,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.884172,"costLogSd":0.6227,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.2839,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:eae5a29e0ed56a9c2aa5b6e82dd082e2","observedHarness":"Codex","rawScore":39.87,"zScore":1.252392,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:eae5a29e0ed56a9c2aa5b6e82dd082e2","observedHarness":"Codex","rawScore":53.76,"zScore":-0.586791,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"anthropic:claude-opus-5.5:xhigh:claude-code:native","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"xhigh","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.662509,"shrinkageFactor":0.971481,"score":59.94,"se":7.21,"samplingSe":2.0508,"residualSe":6.9112,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.076333,"costFixedFloorUsd":2.055779,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.0775,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Opus 5.5:Claude Code:xhigh","observedHarness":"Claude Code","rawScore":90.68,"zScore":0.446004,"seModel":"upstream_se","seRaw":1.0968,"ciHalfWidth":2.1497,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.111557,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"google:gemini-4-hydrogen:high:antigravity:native","provider":"google","modelSlug":"gemini-4-hydrogen","displayName":"Gemini 4 Hydrogen","effort":"high","harness":"antigravity","harnessRegime":"native","domainId":"harnesses","theta":0.657528,"shrinkageFactor":0.841089,"score":59.86,"se":8.12,"samplingSe":4.8409,"residualSe":6.5151,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.758921,"costFixedFloorUsd":1.188243,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":1.5,"outputPer1mUsd":7.5,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.8162,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Gemini 4 Hydrogen:high:Antigravity CLI","observedHarness":"Antigravity CLI","rawScore":56.36,"zScore":1.122205,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.376485,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"google:gemini-4-hydrogen:xhigh:antigravity:native","provider":"google","modelSlug":"gemini-4-hydrogen","displayName":"Gemini 4 Hydrogen","effort":"xhigh","harness":"antigravity","harnessRegime":"native","domainId":"harnesses","theta":0.64943,"shrinkageFactor":0.794676,"score":59.74,"se":7.86,"samplingSe":5.5026,"residualSe":5.6118,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.701811,"costLogSd":0.6227,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":1.5,"outputPer1mUsd":7.5,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.5553,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:8e5d9d2dc96b716e4d0b96706efc4548","observedHarness":"Antigravity CLI","rawScore":36.63,"zScore":0.868671,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:8e5d9d2dc96b716e4d0b96706efc4548","observedHarness":"Antigravity CLI","rawScore":60.22,"zScore":0.410814,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"openai:gpt-6-astra:high:codex:native","provider":"openai","modelSlug":"gpt-6-astra","displayName":"GPT-6 Astra","effort":"high","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.629834,"shrinkageFactor":0.96081,"score":59.45,"se":3.84,"samplingSe":2.404,"residualSe":2.9955,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.294048,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0228,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:GPT-6 Astra:high:Codex","observedHarness":"Codex","rawScore":60,"zScore":1.33198,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.87703,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6 Astra:Codex:high","observedHarness":"Codex","rawScore":88.16,"zScore":0.127307,"seModel":"upstream_se","seRaw":0.4248,"ciHalfWidth":0.8326,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.711065,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-fable-5.1:high:claude-code:native","provider":"anthropic","modelSlug":"claude-fable-5.1","displayName":"Claude Fable 5.1","effort":"high","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.619899,"shrinkageFactor":0.956895,"score":59.3,"se":3.91,"samplingSe":2.5212,"residualSe":2.9833,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":10.889242,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-09-12","paretoProb":0.0069,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Fable 5.1:high:Claude Code","observedHarness":"Claude Code","rawScore":54.85,"zScore":1.035183,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":12.076909,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Fable 5.1:Claude Code:high","observedHarness":"Claude Code","rawScore":90.07,"zScore":0.368859,"seModel":"upstream_se","seRaw":0.7822,"ciHalfWidth":1.5331,"ciLower":null,"ciUpper":null,"measuredCostUsd":9.701575,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-sonnet-5.5:xhigh:claude-code:native","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"xhigh","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.616904,"shrinkageFactor":0.992178,"score":59.25,"se":7.14,"samplingSe":1.074,"residualSe":7.0585,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.630217,"costFixedFloorUsd":1.687679,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.1029,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Sonnet 5.5:Claude Code:xhigh","observedHarness":"Claude Code","rawScore":90.2,"zScore":0.3853,"seModel":"upstream_se","seRaw":0.5684,"ciHalfWidth":1.1141,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.375358,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6.2-sol:xhigh:codex:native","provider":"openai","modelSlug":"gpt-6.2-sol","displayName":"GPT-6.2 Sol","effort":"xhigh","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.602251,"shrinkageFactor":0.947025,"score":59.03,"se":4.45,"samplingSe":2.795,"residualSe":3.4689,"coverageCount":3,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.900918,"costFixedFloorUsd":0.229323,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.318,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:2b66aa9f8afa482977259846d9b207ba","observedHarness":"Codex","rawScore":40.44,"zScore":1.319899,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:2b66aa9f8afa482977259846d9b207ba","observedHarness":"Codex","rawScore":51.34,"zScore":-0.960507,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6.2 Sol:Codex:xhigh","observedHarness":"Codex","rawScore":89.01,"zScore":0.234804,"seModel":"upstream_se","seRaw":0.4319,"ciHalfWidth":0.8465,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.458646,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-fable-5.1:medium:claude-code:native","provider":"anthropic","modelSlug":"claude-fable-5.1","displayName":"Claude Fable 5.1","effort":"medium","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.599362,"shrinkageFactor":0.955037,"score":58.99,"se":3.94,"samplingSe":2.575,"residualSe":2.9775,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.885278,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-09-12","paretoProb":0.0104,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Fable 5.1:medium:Claude Code","observedHarness":"Claude Code","rawScore":53.94,"zScore":0.98274,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":8.584545,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Fable 5.1:Claude Code:medium","observedHarness":"Claude Code","rawScore":90.13,"zScore":0.376447,"seModel":"upstream_se","seRaw":0.9394,"ciHalfWidth":1.8412,"ciLower":null,"ciUpper":null,"measuredCostUsd":9.18601,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-fable-5.2:low:claude-code:native","provider":"anthropic","modelSlug":"claude-fable-5.2","displayName":"Claude Fable 5.2","effort":"low","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.565568,"shrinkageFactor":0.952166,"score":58.48,"se":4.38,"samplingSe":2.6559,"residualSe":3.4878,"coverageCount":3,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.386352,"costFixedFloorUsd":0.861974,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":true,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1000000,"releaseDate":"2026-11-09","paretoProb":0.1869,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:09e56f8cd938c59b1a9a600141535ab0","observedHarness":"Claude Code","rawScore":30.91,"zScore":0.191236,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:09e56f8cd938c59b1a9a600141535ab0","observedHarness":"Claude Code","rawScore":59.95,"zScore":0.369119,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"fallbackRouting":true,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Fable 5.2:Claude Code:low","observedHarness":"Claude Code","rawScore":91.92,"zScore":0.602824,"seModel":"upstream_se","seRaw":0.3929,"ciHalfWidth":0.7701,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.723948,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-opus-5:xhigh:claude-code:native","provider":"anthropic","modelSlug":"claude-opus-5","displayName":"Claude Opus 5","effort":"xhigh","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.509675,"shrinkageFactor":0.839544,"score":57.65,"se":8.12,"samplingSe":4.8644,"residualSe":6.5031,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":13.650386,"costFixedFloorUsd":9.221545,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-07-28","paretoProb":0.0444,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Opus 5:xhigh:Claude Code","observedHarness":"Claude Code","rawScore":53.33,"zScore":0.947585,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":18.443091,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"openai:gpt-6.2-sol:high:codex:native","provider":"openai","modelSlug":"gpt-6.2-sol","displayName":"GPT-6.2 Sol","effort":"high","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.497709,"shrinkageFactor":0.948322,"score":57.47,"se":4.44,"samplingSe":2.7606,"residualSe":3.4737,"coverageCount":3,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.798504,"costFixedFloorUsd":0.203254,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.2996,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:c4f606fc3bab9772d3e2ab6c95ebed5a","observedHarness":"Codex","rawScore":38.29,"zScore":1.065269,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:c4f606fc3bab9772d3e2ab6c95ebed5a","observedHarness":"Codex","rawScore":50.54,"zScore":-1.084049,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6.2 Sol:Codex:high","observedHarness":"Codex","rawScore":88.86,"zScore":0.215834,"seModel":"upstream_se","seRaw":0.3263,"ciHalfWidth":0.6395,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.406508,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6-astra:medium:codex:native","provider":"openai","modelSlug":"gpt-6-astra","displayName":"GPT-6 Astra","effort":"medium","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.477181,"shrinkageFactor":0.960242,"score":57.16,"se":3.85,"samplingSe":2.4214,"residualSe":2.9938,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.641885,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0103,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:GPT-6 Astra:medium:Codex","observedHarness":"Codex","rawScore":55.15,"zScore":1.052473,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.802424,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6 Astra:Codex:medium","observedHarness":"Codex","rawScore":87.73,"zScore":0.072926,"seModel":"upstream_se","seRaw":0.3185,"ciHalfWidth":0.6243,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.481347,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-opus-5:max:claude-code:native","provider":"anthropic","modelSlug":"claude-opus-5","displayName":"Claude Opus 5","effort":"max","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.450935,"shrinkageFactor":0.839198,"score":56.76,"se":8.12,"samplingSe":4.8696,"residualSe":6.5004,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":13.387727,"costFixedFloorUsd":9.044106,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-07-28","paretoProb":0.0332,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Opus 5:max:Claude Code","observedHarness":"Claude Code","rawScore":52.12,"zScore":0.877852,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":18.088212,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"openai:gpt-6.1-sol:xhigh:codex:native","provider":"openai","modelSlug":"gpt-6.1-sol","displayName":"GPT-6.1 Sol","effort":"xhigh","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.440525,"shrinkageFactor":0.996037,"score":56.61,"se":7.13,"samplingSe":0.7645,"residualSe":7.0859,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.853667,"costFixedFloorUsd":0.217296,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.2601,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6.1 Sol:Codex:xhigh","observedHarness":"Codex","rawScore":88.78,"zScore":0.205716,"seModel":"upstream_se","seRaw":0.4038,"ciHalfWidth":0.7914,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.434591,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6.2-sol:medium:codex:native","provider":"openai","modelSlug":"gpt-6.2-sol","displayName":"GPT-6.2 Sol","effort":"medium","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.421659,"shrinkageFactor":0.948446,"score":56.32,"se":4.44,"samplingSe":2.7573,"residualSe":3.4741,"coverageCount":3,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.639095,"costFixedFloorUsd":0.162678,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.387,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:2827f8a25558cdb94dfada79c70b0d8c","observedHarness":"Codex","rawScore":36.29,"zScore":0.828404,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:2827f8a25558cdb94dfada79c70b0d8c","observedHarness":"Codex","rawScore":50.27,"zScore":-1.125745,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6.2 Sol:Codex:medium","observedHarness":"Codex","rawScore":88.94,"zScore":0.225951,"seModel":"upstream_se","seRaw":0.4667,"ciHalfWidth":0.9147,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.325355,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6.1-sol:high:codex:native","provider":"openai","modelSlug":"gpt-6.1-sol","displayName":"GPT-6.1 Sol","effort":"high","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.371267,"shrinkageFactor":0.996095,"score":55.57,"se":7.13,"samplingSe":0.7588,"residualSe":7.0863,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.648357,"costFixedFloorUsd":0.165035,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.355,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6.1 Sol:Codex:high","observedHarness":"Codex","rawScore":88.23,"zScore":0.136159,"seModel":"upstream_se","seRaw":0.4008,"ciHalfWidth":0.7856,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.33007,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6-astra:low:codex:native","provider":"openai","modelSlug":"gpt-6-astra","displayName":"GPT-6 Astra","effort":"low","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.348471,"shrinkageFactor":0.958833,"score":55.23,"se":3.87,"samplingSe":2.4639,"residualSe":2.9894,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.220394,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0041,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:GPT-6 Astra:low:Codex","observedHarness":"Codex","rawScore":50.91,"zScore":0.808119,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.719091,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6 Astra:Codex:low","observedHarness":"Codex","rawScore":87.43,"zScore":0.034985,"seModel":"upstream_se","seRaw":0.5104,"ciHalfWidth":1.0004,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.721698,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-opus-5:high:claude-code:native","provider":"anthropic","modelSlug":"claude-opus-5","displayName":"Claude Opus 5","effort":"high","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.333797,"shrinkageFactor":0.838967,"score":55.01,"se":8.12,"samplingSe":4.8731,"residualSe":6.4987,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":10.456701,"costFixedFloorUsd":7.064045,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-07-28","paretoProb":0.0319,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Opus 5:high:Claude Code","observedHarness":"Claude Code","rawScore":49.7,"zScore":0.738386,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":14.128091,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"anthropic:claude-haiku-5.5:high:claude-code:native","provider":"anthropic","modelSlug":"claude-haiku-5.5","displayName":"Claude Haiku 5.5","effort":"high","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.328895,"shrinkageFactor":0.953102,"score":54.93,"se":7.27,"samplingSe":2.6298,"residualSe":6.7805,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.45027,"costFixedFloorUsd":0.6237,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1000000,"releaseDate":"2026-10-07","paretoProb":0.263,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Haiku 5.5:Claude Code:high","observedHarness":"Claude Code","rawScore":88.02,"zScore":0.109601,"seModel":"upstream_se","seRaw":1.42,"ciHalfWidth":2.7832,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.247401,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6-sol:max:codex:native","provider":"openai","modelSlug":"gpt-6-sol","displayName":"GPT-6 Sol","effort":"max","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.291913,"shrinkageFactor":0.955137,"score":54.38,"se":3.93,"samplingSe":2.5721,"residualSe":2.9778,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.459278,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0018,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:GPT-6 Sol:max:Codex","observedHarness":"Codex","rawScore":50.3,"zScore":0.772965,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.397939,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6 Sol:Codex:max","observedHarness":"Codex","rawScore":86.82,"zScore":-0.04216,"seModel":"upstream_se","seRaw":0.9121,"ciHalfWidth":1.7877,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.520618,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-fable-5.1:low:claude-code:native","provider":"anthropic","modelSlug":"claude-fable-5.1","displayName":"Claude Fable 5.1","effort":"low","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.262829,"shrinkageFactor":0.952582,"score":53.94,"se":3.98,"samplingSe":2.6444,"residualSe":2.9699,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":7.555677,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-09-12","paretoProb":0.0018,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Fable 5.1:low:Claude Code","observedHarness":"Claude Code","rawScore":43.33,"zScore":0.37128,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.147636,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Fable 5.1:Claude Code:low","observedHarness":"Claude Code","rawScore":89.12,"zScore":0.248715,"seModel":"upstream_se","seRaw":1.1538,"ciHalfWidth":2.2614,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.963719,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"meta:muse-wonder:xhigh:muse-code:native","provider":"meta","modelSlug":"muse-wonder","displayName":"Muse Wonder","effort":"xhigh","harness":"muse-code","harnessRegime":"native","domainId":"harnesses","theta":0.20193,"shrinkageFactor":0.839444,"score":53.03,"se":8.12,"samplingSe":4.8659,"residualSe":6.5023,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":5.500126,"costFixedFloorUsd":3.715621,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.6404,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Muse Wonder:xhigh:Muse Code","observedHarness":"Muse Code","rawScore":46.97,"zScore":0.581055,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.431242,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"anthropic:claude-sonnet-5.5:high:claude-code:native","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"high","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.183994,"shrinkageFactor":0.968747,"score":52.76,"se":7.22,"samplingSe":2.1468,"residualSe":6.8918,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.695509,"costFixedFloorUsd":1.195212,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.0466,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Sonnet 5.5:Claude Code:high","observedHarness":"Claude Code","rawScore":86.79,"zScore":-0.045954,"seModel":"upstream_se","seRaw":1.1498,"ciHalfWidth":2.2536,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.390423,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"google:gemini-4-argon:high:antigravity:native","provider":"google","modelSlug":"gemini-4-argon","displayName":"Gemini 4 Argon","effort":"high","harness":"antigravity","harnessRegime":"native","domainId":"harnesses","theta":0.148343,"shrinkageFactor":0.907809,"score":52.23,"se":5.32,"samplingSe":3.6872,"residualSe":3.8365,"coverageCount":3,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.487258,"costFixedFloorUsd":1.680273,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1048576,"releaseDate":"2026-09-20","paretoProb":0.1577,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:bdeb89e3a86551106e671c63fa1415bc","observedHarness":"Antigravity CLI","rawScore":24.82,"zScore":-0.530019,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:bdeb89e3a86551106e671c63fa1415bc","observedHarness":"Antigravity CLI","rawScore":55.11,"zScore":-0.378313,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Gemini 4 Argon:high:Antigravity CLI","observedHarness":"Antigravity CLI","rawScore":55.45,"zScore":1.069762,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.360545,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"openai:gpt-5.6-sol:xhigh:codex:native","provider":"openai","modelSlug":"gpt-5.6-sol","displayName":"GPT-5.6 Sol","effort":"xhigh","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.147544,"shrinkageFactor":0.984558,"score":52.21,"se":7.16,"samplingSe":1.509,"residualSe":7.0043,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.160897,"costFixedFloorUsd":0.804586,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0191,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-5.6 Sol:Codex:xhigh","observedHarness":"Codex","rawScore":86.47,"zScore":-0.086423,"seModel":"upstream_se","seRaw":0.8017,"ciHalfWidth":1.5713,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.609172,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6.1-sol:medium:codex:native","provider":"openai","modelSlug":"gpt-6.1-sol","displayName":"GPT-6.1 Sol","effort":"medium","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.143932,"shrinkageFactor":0.985285,"score":52.16,"se":7.16,"samplingSe":1.4731,"residualSe":7.0094,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.78297,"costFixedFloorUsd":0.1993,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.148,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6.1 Sol:Codex:medium","observedHarness":"Codex","rawScore":86.44,"zScore":-0.090217,"seModel":"upstream_se","seRaw":0.7823,"ciHalfWidth":1.5333,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.3986,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-opus-5.5:low:claude-code:native","provider":"anthropic","modelSlug":"claude-opus-5.5","displayName":"Claude Opus 5.5","effort":"low","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.129971,"shrinkageFactor":0.960375,"score":51.95,"se":7.25,"samplingSe":2.4173,"residualSe":6.8322,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.330096,"costFixedFloorUsd":0.847654,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1000000,"releaseDate":"2026-09-22","paretoProb":0.1054,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Opus 5.5:Claude Code:low","observedHarness":"Claude Code","rawScore":86.36,"zScore":-0.100335,"seModel":"upstream_se","seRaw":1.3003,"ciHalfWidth":2.5486,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.695309,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6.1-sol:low:codex:native","provider":"openai","modelSlug":"gpt-6.1-sol","displayName":"GPT-6.1 Sol","effort":"low","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.116305,"shrinkageFactor":0.983787,"score":51.74,"se":7.17,"samplingSe":1.5463,"residualSe":6.9988,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.601029,"costFixedFloorUsd":0.152988,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.2501,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6.1 Sol:Codex:low","observedHarness":"Codex","rawScore":86.22,"zScore":-0.11804,"seModel":"upstream_se","seRaw":0.8218,"ciHalfWidth":1.6107,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.305976,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-opus-5:medium:claude-code:native","provider":"anthropic","modelSlug":"claude-opus-5","displayName":"Claude Opus 5","effort":"medium","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.114006,"shrinkageFactor":0.840198,"score":51.71,"se":8.12,"samplingSe":4.8545,"residualSe":6.5082,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":7.158209,"costFixedFloorUsd":4.835743,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-07-28","paretoProb":0.0209,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Opus 5:medium:Claude Code","observedHarness":"Claude Code","rawScore":45.15,"zScore":0.476168,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":9.671485,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"anthropic:claude-fable-5:max:claude-code:native","provider":"anthropic","modelSlug":"claude-fable-5","displayName":"Claude Fable 5","effort":"max","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":0.099503,"shrinkageFactor":0.840356,"score":51.49,"se":8.12,"samplingSe":4.8521,"residualSe":6.5094,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":16.294217,"costFixedFloorUsd":11.007591,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":20,"inputPer1mUsd":10,"outputPer1mUsd":50,"contextWindow":1000000,"releaseDate":"2026-06-09","paretoProb":0.0081,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Fable 5:max:Claude Code","observedHarness":"Claude Code","rawScore":44.85,"zScore":0.458879,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":22.015182,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"openai:gpt-6-sol:xhigh:codex:native","provider":"openai","modelSlug":"gpt-6-sol","displayName":"GPT-6 Sol","effort":"xhigh","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.078465,"shrinkageFactor":0.955763,"score":51.18,"se":7.26,"samplingSe":2.5541,"residualSe":6.7994,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.92128,"costFixedFloorUsd":0.998136,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0131,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6 Sol:Codex:xhigh","observedHarness":"Codex","rawScore":85.94,"zScore":-0.153451,"seModel":"upstream_se","seRaw":1.3772,"ciHalfWidth":2.6993,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.996273,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-5.6-sol:high:codex:native","provider":"openai","modelSlug":"gpt-5.6-sol","displayName":"GPT-5.6 Sol","effort":"high","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.058507,"shrinkageFactor":0.977077,"score":50.88,"se":7.19,"samplingSe":1.8386,"residualSe":6.9511,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.017615,"costFixedFloorUsd":1.022658,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0115,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-5.6 Sol:Codex:high","observedHarness":"Codex","rawScore":85.76,"zScore":-0.176215,"seModel":"upstream_se","seRaw":0.9805,"ciHalfWidth":1.9218,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.045316,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-5.6-sol:max:codex:native","provider":"openai","modelSlug":"gpt-5.6-sol","displayName":"GPT-5.6 Sol","effort":"max","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":0.014349,"shrinkageFactor":0.959932,"score":50.22,"se":3.86,"samplingSe":2.4308,"residualSe":2.9928,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.737824,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:GPT-5.6 Sol:max:Codex","observedHarness":"Codex","rawScore":38.48,"zScore":0.091772,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.702121,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-5.6 Sol:Codex:max","observedHarness":"Codex","rawScore":87.18,"zScore":0.003369,"seModel":"upstream_se","seRaw":0.6243,"ciHalfWidth":1.2236,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.773526,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6.2-sol:low:codex:native","provider":"openai","modelSlug":"gpt-6.2-sol","displayName":"GPT-6.2 Sol","effort":"low","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-0.050406,"shrinkageFactor":0.954228,"score":49.24,"se":4.36,"samplingSe":2.598,"residualSe":3.4953,"coverageCount":3,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.457894,"costFixedFloorUsd":0.116554,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.3322,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:8178a0ab530e25e65f53d836ef248eae","observedHarness":"Codex","rawScore":28.1,"zScore":-0.14156,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:8178a0ab530e25e65f53d836ef248eae","observedHarness":"Codex","rawScore":42.47,"zScore":-2.330284,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6.2 Sol:Codex:low","observedHarness":"Codex","rawScore":88.44,"zScore":0.162718,"seModel":"upstream_se","seRaw":0.3481,"ciHalfWidth":0.6823,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.233108,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"zhipu:glm-5.3:max:claude-code:native","provider":"zhipu","modelSlug":"glm-5.3","displayName":"GLM-5.3","effort":"max","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":-0.076951,"shrinkageFactor":0.843034,"score":48.85,"se":8.11,"samplingSe":4.8112,"residualSe":6.5302,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":6.117623,"costFixedFloorUsd":4.132772,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2.15,"inputPer1mUsd":1.4,"outputPer1mUsd":4.4,"contextWindow":1048576,"releaseDate":"2026-09-18","paretoProb":0.4983,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:GLM-5.3:max:Claude Code","observedHarness":"Claude Code","rawScore":41.21,"zScore":0.249104,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":8.265545,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"anthropic:claude-haiku-5.5:max:claude-code:native","provider":"anthropic","modelSlug":"claude-haiku-5.5","displayName":"Claude Haiku 5.5","effort":"max","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":-0.094867,"shrinkageFactor":0.962708,"score":48.58,"se":2.35,"samplingSe":2.3451,"residualSe":0,"coverageCount":4,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.211191,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1000000,"releaseDate":"2026-10-07","paretoProb":0.0477,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:2b8b120200b4dc63bb579d5a53b52b70","observedHarness":"Claude Code","rawScore":23.08,"zScore":-0.736092,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:2b8b120200b4dc63bb579d5a53b52b70","observedHarness":"Claude Code","rawScore":33.87,"zScore":-3,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Haiku 5.5:max:Claude Code","observedHarness":"Claude Code","rawScore":40,"zScore":0.179371,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.886273,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Haiku 5.5:Claude Code:max","observedHarness":"Claude Code","rawScore":92.16,"zScore":0.633176,"seModel":"upstream_se","seRaw":0.3681,"ciHalfWidth":0.7215,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.53611,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-haiku-5.5:low:claude-code:native","provider":"anthropic","modelSlug":"claude-haiku-5.5","displayName":"Claude Haiku 5.5","effort":"low","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":-0.101637,"shrinkageFactor":0.913151,"score":48.48,"se":7.42,"samplingSe":3.5788,"residualSe":6.4963,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.034005,"costFixedFloorUsd":0.517743,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1000000,"releaseDate":"2026-10-07","paretoProb":0.2803,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Haiku 5.5:Claude Code:low","observedHarness":"Claude Code","rawScore":84.42,"zScore":-0.345681,"seModel":"upstream_se","seRaw":1.9742,"ciHalfWidth":3.8694,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.035486,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-haiku-5.5:medium:claude-code:native","provider":"anthropic","modelSlug":"claude-haiku-5.5","displayName":"Claude Haiku 5.5","effort":"medium","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":-0.140598,"shrinkageFactor":0.867922,"score":47.89,"se":7.59,"samplingSe":4.4133,"residualSe":6.1745,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.661794,"costFixedFloorUsd":0.677543,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1000000,"releaseDate":"2026-10-07","paretoProb":0.1119,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Haiku 5.5:Claude Code:medium","observedHarness":"Claude Code","rawScore":84.03,"zScore":-0.395004,"seModel":"upstream_se","seRaw":2.4972,"ciHalfWidth":4.8945,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.355085,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-sonnet-5.5:medium:claude-code:native","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"medium","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":-0.145266,"shrinkageFactor":0.966315,"score":47.82,"se":7.23,"samplingSe":2.2288,"residualSe":6.8745,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":5.89725,"costFixedFloorUsd":1.501107,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.0063,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Sonnet 5.5:Claude Code:medium","observedHarness":"Claude Code","rawScore":84.1,"zScore":-0.386151,"seModel":"upstream_se","seRaw":1.1952,"ciHalfWidth":2.3426,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.002214,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6.1-terra:xhigh:codex:native","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"xhigh","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-0.185365,"shrinkageFactor":0.903465,"score":47.22,"se":7.45,"samplingSe":3.773,"residualSe":6.4274,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.472404,"costFixedFloorUsd":0.120247,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.2749,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6.1 Terra:Codex:xhigh","observedHarness":"Codex","rawScore":83.68,"zScore":-0.439267,"seModel":"upstream_se","seRaw":2.0925,"ciHalfWidth":4.1013,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.240495,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"meta:muse-wonder:max:muse-code:native","provider":"meta","modelSlug":"muse-wonder","displayName":"Muse Wonder","effort":"max","harness":"muse-code","harnessRegime":"native","domainId":"harnesses","theta":-0.193938,"shrinkageFactor":0.823881,"score":47.09,"se":7.73,"samplingSe":5.0963,"residualSe":5.8181,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":7.135241,"costLogSd":0.6227,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.3467,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:f83b8dc1b834e06b1dd1142e29bc77fc","observedHarness":"Muse Code","rawScore":24.39,"zScore":-0.580945,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:f83b8dc1b834e06b1dd1142e29bc77fc","observedHarness":"Muse Code","rawScore":61.56,"zScore":0.617748,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"spacexai:grok-4.7:xhigh:grok-build:native","provider":"spacexai","modelSlug":"grok-4.7","displayName":"Grok 4.7","effort":"xhigh","harness":"grok-build","harnessRegime":"native","domainId":"harnesses","theta":-0.195045,"shrinkageFactor":0.845604,"score":47.07,"se":8.1,"samplingSe":4.7716,"residualSe":6.5501,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.261004,"costFixedFloorUsd":5.580738,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-10-18","paretoProb":0.9318,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Grok 4.7:xhigh:Grok Build","observedHarness":"Grok Build","rawScore":38.79,"zScore":0.109638,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":11.161476,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"openai:gpt-5.6-terra:xhigh:codex:native","provider":"openai","modelSlug":"gpt-5.6-terra","displayName":"GPT-5.6 Terra","effort":"xhigh","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-0.198805,"shrinkageFactor":0.846811,"score":47.02,"se":7.67,"samplingSe":4.7529,"residualSe":6.0243,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.159895,"costFixedFloorUsd":1.058874,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4.5,"inputPer1mUsd":2,"outputPer1mUsd":12,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0037,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-5.6 Terra:Codex:xhigh","observedHarness":"Codex","rawScore":83.46,"zScore":-0.46709,"seModel":"upstream_se","seRaw":2.7227,"ciHalfWidth":5.3365,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.117749,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"deepseek:deepseek-v4.1-flash-1017:max:claude-code:native","provider":"deepseek","modelSlug":"deepseek-v4.1-flash-1017","displayName":"DeepSeek V4.1 Flash (1017)","effort":"max","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":-0.210235,"shrinkageFactor":0.84598,"score":46.85,"se":8.1,"samplingSe":4.7658,"residualSe":6.553,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.414678,"costFixedFloorUsd":0.280137,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.525,"inputPer1mUsd":0.3,"outputPer1mUsd":1.2,"contextWindow":1048576,"releaseDate":"2026-10-17","paretoProb":1,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:DeepSeek V4.1 Flash 1017:max:Claude Code","observedHarness":"Claude Code","rawScore":38.48,"zScore":0.091772,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.560273,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"openai:gpt-6-sol:high:codex:native","provider":"openai","modelSlug":"gpt-6-sol","displayName":"GPT-6 Sol","effort":"high","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-0.273952,"shrinkageFactor":0.873644,"score":45.89,"se":7.57,"samplingSe":4.3167,"residualSe":6.2152,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.577491,"costFixedFloorUsd":1.165171,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0018,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6 Sol:Codex:high","observedHarness":"Codex","rawScore":82.83,"zScore":-0.546765,"seModel":"upstream_se","seRaw":2.4345,"ciHalfWidth":4.7716,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.330342,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"zhipu:glm-5.3-flash:max:claude-code:native","provider":"zhipu","modelSlug":"glm-5.3-flash","displayName":"GLM-5.3 Flash","effort":"max","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":-0.284462,"shrinkageFactor":0.847961,"score":45.73,"se":8.1,"samplingSe":4.7351,"residualSe":6.5683,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.840435,"costFixedFloorUsd":0.567758,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2375,"inputPer1mUsd":0.15,"outputPer1mUsd":0.5,"contextWindow":1048576,"releaseDate":"2026-09-28","paretoProb":0.548,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:GLM-5.3-Flash:max:Claude Code","observedHarness":"Claude Code","rawScore":36.97,"zScore":0.00475,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.135515,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"openai:gpt-5.6-terra:max:codex:native","provider":"openai","modelSlug":"gpt-5.6-terra","displayName":"GPT-5.6 Terra","effort":"max","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-0.300575,"shrinkageFactor":0.969219,"score":45.49,"se":3.7,"samplingSe":2.1305,"residualSe":3.0217,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.567646,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4.5,"inputPer1mUsd":2,"outputPer1mUsd":12,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:GPT-5.6 Terra:max:Codex","observedHarness":"Codex","rawScore":21.52,"zScore":-0.885641,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.253091,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-5.6 Terra:Codex:max","observedHarness":"Codex","rawScore":89.05,"zScore":0.239863,"seModel":"upstream_se","seRaw":0.7139,"ciHalfWidth":1.3992,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.8822,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-5.6-sol:medium:codex:native","provider":"openai","modelSlug":"gpt-5.6-sol","displayName":"GPT-5.6 Sol","effort":"medium","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-0.332566,"shrinkageFactor":0.963508,"score":45.01,"se":7.24,"samplingSe":2.3198,"residualSe":6.8545,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.369156,"costFixedFloorUsd":0.857597,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0016,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-5.6 Sol:Codex:medium","observedHarness":"Codex","rawScore":82.56,"zScore":-0.580911,"seModel":"upstream_se","seRaw":1.2458,"ciHalfWidth":2.4418,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.715194,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"zhipu:glm-5.3-flash:none:claude-code:native","provider":"zhipu","modelSlug":"glm-5.3-flash","displayName":"GLM-5.3 Flash","effort":"none","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":-0.359631,"shrinkageFactor":0.850213,"score":44.61,"se":8.09,"samplingSe":4.6999,"residualSe":6.5858,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.266309,"costFixedFloorUsd":0.179905,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2375,"inputPer1mUsd":0.15,"outputPer1mUsd":0.5,"contextWindow":1048576,"releaseDate":"2026-09-28","paretoProb":0.9999,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:GLM-5.3-Flash:none:Claude Code","observedHarness":"Claude Code","rawScore":35.45,"zScore":-0.082848,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.359811,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"anthropic:claude-opus-5:low:claude-code:native","provider":"anthropic","modelSlug":"claude-opus-5","displayName":"Claude Opus 5","effort":"low","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":-0.389443,"shrinkageFactor":0.851174,"score":44.16,"se":8.09,"samplingSe":4.6848,"residualSe":6.5932,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":5.369077,"costFixedFloorUsd":3.627091,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-07-28","paretoProb":0.0041,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Opus 5:low:Claude Code","observedHarness":"Claude Code","rawScore":34.85,"zScore":-0.117426,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":7.254182,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"anthropic:claude-sonnet-5.5:low:claude-code:native","provider":"anthropic","modelSlug":"claude-sonnet-5.5","displayName":"Claude Sonnet 5.5","effort":"low","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":-0.442373,"shrinkageFactor":0.95548,"score":43.36,"se":7.26,"samplingSe":2.5623,"residualSe":6.7974,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":5.649632,"costFixedFloorUsd":1.438077,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-09-28","paretoProb":0.001,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Anthropic:Sonnet 5.5:Claude Code:low","observedHarness":"Claude Code","rawScore":81.63,"zScore":-0.698525,"seModel":"upstream_se","seRaw":1.3818,"ciHalfWidth":2.7083,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.876155,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"meta:muse-spark-1.3:max:muse-code:native","provider":"meta","modelSlug":"muse-spark-1.3","displayName":"Muse Spark 1.3","effort":"max","harness":"muse-code","harnessRegime":"native","domainId":"harnesses","theta":-0.472283,"shrinkageFactor":0.831645,"score":42.92,"se":7.7,"samplingSe":4.9827,"residualSe":5.8729,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.799562,"costLogSd":0.907,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2,"inputPer1mUsd":1.25,"outputPer1mUsd":4.25,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.3934,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:8180f45e6f53fc70033de79909aa3b6c","observedHarness":"Muse Code","rawScore":21.91,"zScore":-0.874658,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:8180f45e6f53fc70033de79909aa3b6c","observedHarness":"Muse Code","rawScore":58.6,"zScore":0.160641,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"openai:gpt-6-sol:medium:codex:native","provider":"openai","modelSlug":"gpt-6-sol","displayName":"GPT-6 Sol","effort":"medium","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-0.512154,"shrinkageFactor":0.859237,"score":42.32,"se":7.62,"samplingSe":4.5561,"residualSe":6.1127,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.454147,"costFixedFloorUsd":0.879231,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.0011,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6 Sol:Codex:medium","observedHarness":"Codex","rawScore":80.6,"zScore":-0.828787,"seModel":"upstream_se","seRaw":2.591,"ciHalfWidth":5.0784,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.758462,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6.1-terra:medium:codex:native","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"medium","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-0.56527,"shrinkageFactor":0.85731,"score":41.52,"se":7.63,"samplingSe":4.5872,"residualSe":6.099,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.272649,"costFixedFloorUsd":0.069401,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.4349,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6.1 Terra:Codex:medium","observedHarness":"Codex","rawScore":80.1,"zScore":-0.892021,"seModel":"upstream_se","seRaw":2.6116,"ciHalfWidth":5.1187,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.138802,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"meta:muse-ember:xhigh:muse-code:native","provider":"meta","modelSlug":"muse-ember","displayName":"Muse Ember","effort":"xhigh","harness":"muse-code","harnessRegime":"native","domainId":"harnesses","theta":-0.602806,"shrinkageFactor":0.859153,"score":40.96,"se":8.07,"samplingSe":4.5575,"residualSe":6.655,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.352529,"costFixedFloorUsd":0.238151,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.7798,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Muse Ember:xhigh:Muse Code","observedHarness":"Muse Code","rawScore":30.61,"zScore":-0.36178,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.476303,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"openai:gpt-6.1-terra:high:codex:native","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"high","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-0.607973,"shrinkageFactor":0.782344,"score":40.88,"se":7.94,"samplingSe":5.6655,"residualSe":5.5657,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.252083,"costFixedFloorUsd":0.064166,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.4756,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6.1 Terra:Codex:high","observedHarness":"Codex","rawScore":79.19,"zScore":-1.007106,"seModel":"upstream_se","seRaw":3.3765,"ciHalfWidth":6.6179,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.128332,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-5.6-luna:xhigh:codex:native","provider":"openai","modelSlug":"gpt-5.6-luna","displayName":"GPT-5.6 Luna","effort":"xhigh","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-0.639234,"shrinkageFactor":0.8132,"score":40.41,"se":7.81,"samplingSe":5.2485,"residualSe":5.7852,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.539605,"costFixedFloorUsd":0.137353,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.45,"inputPer1mUsd":0.2,"outputPer1mUsd":1.2,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0763,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-5.6 Luna:Codex:xhigh","observedHarness":"Codex","rawScore":79.11,"zScore":-1.017223,"seModel":"upstream_se","seRaw":3.0681,"ciHalfWidth":6.0135,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.274706,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"meta:muse-ember:max:muse-code:native","provider":"meta","modelSlug":"muse-ember","displayName":"Muse Ember","effort":"max","harness":"muse-code","harnessRegime":"native","domainId":"harnesses","theta":-0.674405,"shrinkageFactor":0.835189,"score":39.88,"se":7.69,"samplingSe":4.9299,"residualSe":5.8979,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.457331,"costLogSd":0.6227,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1048576,"releaseDate":null,"paretoProb":0.6889,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:ab4f6cb89f990484067eca06109875b3","observedHarness":"Muse Code","rawScore":20.79,"zScore":-1.007303,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:ab4f6cb89f990484067eca06109875b3","observedHarness":"Muse Code","rawScore":54.84,"zScore":-0.420009,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"openai:gpt-6-luna:xhigh:codex:native","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"xhigh","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-0.676246,"shrinkageFactor":0.818347,"score":39.86,"se":7.79,"samplingSe":5.1757,"residualSe":5.8218,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.313671,"costFixedFloorUsd":0.079843,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.2994,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6 Luna:Codex:xhigh","observedHarness":"Codex","rawScore":78.79,"zScore":-1.057693,"seModel":"upstream_se","seRaw":3.016,"ciHalfWidth":5.9114,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.159686,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-5.6-luna:max:codex:native","provider":"openai","modelSlug":"gpt-5.6-luna","displayName":"GPT-5.6 Luna","effort":"max","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-0.721421,"shrinkageFactor":0.964168,"score":39.18,"se":3.78,"samplingSe":2.2987,"residualSe":3.006,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.749451,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.45,"inputPer1mUsd":0.2,"outputPer1mUsd":1.2,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:GPT-5.6 Luna:max:Codex","observedHarness":"Codex","rawScore":17.58,"zScore":-1.112705,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.050515,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-5.6 Luna:Codex:max","observedHarness":"Codex","rawScore":84.15,"zScore":-0.379828,"seModel":"upstream_se","seRaw":1.3494,"ciHalfWidth":2.6448,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.448386,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6.2-astra:low:codex:native","provider":"openai","modelSlug":"gpt-6.2-astra","displayName":"GPT-6.2 Astra","effort":"low","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-0.743723,"shrinkageFactor":0.817997,"score":38.84,"se":7.76,"samplingSe":5.1807,"residualSe":5.7765,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.513457,"costLogSd":0.6227,"costImputed":true,"costEstimated":true,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":16,"inputPer1mUsd":8,"outputPer1mUsd":40,"contextWindow":1050000,"releaseDate":"2026-11-17","paretoProb":0.0096,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:e7d93c34f5c806bec7608750fc56e91a","observedHarness":"Codex","rawScore":26.51,"zScore":-0.329868,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:e7d93c34f5c806bec7608750fc56e91a","observedHarness":"Codex","rawScore":36.29,"zScore":-3,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"openai:gpt-5.5:xhigh:codex:native","provider":"openai","modelSlug":"gpt-5.5","displayName":"GPT-5.5","effort":"xhigh","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-0.751166,"shrinkageFactor":0.866948,"score":38.73,"se":7.59,"samplingSe":4.4295,"residualSe":6.1676,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.962257,"costFixedFloorUsd":2.281285,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":11.25,"inputPer1mUsd":5,"outputPer1mUsd":30,"contextWindow":1050000,"releaseDate":"2026-04-23","paretoProb":0,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-5.5:Codex:xhigh","observedHarness":"Codex","rawScore":78.46,"zScore":-1.099427,"seModel":"upstream_se","seRaw":2.5078,"ciHalfWidth":4.9153,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.56257,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"alibaba:qwen3.8-max-0902:max:claude-code:native","provider":"alibaba","modelSlug":"qwen3.8-max-0902","displayName":"Qwen3.8 Max (0902)","effort":"max","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":-0.790612,"shrinkageFactor":0.86773,"score":38.14,"se":8.04,"samplingSe":4.4165,"residualSe":6.7214,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":10.397042,"costFixedFloorUsd":7.023742,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":1000000,"releaseDate":"2026-09-02","paretoProb":1,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Qwen3.8-Max-0902:max:Claude Code","observedHarness":"Claude Code","rawScore":26.97,"zScore":-0.571555,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":14.047485,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"openai:gpt-5.6-terra:high:codex:native","provider":"openai","modelSlug":"gpt-5.6-terra","displayName":"GPT-5.6 Terra","effort":"high","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-0.846621,"shrinkageFactor":0.773233,"score":37.3,"se":7.98,"samplingSe":5.7828,"residualSe":5.5009,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.324398,"costFixedFloorUsd":0.591661,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4.5,"inputPer1mUsd":2,"outputPer1mUsd":12,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0002,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-5.6 Terra:Codex:high","observedHarness":"Codex","rawScore":76.68,"zScore":-1.324539,"seModel":"upstream_se","seRaw":3.4669,"ciHalfWidth":6.7951,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.183321,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6.1-terra:low:codex:native","provider":"openai","modelSlug":"gpt-6.1-terra","displayName":"GPT-6.1 Terra","effort":"low","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-0.960083,"shrinkageFactor":0.729798,"score":35.6,"se":8.17,"samplingSe":6.3124,"residualSe":5.1919,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":true,"effectiveCostUsd":0.223696,"costFixedFloorUsd":0.05694,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":2,"inputPer1mUsd":1,"outputPer1mUsd":5,"contextWindow":1050000,"releaseDate":"2026-09-25","paretoProb":0.4083,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6.1 Terra:Codex:low","observedHarness":"Codex","rawScore":74.95,"zScore":-1.543328,"seModel":"upstream_se","seRaw":3.9008,"ciHalfWidth":7.6456,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.113881,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-opus-4.8:max:claude-code:native","provider":"anthropic","modelSlug":"claude-opus-4.8","displayName":"Claude Opus 4.8","effort":"max","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":-0.967166,"shrinkageFactor":0.877055,"score":35.49,"se":8.02,"samplingSe":4.258,"residualSe":6.7937,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":14.536395,"costFixedFloorUsd":9.820091,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":10,"inputPer1mUsd":5,"outputPer1mUsd":25,"contextWindow":1000000,"releaseDate":"2026-06-12","paretoProb":0,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Opus 4.8:max:Claude Code","observedHarness":"Claude Code","rawScore":23.64,"zScore":-0.763464,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":19.640182,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"openai:gpt-6-luna:max:codex:native","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"max","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-0.976489,"shrinkageFactor":0.944706,"score":35.35,"se":2.86,"samplingSe":2.8555,"residualSe":0,"coverageCount":4,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.158067,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.6336,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:44059b1749b7773326c3ee04acbfaba2","observedHarness":"Codex","rawScore":24.09,"zScore":-0.616475,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:44059b1749b7773326c3ee04acbfaba2","observedHarness":"Codex","rawScore":38.44,"zScore":-2.952629,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:GPT-6 Luna:max:Codex","observedHarness":"Codex","rawScore":15.15,"zScore":-1.252747,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.218606,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6 Luna:Codex:max","observedHarness":"Codex","rawScore":81.42,"zScore":-0.725084,"seModel":"upstream_se","seRaw":2.8822,"ciHalfWidth":5.6491,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.097529,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"moonshot:kimi-k3:max:claude-code:native","provider":"moonshot","modelSlug":"kimi-k3","displayName":"Kimi K3","effort":"max","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":-1.000074,"shrinkageFactor":0.878924,"score":35,"se":8.01,"samplingSe":4.2255,"residualSe":6.8082,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.305862,"costFixedFloorUsd":2.908834,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":6,"inputPer1mUsd":3,"outputPer1mUsd":15,"contextWindow":1048576,"releaseDate":"2026-08-14","paretoProb":0.0208,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Kimi K3:max:Claude Code","observedHarness":"Claude Code","rawScore":23.03,"zScore":-0.798619,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.817667,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"spacexai:grok-4.6:high:grok-build:native","provider":"spacexai","modelSlug":"grok-4.6","displayName":"Grok 4.6","effort":"high","harness":"grok-build","harnessRegime":"native","domainId":"harnesses","theta":-1.183225,"shrinkageFactor":0.890039,"score":32.25,"se":7.98,"samplingSe":4.0269,"residualSe":6.8943,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.05532,"costFixedFloorUsd":5.441788,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-08-25","paretoProb":0.4428,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Grok 4.6:high:Grok Build","observedHarness":"Grok Build","rawScore":19.7,"zScore":-0.990528,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":10.883576,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"openai:gpt-5.6-luna:high:codex:native","provider":"openai","modelSlug":"gpt-5.6-luna","displayName":"GPT-5.6 Luna","effort":"high","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-1.347132,"shrinkageFactor":0.754186,"score":29.79,"se":8.06,"samplingSe":6.0208,"residualSe":5.3654,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.425799,"costFixedFloorUsd":0.108384,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.45,"inputPer1mUsd":0.2,"outputPer1mUsd":1.2,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.0244,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-5.6 Luna:Codex:high","observedHarness":"Codex","rawScore":71.22,"zScore":-2.015051,"seModel":"upstream_se","seRaw":3.714,"ciHalfWidth":7.2794,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.216769,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-5.5:high:codex:native","provider":"openai","modelSlug":"gpt-5.5","displayName":"GPT-5.5","effort":"high","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-1.469412,"shrinkageFactor":0.753293,"score":27.96,"se":8.07,"samplingSe":6.0317,"residualSe":5.359,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.244773,"costFixedFloorUsd":2.098654,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":11.25,"inputPer1mUsd":5,"outputPer1mUsd":30,"contextWindow":1050000,"releaseDate":"2026-04-23","paretoProb":0,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-5.5:Codex:high","observedHarness":"Codex","rawScore":69.92,"zScore":-2.179459,"seModel":"upstream_se","seRaw":3.8102,"ciHalfWidth":7.468,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.197308,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"spacexai:grok-4.5:high:grok-build:native","provider":"spacexai","modelSlug":"grok-4.5","displayName":"Grok 4.5","effort":"high","harness":"grok-build","harnessRegime":"native","domainId":"harnesses","theta":-1.608567,"shrinkageFactor":0.920193,"score":25.87,"se":7.91,"samplingSe":3.4306,"residualSe":7.1278,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":4.696743,"costFixedFloorUsd":3.172894,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-06-30","paretoProb":0.9741,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Grok 4.5:high:Grok Build","observedHarness":"Grok Build","rawScore":12.42,"zScore":-1.410078,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":6.345788,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"openai:gpt-5.6-sol:low:codex:native","provider":"openai","modelSlug":"gpt-5.6-sol","displayName":"GPT-5.6 Sol","effort":"low","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-1.619153,"shrinkageFactor":0.786247,"score":25.71,"se":7.93,"samplingSe":5.6144,"residualSe":5.5935,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.669814,"costFixedFloorUsd":0.679584,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":8,"inputPer1mUsd":4,"outputPer1mUsd":20,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-5.6 Sol:Codex:low","observedHarness":"Codex","rawScore":69.05,"zScore":-2.289485,"seModel":"upstream_se","seRaw":3.5024,"ciHalfWidth":6.8647,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.359168,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"anthropic:claude-sonnet-5:max:claude-code:native","provider":"anthropic","modelSlug":"claude-sonnet-5","displayName":"Claude Sonnet 5","effort":"max","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":-1.626964,"shrinkageFactor":0.921623,"score":25.6,"se":7.91,"samplingSe":3.3997,"residualSe":7.1389,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":21.53987,"costFixedFloorUsd":14.551303,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1000000,"releaseDate":"2026-07-15","paretoProb":0,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Sonnet 5:max:Claude Code","observedHarness":"Claude Code","rawScore":12.12,"zScore":-1.427367,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":29.102606,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"openai:gpt-6-sol:low:codex:native","provider":"openai","modelSlug":"gpt-6-sol","displayName":"GPT-6 Sol","effort":"low","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-1.78683,"shrinkageFactor":0.79778,"score":23.2,"se":7.88,"samplingSe":5.4609,"residualSe":5.6755,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":2.002518,"costFixedFloorUsd":0.509728,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4,"inputPer1mUsd":2,"outputPer1mUsd":10,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6 Sol:Codex:low","observedHarness":"Codex","rawScore":67.62,"zScore":-2.470334,"seModel":"upstream_se","seRaw":3.5937,"ciHalfWidth":7.0437,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.019456,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-5.6-terra:medium:codex:native","provider":"openai","modelSlug":"gpt-5.6-terra","displayName":"GPT-5.6 Terra","effort":"medium","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-1.942028,"shrinkageFactor":0.809198,"score":20.87,"se":7.83,"samplingSe":5.3045,"residualSe":5.7567,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.348131,"costFixedFloorUsd":0.343158,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4.5,"inputPer1mUsd":2,"outputPer1mUsd":12,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-5.6 Terra:Codex:medium","observedHarness":"Codex","rawScore":66.35,"zScore":-2.630947,"seModel":"upstream_se","seRaw":3.8156,"ciHalfWidth":7.4786,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.686316,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"deepseek:deepseek-v4-pro-0813:max:claude-code:native","provider":"deepseek","modelSlug":"deepseek-v4-pro-0813","displayName":"DeepSeek V4 Pro (0813)","effort":"max","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":-1.975749,"shrinkageFactor":0.95051,"score":20.36,"se":7.84,"samplingSe":2.7015,"residualSe":7.3627,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.630586,"costFixedFloorUsd":1.101546,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.98,"inputPer1mUsd":1.32,"outputPer1mUsd":3.96,"contextWindow":1048576,"releaseDate":"2026-08-13","paretoProb":0.0074,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:DeepSeek V4 Pro 0813:max:Claude Code","observedHarness":"Claude Code","rawScore":6.67,"zScore":-1.741453,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.203091,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"google:gemini-3.1-pro:high:antigravity:native","provider":"google","modelSlug":"gemini-3.1-pro","displayName":"Gemini 3.1 Pro","effort":"high","harness":"antigravity","harnessRegime":"native","domainId":"harnesses","theta":-2.247792,"shrinkageFactor":0.975161,"score":16.28,"se":7.79,"samplingSe":1.9139,"residualSe":7.5536,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":8.188926,"costFixedFloorUsd":5.532045,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4.5,"inputPer1mUsd":2,"outputPer1mUsd":12,"contextWindow":1048576,"releaseDate":"2026-02-19","paretoProb":0,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:Gemini 3.1 Pro:high:Antigravity CLI","observedHarness":"Antigravity CLI","rawScore":2.73,"zScore":-1.968518,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":11.064091,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"openai:gpt-5.4-mini:xhigh:codex:native","provider":"openai","modelSlug":"gpt-5.4-mini","displayName":"GPT-5.4 Mini","effort":"xhigh","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-2.313381,"shrinkageFactor":0.981354,"score":15.3,"se":7.78,"samplingSe":1.6582,"residualSe":7.6016,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.506221,"costFixedFloorUsd":1.01753,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":1.6875,"inputPer1mUsd":0.75,"outputPer1mUsd":4.5,"contextWindow":400000,"releaseDate":"2026-03-17","paretoProb":0,"benchmarks":[{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","sourceId":"tbench","sourceRowId":"tbench:4.1:GPT-5.4 Mini:xhigh:Codex","observedHarness":"Codex","rawScore":1.82,"zScore":-2.020961,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.035061,"evalDate":null,"publishedDate":null,"sourceUrl":"https://www.tbench.ai/"}]},{"id":"openai:gpt-5.5:medium:codex:native","provider":"openai","modelSlug":"gpt-5.5","displayName":"GPT-5.5","effort":"medium","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-2.442935,"shrinkageFactor":0.887468,"score":13.36,"se":7.51,"samplingSe":4.0737,"residualSe":6.3136,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":5.319455,"costFixedFloorUsd":1.354033,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":11.25,"inputPer1mUsd":5,"outputPer1mUsd":30,"contextWindow":1050000,"releaseDate":"2026-04-23","paretoProb":0,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-5.5:Codex:medium","observedHarness":"Codex","rawScore":63.54,"zScore":-2.986321,"seModel":"upstream_se","seRaw":3.8413,"ciHalfWidth":7.5289,"ciLower":null,"ciUpper":null,"measuredCostUsd":2.708066,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-5.6-terra:low:codex:native","provider":"openai","modelSlug":"gpt-5.6-terra","displayName":"GPT-5.6 Terra","effort":"low","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-2.719334,"shrinkageFactor":0.983935,"score":9.21,"se":7.17,"samplingSe":1.5392,"residualSe":6.9998,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":1.13214,"costFixedFloorUsd":0.288179,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":4.5,"inputPer1mUsd":2,"outputPer1mUsd":12,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-5.6 Terra:Codex:low","observedHarness":"Codex","rawScore":59.7,"zScore":-3,"seModel":"upstream_se","seRaw":3.4294,"ciHalfWidth":6.7216,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.576358,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6-luna:high:codex:native","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"high","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-2.747102,"shrinkageFactor":0.994071,"score":8.79,"se":7.13,"samplingSe":0.935,"residualSe":7.072,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.101971,"costFixedFloorUsd":0.025956,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.215,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6 Luna:Codex:high","observedHarness":"Codex","rawScore":57.35,"zScore":-3,"seModel":"upstream_se","seRaw":3.7954,"ciHalfWidth":7.439,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.051912,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-5.5:low:codex:native","provider":"openai","modelSlug":"gpt-5.5","displayName":"GPT-5.5","effort":"low","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-2.763343,"shrinkageFactor":1,"score":8.55,"se":7.11,"samplingSe":0.0023,"residualSe":7.1141,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":3.525345,"costFixedFloorUsd":0.897354,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":11.25,"inputPer1mUsd":5,"outputPer1mUsd":30,"contextWindow":1050000,"releaseDate":"2026-04-23","paretoProb":0,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-5.5:Codex:low","observedHarness":"Codex","rawScore":49.8,"zScore":-3,"seModel":"upstream_se","seRaw":2.9303,"ciHalfWidth":5.7434,"ciLower":null,"ciUpper":null,"measuredCostUsd":1.794708,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-5.6-luna:low:codex:native","provider":"openai","modelSlug":"gpt-5.6-luna","displayName":"GPT-5.6 Luna","effort":"low","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-2.763343,"shrinkageFactor":1,"score":8.55,"se":7.11,"samplingSe":0,"residualSe":7.1141,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.047135,"costFixedFloorUsd":0.011998,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.45,"inputPer1mUsd":0.2,"outputPer1mUsd":1.2,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.4649,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-5.6 Luna:Codex:low","observedHarness":"Codex","rawScore":41.29,"zScore":-3,"seModel":"upstream_se","seRaw":0.9566,"ciHalfWidth":1.8749,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.023996,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-5.6-luna:medium:codex:native","provider":"openai","modelSlug":"gpt-5.6-luna","displayName":"GPT-5.6 Luna","effort":"medium","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-2.763323,"shrinkageFactor":0.999993,"score":8.55,"se":7.11,"samplingSe":0.0328,"residualSe":7.1141,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.130426,"costFixedFloorUsd":0.033199,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.45,"inputPer1mUsd":0.2,"outputPer1mUsd":1.2,"contextWindow":1050000,"releaseDate":"2026-06-18","paretoProb":0.1623,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-5.6 Luna:Codex:medium","observedHarness":"Codex","rawScore":53.28,"zScore":-3,"seModel":"upstream_se","seRaw":2.9402,"ciHalfWidth":5.7628,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.066398,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6-luna:low:codex:native","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"low","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-2.763343,"shrinkageFactor":1,"score":8.55,"se":7.11,"samplingSe":0,"residualSe":7.1141,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.054588,"costFixedFloorUsd":0.013895,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.3797,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6 Luna:Codex:low","observedHarness":"Codex","rawScore":40.83,"zScore":-3,"seModel":"upstream_se","seRaw":2.0415,"ciHalfWidth":4.0013,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.02779,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"openai:gpt-6-luna:medium:codex:native","provider":"openai","modelSlug":"gpt-6-luna","displayName":"GPT-6 Luna","effort":"medium","harness":"codex","harnessRegime":"native","domainId":"harnesses","theta":-2.763343,"shrinkageFactor":1,"score":8.55,"se":7.11,"samplingSe":0,"residualSe":7.1141,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":true,"imputationSensitive":false,"effectiveCostUsd":0.01873,"costFixedFloorUsd":0.004768,"costLogSd":0.5841,"costImputed":true,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.2,"inputPer1mUsd":0.1,"outputPer1mUsd":0.5,"contextWindow":1050000,"releaseDate":"2026-08-22","paretoProb":0.9614,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:OpenAI:GPT-6 Luna:Codex:medium","observedHarness":"Codex","rawScore":45.72,"zScore":-3,"seModel":"upstream_se","seRaw":1.3221,"ciHalfWidth":2.5913,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.009535,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]}],"provisionalConfigs":[{"id":"spacexai:grok-4.7:xhigh:cursor:native","provider":"spacexai","modelSlug":"grok-4.7","displayName":"Grok 4.7","effort":"xhigh","harness":"cursor","harnessRegime":"native","domainId":"harnesses","theta":0.992432,"shrinkageFactor":0.997445,"score":64.89,"se":7.12,"samplingSe":0.6138,"residualSe":7.0959,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-10-18","paretoProb":0,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:xAI:Grok 4.7:Cursor:xhigh","observedHarness":"Cursor","rawScore":93.15,"zScore":0.758379,"seModel":"upstream_se","seRaw":0.324,"ciHalfWidth":0.635,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"spacexai:grok-4.7:high:cursor:native","provider":"spacexai","modelSlug":"grok-4.7","displayName":"Grok 4.7","effort":"high","harness":"cursor","harnessRegime":"native","domainId":"harnesses","theta":0.935368,"shrinkageFactor":0.995825,"score":64.03,"se":7.13,"samplingSe":0.7847,"residualSe":7.0844,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-10-18","paretoProb":0,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:xAI:Grok 4.7:Cursor:high","observedHarness":"Cursor","rawScore":92.71,"zScore":0.702733,"seModel":"upstream_se","seRaw":0.4145,"ciHalfWidth":0.8124,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"spacexai:grok-4.7:medium:cursor:native","provider":"spacexai","modelSlug":"grok-4.7","displayName":"Grok 4.7","effort":"medium","harness":"cursor","harnessRegime":"native","domainId":"harnesses","theta":0.871556,"shrinkageFactor":0.982465,"score":63.07,"se":7.17,"samplingSe":1.608,"residualSe":6.9894,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-10-18","paretoProb":0,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:xAI:Grok 4.7:Cursor:medium","observedHarness":"Cursor","rawScore":92.3,"zScore":0.650882,"seModel":"upstream_se","seRaw":0.8552,"ciHalfWidth":1.6762,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"spacexai:grok-4.7:low:cursor:native","provider":"spacexai","modelSlug":"grok-4.7","displayName":"Grok 4.7","effort":"low","harness":"cursor","harnessRegime":"native","domainId":"harnesses","theta":0.493173,"shrinkageFactor":0.944929,"score":57.4,"se":7.3,"samplingSe":2.8498,"residualSe":6.7223,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":3,"inputPer1mUsd":2,"outputPer1mUsd":6,"contextWindow":500000,"releaseDate":"2026-10-18","paretoProb":0,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:xAI:Grok 4.7:Cursor:low","observedHarness":"Cursor","rawScore":89.42,"zScore":0.286656,"seModel":"upstream_se","seRaw":1.5454,"ciHalfWidth":3.029,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"deepseek:deepseek-v4.1-flash:low:deepseek-harness:native","provider":"deepseek","modelSlug":"deepseek-v4.1-flash","displayName":"DeepSeek V4.1 Flash","effort":"low","harness":"deepseek-harness","harnessRegime":"native","domainId":"harnesses","theta":0.421009,"shrinkageFactor":0.989055,"score":56.32,"se":7.15,"samplingSe":1.2704,"residualSe":7.0363,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-10","paretoProb":0,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:DeepSeek:DeepSeek V4.1 Flash:DeepSeek Harness:low","observedHarness":"DeepSeek Harness 0.2.1-rc.2","rawScore":88.65,"zScore":0.189276,"seModel":"upstream_se","seRaw":0.6734,"ciHalfWidth":1.3199,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.057727,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"deepseek:deepseek-v4.1-flash:high:deepseek-harness:native","provider":"deepseek","modelSlug":"deepseek-v4.1-flash","displayName":"DeepSeek V4.1 Flash","effort":"high","harness":"deepseek-harness","harnessRegime":"native","domainId":"harnesses","theta":0.41102,"shrinkageFactor":0.977848,"score":56.17,"se":7.19,"samplingSe":1.8074,"residualSe":6.9565,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-10","paretoProb":0,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:DeepSeek:DeepSeek V4.1 Flash:DeepSeek Harness:high","observedHarness":"DeepSeek Harness 0.2.1-rc.2","rawScore":88.61,"zScore":0.184217,"seModel":"upstream_se","seRaw":0.9635,"ciHalfWidth":1.8885,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.066663,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"meta:muse-wonder-full:max:muse-code:native","provider":"meta","modelSlug":"muse-wonder-full","displayName":"Muse Wonder Full","effort":"max","harness":"muse-code","harnessRegime":"native","domainId":"harnesses","theta":0.383575,"shrinkageFactor":0.986248,"score":55.75,"se":7.16,"samplingSe":1.4241,"residualSe":7.0163,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Meta:Muse Wonder Full:Muse Code:max","observedHarness":"Muse Code","rawScore":88.36,"zScore":0.1526,"seModel":"upstream_se","seRaw":0.7559,"ciHalfWidth":1.4816,"ciLower":null,"ciUpper":null,"measuredCostUsd":5.910996,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"deepseek:deepseek-v4.1-flash:max:deepseek-harness:native","provider":"deepseek","modelSlug":"deepseek-v4.1-flash","displayName":"DeepSeek V4.1 Flash","effort":"max","harness":"deepseek-harness","harnessRegime":"native","domainId":"harnesses","theta":0.379864,"shrinkageFactor":0.977269,"score":55.7,"se":7.19,"samplingSe":1.8309,"residualSe":6.9524,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-09-10","paretoProb":0,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:DeepSeek:DeepSeek V4.1 Flash:DeepSeek Harness:max","observedHarness":"DeepSeek Harness 0.2.1-rc.2","rawScore":88.36,"zScore":0.1526,"seModel":"upstream_se","seRaw":0.9763,"ciHalfWidth":1.9135,"ciLower":null,"ciUpper":null,"measuredCostUsd":0.074057,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"meta:muse-wonder-full:medium:muse-code:native","provider":"meta","modelSlug":"muse-wonder-full","displayName":"Muse Wonder Full","effort":"medium","harness":"muse-code","harnessRegime":"native","domainId":"harnesses","theta":0.369262,"shrinkageFactor":0.978566,"score":55.54,"se":7.19,"samplingSe":1.7779,"residualSe":6.9616,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Meta:Muse Wonder Full:Muse Code:medium","observedHarness":"Muse Code","rawScore":88.27,"zScore":0.141218,"seModel":"upstream_se","seRaw":0.9474,"ciHalfWidth":1.8569,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.085146,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"meta:muse-wonder-full:high:muse-code:native","provider":"meta","modelSlug":"muse-wonder-full","displayName":"Muse Wonder Full","effort":"high","harness":"muse-code","harnessRegime":"native","domainId":"harnesses","theta":0.357622,"shrinkageFactor":0.974125,"score":55.36,"se":7.2,"samplingSe":1.9534,"residualSe":6.9301,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Meta:Muse Wonder Full:Muse Code:high","observedHarness":"Muse Code","rawScore":88.19,"zScore":0.131101,"seModel":"upstream_se","seRaw":1.0433,"ciHalfWidth":2.0449,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.719413,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"meta:muse-wonder-full:xhigh:muse-code:native","provider":"meta","modelSlug":"muse-wonder-full","displayName":"Muse Wonder Full","effort":"xhigh","harness":"muse-code","harnessRegime":"native","domainId":"harnesses","theta":0.256608,"shrinkageFactor":0.978442,"score":53.85,"se":7.19,"samplingSe":1.783,"residualSe":6.9608,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Meta:Muse Wonder Full:Muse Code:xhigh","observedHarness":"Muse Code","rawScore":87.36,"zScore":0.026133,"seModel":"upstream_se","seRaw":0.9502,"ciHalfWidth":1.8624,"ciLower":null,"ciUpper":null,"measuredCostUsd":4.857141,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"meta:muse-wonder-full:low:muse-code:native","provider":"meta","modelSlug":"muse-wonder-full","displayName":"Muse Wonder Full","effort":"low","harness":"muse-code","harnessRegime":"native","domainId":"harnesses","theta":0.203421,"shrinkageFactor":0.968,"score":53.05,"se":7.22,"samplingSe":2.1723,"residualSe":6.8865,"coverageCount":1,"activeBenchmarkCount":4,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":null,"paretoProb":0,"benchmarks":[{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","sourceId":"vulcanbench","sourceRowId":"vulcanbench:swe_v4:Meta:Muse Wonder Full:Muse Code:low","observedHarness":"Muse Code","rawScore":86.95,"zScore":-0.025719,"seModel":"upstream_se","seRaw":1.1639,"ciHalfWidth":2.2812,"ciLower":null,"ciUpper":null,"measuredCostUsd":3.931575,"evalDate":null,"publishedDate":null,"sourceUrl":"https://vulcanbench.com/"}]},{"id":"alibaba:qwen3.8-max:xhigh:claude-code:native","provider":"alibaba","modelSlug":"qwen3.8-max","displayName":"Qwen3.8 Max","effort":"xhigh","harness":"claude-code","harnessRegime":"native","domainId":"harnesses","theta":-0.323966,"shrinkageFactor":0.821417,"score":45.14,"se":7.74,"samplingSe":5.1318,"residualSe":5.8007,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":null,"inputPer1mUsd":null,"outputPer1mUsd":null,"contextWindow":null,"releaseDate":"2026-08-03","paretoProb":0,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:6e872dc3aa0bc90635d8ae831e28f074","observedHarness":"Claude Code","rawScore":24.83,"zScore":-0.528835,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:6e872dc3aa0bc90635d8ae831e28f074","observedHarness":"Claude Code","rawScore":56.18,"zScore":-0.213075,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"deepseek:deepseek-v4.1-flash-1017:max:deepseek-harness:native","provider":"deepseek","modelSlug":"deepseek-v4.1-flash-1017","displayName":"DeepSeek V4.1 Flash (1017)","effort":"max","harness":"deepseek-harness","harnessRegime":"native","domainId":"harnesses","theta":-0.477995,"shrinkageFactor":0.836952,"score":42.83,"se":7.68,"samplingSe":4.9035,"residualSe":5.9104,"coverageCount":2,"activeBenchmarkCount":4,"eligibleForFrontier":false,"imputationSensitive":false,"effectiveCostUsd":null,"costImputed":false,"costEstimated":false,"usesFallbackRouting":false,"costUnit":"usd_per_task","costSource":"measured","blendedListPricePer1mUsd":0.525,"inputPer1mUsd":0.3,"outputPer1mUsd":1.2,"contextWindow":1048576,"releaseDate":"2026-10-17","paretoProb":0,"benchmarks":[{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_frontiercode:4553886307abc03b6a785a5e0a54bcec","observedHarness":"DeepSeek Harness","rawScore":20.66,"zScore":-1.022699,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","sourceId":"artificial-analysis","sourceRowId":"aa:aa_agent_swe_atlas:4553886307abc03b6a785a5e0a54bcec","observedHarness":"DeepSeek Harness","rawScore":61.56,"zScore":0.617748,"seModel":"popoviciu_upper_bound","ciHalfWidth":null,"ciLower":null,"ciUpper":null,"measuredCostUsd":null,"evalDate":null,"publishedDate":null,"sourceUrl":"https://artificialanalysis.ai/agents/coding-agents"}]}],"frontiers":[{"provider":"anthropic","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"anthropic:claude-haiku-5.5:xhigh:claude-code:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.8741,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"anthropic:claude-fable-5.2:medium:claude-code:native","zStepFromPrev":0.018,"pBetterThanPrev":0.5073,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.386,"marginalScorePerDollar":0.087,"costMultipleFromPrev":2.05},{"configId":"anthropic:claude-fable-5.2:high:claude-code:native","zStepFromPrev":1.423,"pBetterThanPrev":0.9226,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.6613,"marginalScorePerDollar":3.336,"costMultipleFromPrev":1.66}],"pruned":[{"configId":"anthropic:claude-haiku-5.5:low:claude-code:native","dominatedById":"anthropic:claude-haiku-5.5:xhigh:claude-code:native","scoreDeficit":13.89,"costRatio":1.23,"costComparisonNote":null},{"configId":"anthropic:claude-haiku-5.5:max:claude-code:native","dominatedById":"anthropic:claude-haiku-5.5:xhigh:claude-code:native","scoreDeficit":13.79,"costRatio":1.34,"costComparisonNote":null},{"configId":"anthropic:claude-haiku-5.5:high:claude-code:native","dominatedById":"anthropic:claude-haiku-5.5:xhigh:claude-code:native","scoreDeficit":7.44,"costRatio":1.49,"costComparisonNote":null},{"configId":"anthropic:claude-haiku-5.5:medium:claude-code:native","dominatedById":"anthropic:claude-haiku-5.5:xhigh:claude-code:native","scoreDeficit":14.48,"costRatio":1.61,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5.5:low:claude-code:native","dominatedById":"anthropic:claude-haiku-5.5:xhigh:claude-code:native","scoreDeficit":10.42,"costRatio":2.02,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.2:low:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:medium:claude-code:native","scoreDeficit":4.04,"costRatio":1,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5.5:high:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:medium:claude-code:native","scoreDeficit":9.76,"costRatio":1.39,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5:low:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:medium:claude-code:native","scoreDeficit":18.36,"costRatio":1.59,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5.5:medium:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:medium:claude-code:native","scoreDeficit":2.05,"costRatio":1.65,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5.5:low:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":26.62,"costRatio":1.01,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5.5:medium:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":22.16,"costRatio":1.05,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5.5:high:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":9.01,"costRatio":1.14,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5.5:xhigh:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":10.73,"costRatio":1.18,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5:medium:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":18.27,"costRatio":1.28,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.1:low:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":16.04,"costRatio":1.35,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5.5:xhigh:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":10.04,"costRatio":1.44,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.1:medium:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":10.99,"costRatio":1.58,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.2:xhigh:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":4.94,"costRatio":1.75,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5:high:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":14.97,"costRatio":1.86,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.1:high:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":10.68,"costRatio":1.94,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5.5:max:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":3.12,"costRatio":2.05,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5:max:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":13.22,"costRatio":2.39,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5:xhigh:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":12.33,"costRatio":2.43,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5.5:max:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":4.53,"costRatio":2.47,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.1:max:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":8.49,"costRatio":2.49,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.8:max:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":34.49,"costRatio":2.59,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.1:xhigh:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":8.97,"costRatio":2.61,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5:max:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":18.49,"costRatio":2.9,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5:max:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":44.38,"costRatio":3.84,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.2:max:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":1.64,"costRatio":5.42,"costComparisonNote":null}],"verdicts":{"budgetId":"anthropic:claude-haiku-5.5:xhigh:claude-code:native","defaultId":null,"maxId":"anthropic:claude-fable-5.2:high:claude-code:native"}},{"provider":"anthropic","harnessRegime":"native","harnessFilter":"claude-code","frontier":[{"configId":"anthropic:claude-haiku-5.5:xhigh:claude-code:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.8741,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"anthropic:claude-fable-5.2:medium:claude-code:native","zStepFromPrev":0.018,"pBetterThanPrev":0.5073,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.386,"marginalScorePerDollar":0.087,"costMultipleFromPrev":2.05},{"configId":"anthropic:claude-fable-5.2:high:claude-code:native","zStepFromPrev":1.423,"pBetterThanPrev":0.9226,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.6613,"marginalScorePerDollar":3.336,"costMultipleFromPrev":1.66}],"pruned":[{"configId":"anthropic:claude-haiku-5.5:low:claude-code:native","dominatedById":"anthropic:claude-haiku-5.5:xhigh:claude-code:native","scoreDeficit":13.89,"costRatio":1.23,"costComparisonNote":null},{"configId":"anthropic:claude-haiku-5.5:max:claude-code:native","dominatedById":"anthropic:claude-haiku-5.5:xhigh:claude-code:native","scoreDeficit":13.79,"costRatio":1.34,"costComparisonNote":null},{"configId":"anthropic:claude-haiku-5.5:high:claude-code:native","dominatedById":"anthropic:claude-haiku-5.5:xhigh:claude-code:native","scoreDeficit":7.44,"costRatio":1.49,"costComparisonNote":null},{"configId":"anthropic:claude-haiku-5.5:medium:claude-code:native","dominatedById":"anthropic:claude-haiku-5.5:xhigh:claude-code:native","scoreDeficit":14.48,"costRatio":1.61,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5.5:low:claude-code:native","dominatedById":"anthropic:claude-haiku-5.5:xhigh:claude-code:native","scoreDeficit":10.42,"costRatio":2.02,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.2:low:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:medium:claude-code:native","scoreDeficit":4.04,"costRatio":1,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5.5:high:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:medium:claude-code:native","scoreDeficit":9.76,"costRatio":1.39,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5:low:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:medium:claude-code:native","scoreDeficit":18.36,"costRatio":1.59,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5.5:medium:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:medium:claude-code:native","scoreDeficit":2.05,"costRatio":1.65,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5.5:low:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":26.62,"costRatio":1.01,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5.5:medium:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":22.16,"costRatio":1.05,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5.5:high:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":9.01,"costRatio":1.14,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5.5:xhigh:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":10.73,"costRatio":1.18,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5:medium:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":18.27,"costRatio":1.28,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.1:low:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":16.04,"costRatio":1.35,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5.5:xhigh:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":10.04,"costRatio":1.44,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.1:medium:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":10.99,"costRatio":1.58,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.2:xhigh:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":4.94,"costRatio":1.75,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5:high:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":14.97,"costRatio":1.86,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.1:high:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":10.68,"costRatio":1.94,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5.5:max:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":3.12,"costRatio":2.05,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5:max:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":13.22,"costRatio":2.39,"costComparisonNote":null},{"configId":"anthropic:claude-opus-5:xhigh:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":12.33,"costRatio":2.43,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5.5:max:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":4.53,"costRatio":2.47,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.1:max:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":8.49,"costRatio":2.49,"costComparisonNote":null},{"configId":"anthropic:claude-opus-4.8:max:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":34.49,"costRatio":2.59,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.1:xhigh:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":8.97,"costRatio":2.61,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5:max:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":18.49,"costRatio":2.9,"costComparisonNote":null},{"configId":"anthropic:claude-sonnet-5:max:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":44.38,"costRatio":3.84,"costComparisonNote":null},{"configId":"anthropic:claude-fable-5.2:max:claude-code:native","dominatedById":"anthropic:claude-fable-5.2:high:claude-code:native","scoreDeficit":1.64,"costRatio":5.42,"costComparisonNote":null}],"verdicts":{"budgetId":"anthropic:claude-haiku-5.5:xhigh:claude-code:native","defaultId":null,"maxId":"anthropic:claude-fable-5.2:high:claude-code:native"}},{"provider":"openai","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"openai:gpt-6-luna:medium:codex:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9614,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"openai:gpt-6-luna:high:codex:native","zStepFromPrev":0.034,"pBetterThanPrev":0.5134,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.215,"marginalScorePerDollar":2.883,"costMultipleFromPrev":5.44},{"configId":"openai:gpt-6-luna:max:codex:native","zStepFromPrev":3.457,"pBetterThanPrev":0.9997,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.6336,"marginalScorePerDollar":473.474,"costMultipleFromPrev":1.55},{"configId":"openai:gpt-6.1-terra:low:codex:native","zStepFromPrev":0.029,"pBetterThanPrev":0.5115,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.4083,"marginalScorePerDollar":3.809,"costMultipleFromPrev":1.42},{"configId":"openai:gpt-6.1-terra:high:codex:native","zStepFromPrev":0.525,"pBetterThanPrev":0.7004,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.4756,"marginalScorePerDollar":186.001,"costMultipleFromPrev":1.13},{"configId":"openai:gpt-6.1-terra:medium:codex:native","zStepFromPrev":0.068,"pBetterThanPrev":0.5273,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.4349,"marginalScorePerDollar":31.119,"costMultipleFromPrev":1.08},{"configId":"openai:gpt-6.2-sol:low:codex:native","zStepFromPrev":0.895,"pBetterThanPrev":0.8145,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.3322,"marginalScorePerDollar":41.675,"costMultipleFromPrev":1.68},{"configId":"openai:gpt-6.1-sol:low:codex:native","zStepFromPrev":0.305,"pBetterThanPrev":0.6198,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.2501,"marginalScorePerDollar":17.466,"costMultipleFromPrev":1.31},{"configId":"openai:gpt-6.2-sol:medium:codex:native","zStepFromPrev":0.556,"pBetterThanPrev":0.7108,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.387,"marginalScorePerDollar":120.317,"costMultipleFromPrev":1.06},{"configId":"openai:gpt-6.2-sol:high:codex:native","zStepFromPrev":0.22,"pBetterThanPrev":0.5871,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.2996,"marginalScorePerDollar":7.214,"costMultipleFromPrev":1.25},{"configId":"openai:gpt-6.2-sol:max:codex:native","zStepFromPrev":0.616,"pBetterThanPrev":0.731,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.3789,"marginalScorePerDollar":606.412,"costMultipleFromPrev":1.01},{"configId":"openai:gpt-6.1-sol:max:codex:native","zStepFromPrev":0.617,"pBetterThanPrev":0.7315,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.3467,"marginalScorePerDollar":5.667,"costMultipleFromPrev":1.48},{"configId":"openai:gpt-6.2-astra:high:codex:native","zStepFromPrev":0.033,"pBetterThanPrev":0.513,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.316,"marginalScorePerDollar":0.183,"costMultipleFromPrev":2.24},{"configId":"openai:gpt-6.2-astra:max:codex:native","zStepFromPrev":0.091,"pBetterThanPrev":0.5362,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.3192,"marginalScorePerDollar":2.177,"costMultipleFromPrev":1.19}],"pruned":[{"configId":"openai:gpt-5.6-luna:low:codex:native","dominatedById":"openai:gpt-6-luna:medium:codex:native","scoreDeficit":0,"costRatio":2.52,"costComparisonNote":null},{"configId":"openai:gpt-6-luna:low:codex:native","dominatedById":"openai:gpt-6-luna:medium:codex:native","scoreDeficit":0,"costRatio":2.91,"costComparisonNote":null},{"configId":"openai:gpt-5.6-luna:medium:codex:native","dominatedById":"openai:gpt-6-luna:high:codex:native","scoreDeficit":0.24,"costRatio":1.28,"costComparisonNote":null},{"configId":"openai:gpt-6-luna:xhigh:codex:native","dominatedById":"openai:gpt-6.1-terra:medium:codex:native","scoreDeficit":1.66,"costRatio":1.15,"costComparisonNote":null},{"configId":"openai:gpt-5.6-luna:high:codex:native","dominatedById":"openai:gpt-6.1-terra:medium:codex:native","scoreDeficit":11.73,"costRatio":1.56,"costComparisonNote":null},{"configId":"openai:gpt-6.1-terra:xhigh:codex:native","dominatedById":"openai:gpt-6.2-sol:low:codex:native","scoreDeficit":2.02,"costRatio":1.03,"costComparisonNote":null},{"configId":"openai:gpt-5.6-luna:xhigh:codex:native","dominatedById":"openai:gpt-6.2-sol:low:codex:native","scoreDeficit":8.83,"costRatio":1.18,"costComparisonNote":null},{"configId":"openai:gpt-6.1-sol:high:codex:native","dominatedById":"openai:gpt-6.2-sol:medium:codex:native","scoreDeficit":0.75,"costRatio":1.01,"costComparisonNote":null},{"configId":"openai:gpt-5.6-luna:max:codex:native","dominatedById":"openai:gpt-6.2-sol:medium:codex:native","scoreDeficit":17.14,"costRatio":1.17,"costComparisonNote":null},{"configId":"openai:gpt-6.1-sol:medium:codex:native","dominatedById":"openai:gpt-6.2-sol:medium:codex:native","scoreDeficit":4.16,"costRatio":1.23,"costComparisonNote":null},{"configId":"openai:gpt-6.1-sol:xhigh:codex:native","dominatedById":"openai:gpt-6.2-sol:max:codex:native","scoreDeficit":4,"costRatio":1.06,"costComparisonNote":null},{"configId":"openai:gpt-6.2-sol:xhigh:codex:native","dominatedById":"openai:gpt-6.2-sol:max:codex:native","scoreDeficit":1.58,"costRatio":1.12,"costComparisonNote":null},{"configId":"openai:gpt-5.6-terra:low:codex:native","dominatedById":"openai:gpt-6.2-sol:max:codex:native","scoreDeficit":51.4,"costRatio":1.41,"costComparisonNote":null},{"configId":"openai:gpt-5.6-terra:medium:codex:native","dominatedById":"openai:gpt-6.1-sol:max:codex:native","scoreDeficit":41.93,"costRatio":1.13,"costComparisonNote":null},{"configId":"openai:gpt-6.1-terra:max:codex:native","dominatedById":"openai:gpt-6.1-sol:max:codex:native","scoreDeficit":2.3,"costRatio":1.26,"costComparisonNote":null},{"configId":"openai:gpt-5.4-mini:xhigh:codex:native","dominatedById":"openai:gpt-6.1-sol:max:codex:native","scoreDeficit":47.5,"costRatio":1.27,"costComparisonNote":null},{"configId":"openai:gpt-6.2-astra:low:codex:native","dominatedById":"openai:gpt-6.1-sol:max:codex:native","scoreDeficit":23.96,"costRatio":1.27,"costComparisonNote":null},{"configId":"openai:gpt-6.2-astra:medium:codex:native","dominatedById":"openai:gpt-6.1-sol:max:codex:native","scoreDeficit":2.49,"costRatio":1.58,"costComparisonNote":null},{"configId":"openai:gpt-6-sol:low:codex:native","dominatedById":"openai:gpt-6.1-sol:max:codex:native","scoreDeficit":39.6,"costRatio":1.68,"costComparisonNote":null},{"configId":"openai:gpt-5.6-terra:high:codex:native","dominatedById":"openai:gpt-6.1-sol:max:codex:native","scoreDeficit":25.5,"costRatio":1.95,"costComparisonNote":null},{"configId":"openai:gpt-5.6-sol:low:codex:native","dominatedById":"openai:gpt-6.2-astra:high:codex:native","scoreDeficit":37.36,"costRatio":1,"costComparisonNote":null},{"configId":"openai:gpt-5.6-sol:xhigh:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":11.94,"costRatio":1,"costComparisonNote":null},{"configId":"openai:gpt-6-astra:low:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":8.92,"costRatio":1.02,"costComparisonNote":null},{"configId":"openai:gpt-5.6-sol:medium:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":19.14,"costRatio":1.07,"costComparisonNote":null},{"configId":"openai:gpt-6.2-astra:xhigh:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":0.05,"costRatio":1.09,"costComparisonNote":null},{"configId":"openai:gpt-6-sol:medium:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":21.83,"costRatio":1.09,"costComparisonNote":null},{"configId":"openai:gpt-6-sol:max:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":9.77,"costRatio":1.09,"costComparisonNote":null},{"configId":"openai:gpt-5.5:low:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":55.6,"costRatio":1.12,"costComparisonNote":null},{"configId":"openai:gpt-5.6-terra:max:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":18.66,"costRatio":1.13,"costComparisonNote":null},{"configId":"openai:gpt-6-astra:medium:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":6.99,"costRatio":1.15,"costComparisonNote":null},{"configId":"openai:gpt-6-sol:xhigh:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":12.97,"costRatio":1.24,"costComparisonNote":null},{"configId":"openai:gpt-5.6-sol:high:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":13.27,"costRatio":1.27,"costComparisonNote":null},{"configId":"openai:gpt-5.6-terra:xhigh:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":17.13,"costRatio":1.32,"costComparisonNote":null},{"configId":"openai:gpt-6-astra:high:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":4.7,"costRatio":1.36,"costComparisonNote":null},{"configId":"openai:gpt-6-sol:high:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":18.26,"costRatio":1.45,"costComparisonNote":null},{"configId":"openai:gpt-6-astra:xhigh:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":3.83,"costRatio":1.49,"costComparisonNote":null},{"configId":"openai:gpt-5.6-sol:max:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":13.93,"costRatio":1.5,"costComparisonNote":null},{"configId":"openai:gpt-5.5:medium:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":50.79,"costRatio":1.68,"costComparisonNote":null},{"configId":"openai:gpt-6-astra:max:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":0.97,"costRatio":1.97,"costComparisonNote":null},{"configId":"openai:gpt-5.5:high:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":36.19,"costRatio":2.61,"costComparisonNote":null},{"configId":"openai:gpt-5.5:xhigh:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":25.42,"costRatio":2.84,"costComparisonNote":null}],"verdicts":{"budgetId":"openai:gpt-6-luna:medium:codex:native","defaultId":"openai:gpt-6.2-sol:max:codex:native","maxId":"openai:gpt-6.2-astra:max:codex:native"}},{"provider":"openai","harnessRegime":"native","harnessFilter":"codex","frontier":[{"configId":"openai:gpt-6-luna:medium:codex:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9614,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"openai:gpt-6-luna:high:codex:native","zStepFromPrev":0.034,"pBetterThanPrev":0.5134,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.215,"marginalScorePerDollar":2.883,"costMultipleFromPrev":5.44},{"configId":"openai:gpt-6-luna:max:codex:native","zStepFromPrev":3.457,"pBetterThanPrev":0.9997,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.6336,"marginalScorePerDollar":473.474,"costMultipleFromPrev":1.55},{"configId":"openai:gpt-6.1-terra:low:codex:native","zStepFromPrev":0.029,"pBetterThanPrev":0.5115,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.4083,"marginalScorePerDollar":3.809,"costMultipleFromPrev":1.42},{"configId":"openai:gpt-6.1-terra:high:codex:native","zStepFromPrev":0.525,"pBetterThanPrev":0.7004,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.4756,"marginalScorePerDollar":186.001,"costMultipleFromPrev":1.13},{"configId":"openai:gpt-6.1-terra:medium:codex:native","zStepFromPrev":0.068,"pBetterThanPrev":0.5273,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.4349,"marginalScorePerDollar":31.119,"costMultipleFromPrev":1.08},{"configId":"openai:gpt-6.2-sol:low:codex:native","zStepFromPrev":0.895,"pBetterThanPrev":0.8145,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.3322,"marginalScorePerDollar":41.675,"costMultipleFromPrev":1.68},{"configId":"openai:gpt-6.1-sol:low:codex:native","zStepFromPrev":0.305,"pBetterThanPrev":0.6198,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.2501,"marginalScorePerDollar":17.466,"costMultipleFromPrev":1.31},{"configId":"openai:gpt-6.2-sol:medium:codex:native","zStepFromPrev":0.556,"pBetterThanPrev":0.7108,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.387,"marginalScorePerDollar":120.317,"costMultipleFromPrev":1.06},{"configId":"openai:gpt-6.2-sol:high:codex:native","zStepFromPrev":0.22,"pBetterThanPrev":0.5871,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.2996,"marginalScorePerDollar":7.214,"costMultipleFromPrev":1.25},{"configId":"openai:gpt-6.2-sol:max:codex:native","zStepFromPrev":0.616,"pBetterThanPrev":0.731,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.3789,"marginalScorePerDollar":606.412,"costMultipleFromPrev":1.01},{"configId":"openai:gpt-6.1-sol:max:codex:native","zStepFromPrev":0.617,"pBetterThanPrev":0.7315,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.3467,"marginalScorePerDollar":5.667,"costMultipleFromPrev":1.48},{"configId":"openai:gpt-6.2-astra:high:codex:native","zStepFromPrev":0.033,"pBetterThanPrev":0.513,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.316,"marginalScorePerDollar":0.183,"costMultipleFromPrev":2.24},{"configId":"openai:gpt-6.2-astra:max:codex:native","zStepFromPrev":0.091,"pBetterThanPrev":0.5362,"statisticallyTied":true,"imputationSensitive":true,"paretoProb":0.3192,"marginalScorePerDollar":2.177,"costMultipleFromPrev":1.19}],"pruned":[{"configId":"openai:gpt-5.6-luna:low:codex:native","dominatedById":"openai:gpt-6-luna:medium:codex:native","scoreDeficit":0,"costRatio":2.52,"costComparisonNote":null},{"configId":"openai:gpt-6-luna:low:codex:native","dominatedById":"openai:gpt-6-luna:medium:codex:native","scoreDeficit":0,"costRatio":2.91,"costComparisonNote":null},{"configId":"openai:gpt-5.6-luna:medium:codex:native","dominatedById":"openai:gpt-6-luna:high:codex:native","scoreDeficit":0.24,"costRatio":1.28,"costComparisonNote":null},{"configId":"openai:gpt-6-luna:xhigh:codex:native","dominatedById":"openai:gpt-6.1-terra:medium:codex:native","scoreDeficit":1.66,"costRatio":1.15,"costComparisonNote":null},{"configId":"openai:gpt-5.6-luna:high:codex:native","dominatedById":"openai:gpt-6.1-terra:medium:codex:native","scoreDeficit":11.73,"costRatio":1.56,"costComparisonNote":null},{"configId":"openai:gpt-6.1-terra:xhigh:codex:native","dominatedById":"openai:gpt-6.2-sol:low:codex:native","scoreDeficit":2.02,"costRatio":1.03,"costComparisonNote":null},{"configId":"openai:gpt-5.6-luna:xhigh:codex:native","dominatedById":"openai:gpt-6.2-sol:low:codex:native","scoreDeficit":8.83,"costRatio":1.18,"costComparisonNote":null},{"configId":"openai:gpt-6.1-sol:high:codex:native","dominatedById":"openai:gpt-6.2-sol:medium:codex:native","scoreDeficit":0.75,"costRatio":1.01,"costComparisonNote":null},{"configId":"openai:gpt-5.6-luna:max:codex:native","dominatedById":"openai:gpt-6.2-sol:medium:codex:native","scoreDeficit":17.14,"costRatio":1.17,"costComparisonNote":null},{"configId":"openai:gpt-6.1-sol:medium:codex:native","dominatedById":"openai:gpt-6.2-sol:medium:codex:native","scoreDeficit":4.16,"costRatio":1.23,"costComparisonNote":null},{"configId":"openai:gpt-6.1-sol:xhigh:codex:native","dominatedById":"openai:gpt-6.2-sol:max:codex:native","scoreDeficit":4,"costRatio":1.06,"costComparisonNote":null},{"configId":"openai:gpt-6.2-sol:xhigh:codex:native","dominatedById":"openai:gpt-6.2-sol:max:codex:native","scoreDeficit":1.58,"costRatio":1.12,"costComparisonNote":null},{"configId":"openai:gpt-5.6-terra:low:codex:native","dominatedById":"openai:gpt-6.2-sol:max:codex:native","scoreDeficit":51.4,"costRatio":1.41,"costComparisonNote":null},{"configId":"openai:gpt-5.6-terra:medium:codex:native","dominatedById":"openai:gpt-6.1-sol:max:codex:native","scoreDeficit":41.93,"costRatio":1.13,"costComparisonNote":null},{"configId":"openai:gpt-6.1-terra:max:codex:native","dominatedById":"openai:gpt-6.1-sol:max:codex:native","scoreDeficit":2.3,"costRatio":1.26,"costComparisonNote":null},{"configId":"openai:gpt-5.4-mini:xhigh:codex:native","dominatedById":"openai:gpt-6.1-sol:max:codex:native","scoreDeficit":47.5,"costRatio":1.27,"costComparisonNote":null},{"configId":"openai:gpt-6.2-astra:low:codex:native","dominatedById":"openai:gpt-6.1-sol:max:codex:native","scoreDeficit":23.96,"costRatio":1.27,"costComparisonNote":null},{"configId":"openai:gpt-6.2-astra:medium:codex:native","dominatedById":"openai:gpt-6.1-sol:max:codex:native","scoreDeficit":2.49,"costRatio":1.58,"costComparisonNote":null},{"configId":"openai:gpt-6-sol:low:codex:native","dominatedById":"openai:gpt-6.1-sol:max:codex:native","scoreDeficit":39.6,"costRatio":1.68,"costComparisonNote":null},{"configId":"openai:gpt-5.6-terra:high:codex:native","dominatedById":"openai:gpt-6.1-sol:max:codex:native","scoreDeficit":25.5,"costRatio":1.95,"costComparisonNote":null},{"configId":"openai:gpt-5.6-sol:low:codex:native","dominatedById":"openai:gpt-6.2-astra:high:codex:native","scoreDeficit":37.36,"costRatio":1,"costComparisonNote":null},{"configId":"openai:gpt-5.6-sol:xhigh:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":11.94,"costRatio":1,"costComparisonNote":null},{"configId":"openai:gpt-6-astra:low:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":8.92,"costRatio":1.02,"costComparisonNote":null},{"configId":"openai:gpt-5.6-sol:medium:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":19.14,"costRatio":1.07,"costComparisonNote":null},{"configId":"openai:gpt-6.2-astra:xhigh:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":0.05,"costRatio":1.09,"costComparisonNote":null},{"configId":"openai:gpt-6-sol:medium:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":21.83,"costRatio":1.09,"costComparisonNote":null},{"configId":"openai:gpt-6-sol:max:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":9.77,"costRatio":1.09,"costComparisonNote":null},{"configId":"openai:gpt-5.5:low:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":55.6,"costRatio":1.12,"costComparisonNote":null},{"configId":"openai:gpt-5.6-terra:max:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":18.66,"costRatio":1.13,"costComparisonNote":null},{"configId":"openai:gpt-6-astra:medium:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":6.99,"costRatio":1.15,"costComparisonNote":null},{"configId":"openai:gpt-6-sol:xhigh:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":12.97,"costRatio":1.24,"costComparisonNote":null},{"configId":"openai:gpt-5.6-sol:high:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":13.27,"costRatio":1.27,"costComparisonNote":null},{"configId":"openai:gpt-5.6-terra:xhigh:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":17.13,"costRatio":1.32,"costComparisonNote":null},{"configId":"openai:gpt-6-astra:high:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":4.7,"costRatio":1.36,"costComparisonNote":null},{"configId":"openai:gpt-6-sol:high:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":18.26,"costRatio":1.45,"costComparisonNote":null},{"configId":"openai:gpt-6-astra:xhigh:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":3.83,"costRatio":1.49,"costComparisonNote":null},{"configId":"openai:gpt-5.6-sol:max:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":13.93,"costRatio":1.5,"costComparisonNote":null},{"configId":"openai:gpt-5.5:medium:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":50.79,"costRatio":1.68,"costComparisonNote":null},{"configId":"openai:gpt-6-astra:max:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":0.97,"costRatio":1.97,"costComparisonNote":null},{"configId":"openai:gpt-5.5:high:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":36.19,"costRatio":2.61,"costComparisonNote":null},{"configId":"openai:gpt-5.5:xhigh:codex:native","dominatedById":"openai:gpt-6.2-astra:max:codex:native","scoreDeficit":25.42,"costRatio":2.84,"costComparisonNote":null}],"verdicts":{"budgetId":"openai:gpt-6-luna:medium:codex:native","defaultId":"openai:gpt-6.2-sol:max:codex:native","maxId":"openai:gpt-6.2-astra:max:codex:native"}},{"provider":"google","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"google:gemini-4-hydrogen:high:antigravity:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.8162,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"google:gemini-4-carbon:high:antigravity:native","zStepFromPrev":0.622,"pBetterThanPrev":0.7329,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.6061,"marginalScorePerDollar":4.622,"costMultipleFromPrev":1.73}],"pruned":[{"configId":"google:gemini-4-argon:high:antigravity:native","dominatedById":"google:gemini-4-hydrogen:high:antigravity:native","scoreDeficit":7.63,"costRatio":1.41,"costComparisonNote":null},{"configId":"google:gemini-4-hydrogen:xhigh:antigravity:native","dominatedById":"google:gemini-4-hydrogen:high:antigravity:native","scoreDeficit":0.12,"costRatio":1.54,"costComparisonNote":null},{"configId":"google:gemini-4-neon:xhigh:antigravity:native","dominatedById":"google:gemini-4-carbon:high:antigravity:native","scoreDeficit":2.35,"costRatio":1.42,"costComparisonNote":null},{"configId":"google:gemini-3.1-pro:high:antigravity:native","dominatedById":"google:gemini-4-carbon:high:antigravity:native","scoreDeficit":49.53,"costRatio":2.69,"costComparisonNote":null}],"verdicts":{"budgetId":"google:gemini-4-hydrogen:high:antigravity:native","defaultId":null,"maxId":"google:gemini-4-carbon:high:antigravity:native"}},{"provider":"google","harnessRegime":"native","harnessFilter":"antigravity","frontier":[{"configId":"google:gemini-4-hydrogen:high:antigravity:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.8162,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"google:gemini-4-carbon:high:antigravity:native","zStepFromPrev":0.622,"pBetterThanPrev":0.7329,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.6061,"marginalScorePerDollar":4.622,"costMultipleFromPrev":1.73}],"pruned":[{"configId":"google:gemini-4-argon:high:antigravity:native","dominatedById":"google:gemini-4-hydrogen:high:antigravity:native","scoreDeficit":7.63,"costRatio":1.41,"costComparisonNote":null},{"configId":"google:gemini-4-hydrogen:xhigh:antigravity:native","dominatedById":"google:gemini-4-hydrogen:high:antigravity:native","scoreDeficit":0.12,"costRatio":1.54,"costComparisonNote":null},{"configId":"google:gemini-4-neon:xhigh:antigravity:native","dominatedById":"google:gemini-4-carbon:high:antigravity:native","scoreDeficit":2.35,"costRatio":1.42,"costComparisonNote":null},{"configId":"google:gemini-3.1-pro:high:antigravity:native","dominatedById":"google:gemini-4-carbon:high:antigravity:native","scoreDeficit":49.53,"costRatio":2.69,"costComparisonNote":null}],"verdicts":{"budgetId":"google:gemini-4-hydrogen:high:antigravity:native","defaultId":null,"maxId":"google:gemini-4-carbon:high:antigravity:native"}},{"provider":"spacexai","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"spacexai:grok-4.5:high:grok-build:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9741,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"spacexai:grok-4.6:high:grok-build:native","zStepFromPrev":0.591,"pBetterThanPrev":0.7228,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.4428,"marginalScorePerDollar":1.9,"costMultipleFromPrev":1.72},{"configId":"spacexai:grok-4.7:xhigh:grok-build:native","zStepFromPrev":1.351,"pBetterThanPrev":0.9116,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9318,"marginalScorePerDollar":72.052,"costMultipleFromPrev":1.03}],"pruned":[],"verdicts":{"budgetId":"spacexai:grok-4.5:high:grok-build:native","defaultId":null,"maxId":"spacexai:grok-4.7:xhigh:grok-build:native"}},{"provider":"spacexai","harnessRegime":"native","harnessFilter":"grok-build","frontier":[{"configId":"spacexai:grok-4.5:high:grok-build:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9741,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"spacexai:grok-4.6:high:grok-build:native","zStepFromPrev":0.591,"pBetterThanPrev":0.7228,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.4428,"marginalScorePerDollar":1.9,"costMultipleFromPrev":1.72},{"configId":"spacexai:grok-4.7:xhigh:grok-build:native","zStepFromPrev":1.351,"pBetterThanPrev":0.9116,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9318,"marginalScorePerDollar":72.052,"costMultipleFromPrev":1.03}],"pruned":[],"verdicts":{"budgetId":"spacexai:grok-4.5:high:grok-build:native","defaultId":null,"maxId":"spacexai:grok-4.7:xhigh:grok-build:native"}},{"provider":"meta","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"meta:muse-ember:xhigh:muse-code:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.7798,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"meta:muse-spark-1.3:max:muse-code:native","zStepFromPrev":0.174,"pBetterThanPrev":0.569,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.3934,"marginalScorePerDollar":0.801,"costMultipleFromPrev":7.94},{"configId":"meta:muse-wonder:xhigh:muse-code:native","zStepFromPrev":0.894,"pBetterThanPrev":0.8142,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.6404,"marginalScorePerDollar":3.744,"costMultipleFromPrev":1.96}],"pruned":[{"configId":"meta:muse-ember:max:muse-code:native","dominatedById":"meta:muse-ember:xhigh:muse-code:native","scoreDeficit":1.08,"costRatio":1.3,"costComparisonNote":null},{"configId":"meta:muse-wonder:max:muse-code:native","dominatedById":"meta:muse-wonder:xhigh:muse-code:native","scoreDeficit":5.94,"costRatio":1.3,"costComparisonNote":null}],"verdicts":{"budgetId":"meta:muse-ember:xhigh:muse-code:native","defaultId":null,"maxId":"meta:muse-wonder:xhigh:muse-code:native"}},{"provider":"meta","harnessRegime":"native","harnessFilter":"muse-code","frontier":[{"configId":"meta:muse-ember:xhigh:muse-code:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.7798,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"meta:muse-spark-1.3:max:muse-code:native","zStepFromPrev":0.174,"pBetterThanPrev":0.569,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.3934,"marginalScorePerDollar":0.801,"costMultipleFromPrev":7.94},{"configId":"meta:muse-wonder:xhigh:muse-code:native","zStepFromPrev":0.894,"pBetterThanPrev":0.8142,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.6404,"marginalScorePerDollar":3.744,"costMultipleFromPrev":1.96}],"pruned":[{"configId":"meta:muse-ember:max:muse-code:native","dominatedById":"meta:muse-ember:xhigh:muse-code:native","scoreDeficit":1.08,"costRatio":1.3,"costComparisonNote":null},{"configId":"meta:muse-wonder:max:muse-code:native","dominatedById":"meta:muse-wonder:xhigh:muse-code:native","scoreDeficit":5.94,"costRatio":1.3,"costComparisonNote":null}],"verdicts":{"budgetId":"meta:muse-ember:xhigh:muse-code:native","defaultId":null,"maxId":"meta:muse-wonder:xhigh:muse-code:native"}},{"provider":"deepseek","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"deepseek:deepseek-v4.1-flash-1017:max:claude-code:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null}],"pruned":[{"configId":"deepseek:deepseek-v4-pro-0813:max:claude-code:native","dominatedById":"deepseek:deepseek-v4.1-flash-1017:max:claude-code:native","scoreDeficit":26.49,"costRatio":3.93,"costComparisonNote":null}],"verdicts":{"budgetId":"deepseek:deepseek-v4.1-flash-1017:max:claude-code:native","defaultId":null,"maxId":"deepseek:deepseek-v4.1-flash-1017:max:claude-code:native"}},{"provider":"deepseek","harnessRegime":"native","harnessFilter":"claude-code","frontier":[{"configId":"deepseek:deepseek-v4.1-flash-1017:max:claude-code:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null}],"pruned":[{"configId":"deepseek:deepseek-v4-pro-0813:max:claude-code:native","dominatedById":"deepseek:deepseek-v4.1-flash-1017:max:claude-code:native","scoreDeficit":26.49,"costRatio":3.93,"costComparisonNote":null}],"verdicts":{"budgetId":"deepseek:deepseek-v4.1-flash-1017:max:claude-code:native","defaultId":null,"maxId":"deepseek:deepseek-v4.1-flash-1017:max:claude-code:native"}},{"provider":"moonshot","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"moonshot:kimi-k3.1-pro:max:claude-code:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null}],"pruned":[{"configId":"moonshot:kimi-k3:max:claude-code:native","dominatedById":"moonshot:kimi-k3.1-pro:max:claude-code:native","scoreDeficit":27.33,"costRatio":1.77,"costComparisonNote":null}],"verdicts":{"budgetId":"moonshot:kimi-k3.1-pro:max:claude-code:native","defaultId":null,"maxId":"moonshot:kimi-k3.1-pro:max:claude-code:native"}},{"provider":"moonshot","harnessRegime":"native","harnessFilter":"claude-code","frontier":[{"configId":"moonshot:kimi-k3.1-pro:max:claude-code:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null}],"pruned":[{"configId":"moonshot:kimi-k3:max:claude-code:native","dominatedById":"moonshot:kimi-k3.1-pro:max:claude-code:native","scoreDeficit":27.33,"costRatio":1.77,"costComparisonNote":null}],"verdicts":{"budgetId":"moonshot:kimi-k3.1-pro:max:claude-code:native","defaultId":null,"maxId":"moonshot:kimi-k3.1-pro:max:claude-code:native"}},{"provider":"alibaba","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"alibaba:qwen3.8-max-0902:max:claude-code:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null}],"pruned":[],"verdicts":{"budgetId":"alibaba:qwen3.8-max-0902:max:claude-code:native","defaultId":null,"maxId":"alibaba:qwen3.8-max-0902:max:claude-code:native"}},{"provider":"alibaba","harnessRegime":"native","harnessFilter":"claude-code","frontier":[{"configId":"alibaba:qwen3.8-max-0902:max:claude-code:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":1,"marginalScorePerDollar":null,"costMultipleFromPrev":null}],"pruned":[],"verdicts":{"budgetId":"alibaba:qwen3.8-max-0902:max:claude-code:native","defaultId":null,"maxId":"alibaba:qwen3.8-max-0902:max:claude-code:native"}},{"provider":"zhipu","harnessRegime":"native","harnessFilter":"all","frontier":[{"configId":"zhipu:glm-5.3-flash:none:claude-code:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9999,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"zhipu:glm-5.3-flash:max:claude-code:native","zStepFromPrev":0.12,"pBetterThanPrev":0.5476,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.548,"marginalScorePerDollar":1.951,"costMultipleFromPrev":3.16},{"configId":"zhipu:glm-5.3:max:claude-code:native","zStepFromPrev":0.282,"pBetterThanPrev":0.6109,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.4983,"marginalScorePerDollar":0.591,"costMultipleFromPrev":7.28}],"pruned":[],"verdicts":{"budgetId":"zhipu:glm-5.3-flash:none:claude-code:native","defaultId":null,"maxId":"zhipu:glm-5.3:max:claude-code:native"}},{"provider":"zhipu","harnessRegime":"native","harnessFilter":"claude-code","frontier":[{"configId":"zhipu:glm-5.3-flash:none:claude-code:native","zStepFromPrev":null,"pBetterThanPrev":null,"statisticallyTied":false,"imputationSensitive":false,"paretoProb":0.9999,"marginalScorePerDollar":null,"costMultipleFromPrev":null},{"configId":"zhipu:glm-5.3-flash:max:claude-code:native","zStepFromPrev":0.12,"pBetterThanPrev":0.5476,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.548,"marginalScorePerDollar":1.951,"costMultipleFromPrev":3.16},{"configId":"zhipu:glm-5.3:max:claude-code:native","zStepFromPrev":0.282,"pBetterThanPrev":0.6109,"statisticallyTied":true,"imputationSensitive":false,"paretoProb":0.4983,"marginalScorePerDollar":0.591,"costMultipleFromPrev":7.28}],"pruned":[],"verdicts":{"budgetId":"zhipu:glm-5.3-flash:none:claude-code:native","defaultId":null,"maxId":"zhipu:glm-5.3:max:claude-code:native"}}]}},"harnessLeaderboard":{"sourceVersion":"1.1-1xhh2cu+0.1.11","upstreamGeneratedAt":"2026-11-21T21:58:14Z","sourceUrl":"https://frontierharness.org/leaderboard/v1.1/","slices":[{"sliceId":"_all","label":"All 8 models","org":null,"entries":[{"harnessId":"opencode","name":"OpenCode","vendor":"Anomaly","color":"#3E8E5E","version":"2.2.6","passRate":48.05,"ciLower":41.82,"ciUpper":54.28,"costPerTrialUsd":0.552258,"costPerPassUsd":1.14934,"passAll3Rate":30.56,"tokensPerTrial":10500000,"agentSecondsPerTrial":1775,"nScored":3564,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":44.14,"ciLower":36.37,"ciUpper":52.3,"ciIsUpperBound":true,"passAll3Rate":23.65,"costPerTrialUsd":0.8414,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":60.92,"ciLower":50.82,"ciUpper":70.09,"ciIsUpperBound":true,"passAll3Rate":46.12,"costPerTrialUsd":0.1163,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"zcode","name":"ZCode","vendor":"Zhipu","color":"#7c4d3f","version":"0.17.0","passRate":47.68,"ciLower":40,"ciUpper":55.36,"costPerTrialUsd":0.620963,"costPerPassUsd":1.302355,"passAll3Rate":30.03,"tokensPerTrial":2807122,"agentSecondsPerTrial":2122.9,"nScored":1240,"onFrontier":false,"dominatedById":"opencode","scoreDeficit":0.37,"costRatio":1.12,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":77.78,"ciLower":69.8,"ciUpper":85.75,"ciIsUpperBound":false,"passAll3Rate":65.56,"costPerTrialUsd":0.38277,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":20,"ciLower":9.95,"ciUpper":30.05,"ciIsUpperBound":false,"passAll3Rate":7.94,"costPerTrialUsd":1.37908,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"kimi-code","name":"Kimi Code","vendor":"Moonshot","color":"#a3305e","version":"2.3.1 / 2.2.0","passRate":47.04,"ciLower":41.7,"ciUpper":52.38,"costPerTrialUsd":0.639096,"costPerPassUsd":1.358622,"passAll3Rate":null,"tokensPerTrial":7003004,"agentSecondsPerTrial":2172.3,"nScored":4803,"onFrontier":false,"dominatedById":"opencode","scoreDeficit":1.01,"costRatio":1.16,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":43.58,"ciLower":35.7,"ciUpper":51.89,"ciIsUpperBound":true,"passAll3Rate":null,"costPerTrialUsd":1.011,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":76.11,"ciLower":67.54,"ciUpper":84.68,"ciIsUpperBound":false,"passAll3Rate":63.22,"costPerTrialUsd":0.441655,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":59.77,"ciLower":49.85,"ciUpper":68.87,"ciIsUpperBound":true,"passAll3Rate":null,"costPerTrialUsd":0.1417,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":17.94,"ciLower":8.36,"ciUpper":27.52,"ciIsUpperBound":false,"passAll3Rate":7.65,"costPerTrialUsd":1.199895,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"grok-build","name":"Grok Build","vendor":"SpaceXAI","color":"#1f2328","version":"1.0.67 / 1.0.77","passRate":46.22,"ciLower":41.04,"ciUpper":51.4,"costPerTrialUsd":0.684225,"costPerPassUsd":1.480366,"passAll3Rate":29.06,"tokensPerTrial":7167299,"agentSecondsPerTrial":2236.6,"nScored":4804,"onFrontier":false,"dominatedById":"opencode","scoreDeficit":1.83,"costRatio":1.24,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":43.54,"ciLower":36.06,"ciUpper":51.45,"ciIsUpperBound":true,"passAll3Rate":23.76,"costPerTrialUsd":1.064,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":76.11,"ciLower":68.09,"ciUpper":84.14,"ciIsUpperBound":false,"passAll3Rate":63.11,"costPerTrialUsd":0.439904,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":57.14,"ciLower":47.31,"ciUpper":66.37,"ciIsUpperBound":true,"passAll3Rate":40.95,"costPerTrialUsd":0.136,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":17.35,"ciLower":6.79,"ciUpper":27.92,"ciIsUpperBound":false,"passAll3Rate":8.82,"costPerTrialUsd":1.438322,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"hermes","name":"Hermes Agent","vendor":"Nous Research","color":"#d99a0b","version":"0.22.5 / 0.22.1","passRate":45.58,"ciLower":40.16,"ciUpper":51,"costPerTrialUsd":0.649169,"costPerPassUsd":1.424241,"passAll3Rate":27.1,"tokensPerTrial":7431822,"agentSecondsPerTrial":2136.8,"nScored":4804,"onFrontier":false,"dominatedById":"opencode","scoreDeficit":2.47,"costRatio":1.18,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":42.45,"ciLower":34.6,"ciUpper":50.8,"ciIsUpperBound":true,"passAll3Rate":21.73,"costPerTrialUsd":0.9869,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":75,"ciLower":66.46,"ciUpper":83.54,"ciIsUpperBound":false,"passAll3Rate":60.67,"costPerTrialUsd":0.440121,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":56.18,"ciLower":45.96,"ciUpper":65.88,"ciIsUpperBound":true,"passAll3Rate":38.65,"costPerTrialUsd":0.1379,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":20,"ciLower":9.88,"ciUpper":30.12,"ciIsUpperBound":false,"passAll3Rate":8.24,"costPerTrialUsd":1.338956,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-harness","name":"DeepSeek Harness","vendor":"DeepSeek","color":"#4d6bfe","version":"0.2.1-rc.3 / 0.2.2-rc.1","passRate":45.46,"ciLower":40.12,"ciUpper":50.8,"costPerTrialUsd":0.624479,"costPerPassUsd":1.373689,"passAll3Rate":29.71,"tokensPerTrial":7621120,"agentSecondsPerTrial":2032.5,"nScored":4803,"onFrontier":false,"dominatedById":"opencode","scoreDeficit":2.59,"costRatio":1.13,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":42.98,"ciLower":35.31,"ciUpper":51.12,"ciIsUpperBound":true,"passAll3Rate":24.77,"costPerTrialUsd":0.8995,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":69.3,"ciLower":59.77,"ciUpper":78.82,"ciIsUpperBound":false,"passAll3Rate":54.33,"costPerTrialUsd":0.434948,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":56.9,"ciLower":46.88,"ciUpper":66.34,"ciIsUpperBound":true,"passAll3Rate":42.67,"costPerTrialUsd":0.1387,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":17.35,"ciLower":5.71,"ciUpper":29,"ciIsUpperBound":false,"passAll3Rate":9.71,"costPerTrialUsd":1.365048,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"mini-swe-agent","name":"mini-SWE-agent","vendor":"SWE-agent","color":"#8b95a5","version":"2.4.6","passRate":45.05,"ciLower":39.71,"ciUpper":50.39,"costPerTrialUsd":0.666353,"costPerPassUsd":1.479141,"passAll3Rate":28.11,"tokensPerTrial":7263562,"agentSecondsPerTrial":2083.5,"nScored":4800,"onFrontier":false,"dominatedById":"opencode","scoreDeficit":3,"costRatio":1.21,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":40.99,"ciLower":33.25,"ciUpper":49.33,"ciIsUpperBound":true,"passAll3Rate":21.62,"costPerTrialUsd":0.9115,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":75.15,"ciLower":66.6,"ciUpper":83.69,"ciIsUpperBound":false,"passAll3Rate":62.33,"costPerTrialUsd":0.532307,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":58.81,"ciLower":48.84,"ciUpper":68.03,"ciIsUpperBound":true,"passAll3Rate":43.25,"costPerTrialUsd":0.1855,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":15.88,"ciLower":5.79,"ciUpper":25.97,"ciIsUpperBound":false,"passAll3Rate":5.59,"costPerTrialUsd":1.324097,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"claude-code","name":"Claude Code","vendor":"Anthropic","color":"#d97757","version":"2.1.308 / 2.1.300","passRate":45.05,"ciLower":39.82,"ciUpper":50.28,"costPerTrialUsd":0.756059,"costPerPassUsd":1.678266,"passAll3Rate":28.24,"tokensPerTrial":6971188,"agentSecondsPerTrial":2406.1,"nScored":4803,"onFrontier":false,"dominatedById":"opencode","scoreDeficit":3,"costRatio":1.37,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":42.45,"ciLower":34.92,"ciUpper":50.48,"ciIsUpperBound":true,"passAll3Rate":23.76,"costPerTrialUsd":1.034,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":74.37,"ciLower":66.06,"ciUpper":82.68,"ciIsUpperBound":false,"passAll3Rate":59.44,"costPerTrialUsd":0.48181,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":55.22,"ciLower":45.38,"ciUpper":64.62,"ciIsUpperBound":true,"passAll3Rate":39.66,"costPerTrialUsd":0.1083,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":17.94,"ciLower":7.75,"ciUpper":28.13,"ciIsUpperBound":false,"passAll3Rate":6.47,"costPerTrialUsd":2.048519,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-creator","name":"DeepSeek Harness: Creator","vendor":"DeepSeek","color":"#06b6d4","version":"0.2.1-rc.3","passRate":45.01,"ciLower":36.1,"ciUpper":53.92,"costPerTrialUsd":0.674148,"costPerPassUsd":1.497774,"passAll3Rate":27.57,"tokensPerTrial":3885702,"agentSecondsPerTrial":2258.8,"nScored":1240,"onFrontier":false,"dominatedById":"opencode","scoreDeficit":3.04,"costRatio":1.22,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":69.78,"ciLower":60.33,"ciUpper":79.23,"ciIsUpperBound":false,"passAll3Rate":57,"costPerTrialUsd":0.424464,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":20,"ciLower":8.15,"ciUpper":31.85,"ciIsUpperBound":false,"passAll3Rate":8.53,"costPerTrialUsd":1.473615,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-minimal","name":"DeepSeek Harness: Minimal","vendor":"DeepSeek","color":"#97a9ff","version":"0.2.1-rc.3","passRate":44.88,"ciLower":36.49,"ciUpper":53.27,"costPerTrialUsd":0.624665,"costPerPassUsd":1.391856,"passAll3Rate":27.93,"tokensPerTrial":3684530,"agentSecondsPerTrial":2172.6,"nScored":1239,"onFrontier":false,"dominatedById":"opencode","scoreDeficit":3.17,"costRatio":1.13,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":72.11,"ciLower":63.16,"ciUpper":81.06,"ciIsUpperBound":false,"passAll3Rate":56.89,"costPerTrialUsd":0.411222,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":18.63,"ciLower":7.56,"ciUpper":29.7,"ciIsUpperBound":false,"passAll3Rate":9.12,"costPerTrialUsd":1.318031,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"pi","name":"Pi","vendor":"Pi","color":"#e0409f","version":"1.4.0 / 1.2.0","passRate":44.76,"ciLower":39.47,"ciUpper":50.05,"costPerTrialUsd":0.520823,"costPerPassUsd":1.16359,"passAll3Rate":27.34,"tokensPerTrial":4745985,"agentSecondsPerTrial":1924.5,"nScored":4803,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":41.4,"ciLower":33.7,"ciUpper":49.68,"ciIsUpperBound":true,"passAll3Rate":21.51,"costPerTrialUsd":0.7905,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":73.67,"ciLower":64.94,"ciUpper":82.39,"ciIsUpperBound":false,"passAll3Rate":59.89,"costPerTrialUsd":0.334098,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":57.38,"ciLower":47.5,"ciUpper":66.64,"ciIsUpperBound":true,"passAll3Rate":41.81,"costPerTrialUsd":0.08018,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":15.59,"ciLower":6.7,"ciUpper":24.48,"ciIsUpperBound":false,"passAll3Rate":4.12,"costPerTrialUsd":1.18353,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"maka","name":"Maka","vendor":"Maka","color":"#9b7a2e","version":"0.2.2-dev.92.20261019","passRate":44.32,"ciLower":36.53,"ciUpper":52.11,"costPerTrialUsd":0.62133,"costPerPassUsd":1.401918,"passAll3Rate":26.02,"tokensPerTrial":2062255,"agentSecondsPerTrial":1992.1,"nScored":1240,"onFrontier":false,"dominatedById":"pi","scoreDeficit":0.44,"costRatio":1.19,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":73,"ciLower":64.14,"ciUpper":81.86,"ciIsUpperBound":false,"passAll3Rate":58.22,"costPerTrialUsd":0.427337,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":17.35,"ciLower":7.3,"ciUpper":27.41,"ciIsUpperBound":false,"passAll3Rate":5.59,"costPerTrialUsd":1.262525,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"opencode-1.19","name":"OpenCode 1.19.4","vendor":"OpenCode","color":"#9aab1a","version":"1.19.4","passRate":43.58,"ciLower":36.12,"ciUpper":51.04,"costPerTrialUsd":0.602074,"costPerPassUsd":1.381537,"passAll3Rate":25.82,"tokensPerTrial":2966144,"agentSecondsPerTrial":2153.2,"nScored":1239,"onFrontier":false,"dominatedById":"pi","scoreDeficit":1.18,"costRatio":1.16,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":73.11,"ciLower":64.04,"ciUpper":82.18,"ciIsUpperBound":false,"passAll3Rate":58.78,"costPerTrialUsd":0.393139,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":16.18,"ciLower":6.76,"ciUpper":25.59,"ciIsUpperBound":false,"passAll3Rate":5,"costPerTrialUsd":1.278865,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-code","name":"DeepSeek Harness: Code","vendor":"DeepSeek","color":"#1d8fe1","version":"0.2.1-rc.3","passRate":42.89,"ciLower":34.63,"ciUpper":51.15,"costPerTrialUsd":0.692223,"costPerPassUsd":1.61395,"passAll3Rate":25.1,"tokensPerTrial":3392020,"agentSecondsPerTrial":2211.8,"nScored":1238,"onFrontier":false,"dominatedById":"pi","scoreDeficit":1.87,"costRatio":1.33,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":69.89,"ciLower":60.68,"ciUpper":79.1,"ciIsUpperBound":false,"passAll3Rate":54.89,"costPerTrialUsd":0.419602,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":16.76,"ciLower":5.97,"ciUpper":27.56,"ciIsUpperBound":false,"passAll3Rate":5.88,"costPerTrialUsd":1.556116,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"codex","name":"Codex CLI","vendor":"OpenAI","color":"#10a37f","version":"0.164.0+baseten-catalog.1 / 0.162.1","passRate":42.87,"ciLower":37.73,"ciUpper":48.01,"costPerTrialUsd":0.546729,"costPerPassUsd":1.275318,"passAll3Rate":25.07,"tokensPerTrial":6545908,"agentSecondsPerTrial":1726.8,"nScored":4804,"onFrontier":false,"dominatedById":"pi","scoreDeficit":1.89,"costRatio":1.05,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":36.6,"ciLower":29.44,"ciUpper":44.65,"ciIsUpperBound":true,"passAll3Rate":16.55,"costPerTrialUsd":0.8353,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":77.33,"ciLower":69.16,"ciUpper":85.51,"ciIsUpperBound":false,"passAll3Rate":63,"costPerTrialUsd":0.369464,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":58.38,"ciLower":48.44,"ciUpper":67.61,"ciIsUpperBound":true,"passAll3Rate":42.1,"costPerTrialUsd":0.1225,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":18.24,"ciLower":8.08,"ciUpper":28.39,"ciIsUpperBound":false,"passAll3Rate":6.18,"costPerTrialUsd":1.110022,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"copilot-cli","name":"GitHub Copilot CLI","vendor":"GitHub","color":"#8957e5","version":"1.0.103","passRate":42.68,"ciLower":34.73,"ciUpper":50.63,"costPerTrialUsd":0.772788,"costPerPassUsd":1.810656,"passAll3Rate":27.59,"tokensPerTrial":2246380,"agentSecondsPerTrial":2206.6,"nScored":1240,"onFrontier":false,"dominatedById":"pi","scoreDeficit":2.08,"costRatio":1.48,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":72.78,"ciLower":63.94,"ciUpper":81.61,"ciIsUpperBound":false,"passAll3Rate":58.22,"costPerTrialUsd":0.47788,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":15,"ciLower":4.67,"ciUpper":25.33,"ciIsUpperBound":false,"passAll3Rate":7.94,"costPerTrialUsd":1.712233,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"minimax-code","name":"MiniMax Code","vendor":"MiniMax","color":"#f14564","version":"0.6.13","passRate":42.47,"ciLower":35.68,"ciUpper":49.26,"costPerTrialUsd":0.606881,"costPerPassUsd":1.428964,"passAll3Rate":24.07,"tokensPerTrial":2614514,"agentSecondsPerTrial":2063.6,"nScored":1239,"onFrontier":false,"dominatedById":"pi","scoreDeficit":2.29,"costRatio":1.17,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":72.74,"ciLower":64.01,"ciUpper":81.47,"ciIsUpperBound":false,"passAll3Rate":57.67,"costPerTrialUsd":0.434498,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":14.71,"ciLower":6.41,"ciUpper":23,"ciIsUpperBound":false,"passAll3Rate":2.94,"costPerTrialUsd":1.187903,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"antigravity-cli","name":"Antigravity CLI","vendor":"Google","color":"#2fb344","version":"1.4.6","passRate":42.3,"ciLower":35.15,"ciUpper":49.45,"costPerTrialUsd":0.845705,"costPerPassUsd":1.999303,"passAll3Rate":24.06,"tokensPerTrial":4857974,"agentSecondsPerTrial":2205.9,"nScored":1238,"onFrontier":false,"dominatedById":"pi","scoreDeficit":2.46,"costRatio":1.62,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":69.89,"ciLower":60.52,"ciUpper":79.26,"ciIsUpperBound":false,"passAll3Rate":54.11,"costPerTrialUsd":0.595436,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":15.88,"ciLower":7.1,"ciUpper":24.67,"ciIsUpperBound":false,"passAll3Rate":4.71,"costPerTrialUsd":1.681969,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"alma","name":"Alma","vendor":"Alma","color":"#de7cb7","version":"0.4.212","passRate":42.26,"ciLower":35.36,"ciUpper":49.16,"costPerTrialUsd":0.685389,"costPerPassUsd":1.621839,"passAll3Rate":null,"tokensPerTrial":2997819,"agentSecondsPerTrial":2047.6,"nScored":1238,"onFrontier":false,"dominatedById":"pi","scoreDeficit":2.5,"costRatio":1.32,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":78,"ciLower":69.52,"ciUpper":86.48,"ciIsUpperBound":false,"passAll3Rate":null,"costPerTrialUsd":0.423302,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":11.76,"ciLower":3.2,"ciUpper":20.33,"ciIsUpperBound":false,"passAll3Rate":4.41,"costPerTrialUsd":1.519995,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"fx","name":"fx","vendor":"Vercel Labs","color":"#000000","version":"0.0.16","passRate":41.92,"ciLower":36.19,"ciUpper":47.65,"costPerTrialUsd":0.631378,"costPerPassUsd":1.50615,"passAll3Rate":24.53,"tokensPerTrial":9075000,"agentSecondsPerTrial":1614,"nScored":3564,"onFrontier":false,"dominatedById":"pi","scoreDeficit":2.84,"costRatio":1.21,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":38.7,"ciLower":31.87,"ciUpper":46.29,"ciIsUpperBound":true,"passAll3Rate":18.47,"costPerTrialUsd":0.9554,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":53.4,"ciLower":44.02,"ciUpper":62.5,"ciIsUpperBound":true,"passAll3Rate":38.36,"costPerTrialUsd":0.1412,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"oh-my-pi","name":"Oh My Pi","vendor":"Oh My Pi","color":"#9b51e0","version":"18.12.6 / 18.18.3","passRate":41.69,"ciLower":36.33,"ciUpper":47.05,"costPerTrialUsd":0.654567,"costPerPassUsd":1.570082,"passAll3Rate":23.33,"tokensPerTrial":7868237,"agentSecondsPerTrial":1898.7,"nScored":4804,"onFrontier":false,"dominatedById":"pi","scoreDeficit":3.07,"costRatio":1.26,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":37.91,"ciLower":30.28,"ciUpper":46.34,"ciIsUpperBound":true,"passAll3Rate":16.78,"costPerTrialUsd":1.259,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":72.67,"ciLower":63.86,"ciUpper":81.47,"ciIsUpperBound":false,"passAll3Rate":58,"costPerTrialUsd":0.356378,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":53.02,"ciLower":42.99,"ciUpper":62.79,"ciIsUpperBound":true,"passAll3Rate":36.93,"costPerTrialUsd":0.1351,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":16.76,"ciLower":7.72,"ciUpper":25.81,"ciIsUpperBound":false,"passAll3Rate":4.71,"costPerTrialUsd":1.098744,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"cline-cli","name":"Cline CLI","vendor":"Cline","color":"#5b3df5","version":"3.0.73","passRate":40.7,"ciLower":33.97,"ciUpper":47.43,"costPerTrialUsd":0.652232,"costPerPassUsd":1.602536,"passAll3Rate":null,"tokensPerTrial":2816555,"agentSecondsPerTrial":2118.2,"nScored":1238,"onFrontier":false,"dominatedById":"pi","scoreDeficit":4.06,"costRatio":1.25,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":75.67,"ciLower":67.3,"ciUpper":84.03,"ciIsUpperBound":false,"passAll3Rate":null,"costPerTrialUsd":0.387358,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":10.59,"ciLower":2.32,"ciUpper":18.86,"ciIsUpperBound":false,"passAll3Rate":4.71,"costPerTrialUsd":1.4874,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"qwen-code","name":"Qwen Code","vendor":"Alibaba","color":"#EC4899","version":"0.25.2","passRate":39.3,"ciLower":33.51,"ciUpper":45.09,"costPerTrialUsd":0.550673,"costPerPassUsd":1.401204,"passAll3Rate":23.08,"tokensPerTrial":6870000,"agentSecondsPerTrial":1533,"nScored":3564,"onFrontier":false,"dominatedById":"pi","scoreDeficit":5.46,"costRatio":1.06,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":35.14,"ciLower":28.46,"ciUpper":42.82,"ciIsUpperBound":true,"passAll3Rate":16.89,"costPerTrialUsd":0.7989,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":52.68,"ciLower":42.94,"ciUpper":62.2,"ciIsUpperBound":true,"passAll3Rate":37.21,"costPerTrialUsd":0.1672,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"exo","name":"Exo","vendor":"Exo","color":"#0f766e","version":"825318306c0e","passRate":39,"ciLower":31.95,"ciUpper":46.05,"costPerTrialUsd":0.656449,"costPerPassUsd":1.683203,"passAll3Rate":21.5,"tokensPerTrial":3312094,"agentSecondsPerTrial":2043.7,"nScored":1240,"onFrontier":false,"dominatedById":"pi","scoreDeficit":5.76,"costRatio":1.26,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":64.67,"ciLower":55.27,"ciUpper":74.06,"ciIsUpperBound":false,"passAll3Rate":47,"costPerTrialUsd":0.463253,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":13.53,"ciLower":4.93,"ciUpper":22.13,"ciIsUpperBound":false,"passAll3Rate":4.41,"costPerTrialUsd":1.30275,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"openclaw","name":"OpenClaw","vendor":"OpenClaw","color":"#EF3E36","version":"2026.11.1","passRate":38.79,"ciLower":32.73,"ciUpper":44.85,"costPerTrialUsd":0.378278,"costPerPassUsd":0.975195,"passAll3Rate":22.27,"tokensPerTrial":7686000,"agentSecondsPerTrial":1535,"nScored":3564,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":36,"ciLower":28.87,"ciUpper":44.07,"ciIsUpperBound":true,"passAll3Rate":15.88,"costPerTrialUsd":0.5445,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":49.43,"ciLower":39.59,"ciUpper":59.31,"ciIsUpperBound":true,"passAll3Rate":36.78,"costPerTrialUsd":0.1204,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]}]},{"sliceId":"RecursiveSI/flint","label":"Flint","org":"Recursive","entries":[{"harnessId":"hermes","name":"Hermes Agent","vendor":"Nous Research","color":"#FF9D00","version":"0.22.1","passRate":23.4,"ciLower":18.26,"ciUpper":29.6,"costPerTrialUsd":0.2563,"costPerPassUsd":1.189,"passAll3Rate":12.85,"tokensPerTrial":17710000,"agentSecondsPerTrial":4135,"nScored":594,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":8.11,"ciLower":4.32,"ciUpper":14.69,"ciIsUpperBound":false,"passAll3Rate":2.7,"costPerTrialUsd":0.43,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":38.7,"ciLower":29.15,"ciUpper":49.2,"ciIsUpperBound":false,"passAll3Rate":22.99,"costPerTrialUsd":0.03474,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"opencode","name":"OpenCode","vendor":"Anomaly","color":"#3E8E5E","version":"2.2.6","passRate":23.1,"ciLower":18,"ciUpper":29.26,"costPerTrialUsd":0.209,"costPerPassUsd":0.9851,"passAll3Rate":14.24,"tokensPerTrial":13600000,"agentSecondsPerTrial":3317,"nScored":594,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":7.51,"ciLower":3.9,"ciUpper":13.95,"ciIsUpperBound":false,"passAll3Rate":0.9,"costPerTrialUsd":0.3545,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":38.7,"ciLower":29.15,"ciUpper":49.2,"ciIsUpperBound":false,"passAll3Rate":27.59,"costPerTrialUsd":0.02323,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"mini-swe-agent","name":"mini-swe-agent","vendor":"Princeton/Stanford","color":"#8C6D46","version":"2.4.6","passRate":21.5,"ciLower":16.52,"ciUpper":27.8,"costPerTrialUsd":0.2243,"costPerPassUsd":1.111,"passAll3Rate":8.95,"tokensPerTrial":10180000,"agentSecondsPerTrial":3647,"nScored":594,"onFrontier":false,"dominatedById":"opencode","scoreDeficit":1.6,"costRatio":1.07,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":10.81,"ciLower":6.29,"ciUpper":17.95,"ciIsUpperBound":false,"passAll3Rate":1.8,"costPerTrialUsd":0.3762,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":32.18,"ciLower":23.3,"ciUpper":42.57,"ciIsUpperBound":false,"passAll3Rate":16.09,"costPerTrialUsd":0.03066,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"deepseek-harness","name":"DeepSeek Harness","vendor":"DeepSeek","color":"#4D6BFE","version":"0.2.2-rc.1","passRate":20.68,"ciLower":15.8,"ciUpper":26.86,"costPerTrialUsd":0.2023,"costPerPassUsd":1.054,"passAll3Rate":9.65,"tokensPerTrial":17640000,"agentSecondsPerTrial":2541,"nScored":594,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":8.41,"ciLower":4.54,"ciUpper":15.06,"ciIsUpperBound":false,"passAll3Rate":0.9,"costPerTrialUsd":0.3165,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":32.95,"ciLower":23.98,"ciUpper":43.36,"ciIsUpperBound":false,"passAll3Rate":18.39,"costPerTrialUsd":0.05647,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"kimi-code","name":"Kimi Code","vendor":"Moonshot","color":"#0EA5E9","version":"2.2.0","passRate":20.49,"ciLower":15.62,"ciUpper":26.66,"costPerTrialUsd":0.2977,"costPerPassUsd":1.562,"passAll3Rate":null,"tokensPerTrial":11500000,"agentSecondsPerTrial":5165,"nScored":593,"onFrontier":false,"dominatedById":"deepseek-harness","scoreDeficit":0.19,"costRatio":1.47,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":8.41,"ciLower":4.54,"ciUpper":15.06,"ciIsUpperBound":false,"passAll3Rate":null,"costPerTrialUsd":0.5005,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":32.57,"ciLower":23.64,"ciUpper":42.97,"ciIsUpperBound":false,"passAll3Rate":null,"costPerTrialUsd":0.0397,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"oh-my-pi","name":"oh-my-pi","vendor":"Stencil Labs","color":"#F97316","version":"18.18.3","passRate":19.95,"ciLower":15.14,"ciUpper":26.1,"costPerTrialUsd":0.1947,"costPerPassUsd":1.052,"passAll3Rate":11.57,"tokensPerTrial":19110000,"agentSecondsPerTrial":2189,"nScored":594,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":8.11,"ciLower":4.32,"ciUpper":14.69,"ciIsUpperBound":false,"passAll3Rate":3.6,"costPerTrialUsd":0.3254,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":31.8,"ciLower":22.96,"ciUpper":42.18,"ciIsUpperBound":false,"passAll3Rate":19.54,"costPerTrialUsd":0.02805,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"pi","name":"Pi","vendor":"Earendil","color":"#A855F7","version":"1.2.0","passRate":18.01,"ciLower":13.43,"ciUpper":24.03,"costPerTrialUsd":0.09591,"costPerPassUsd":0.5755,"passAll3Rate":8.95,"tokensPerTrial":5264000,"agentSecondsPerTrial":2996,"nScored":594,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":6.91,"ciLower":3.49,"ciUpper":13.21,"ciIsUpperBound":false,"passAll3Rate":1.8,"costPerTrialUsd":0.1549,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":29.12,"ciLower":20.62,"ciUpper":39.39,"ciIsUpperBound":false,"passAll3Rate":16.09,"costPerTrialUsd":0.0206,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"codex","name":"Codex","vendor":"OpenAI","color":"#3B7CF6","version":"0.162.1","passRate":17.81,"ciLower":13.25,"ciUpper":23.68,"costPerTrialUsd":0.252,"costPerPassUsd":1.56,"passAll3Rate":6.9,"tokensPerTrial":12860000,"agentSecondsPerTrial":3632,"nScored":594,"onFrontier":false,"dominatedById":"pi","scoreDeficit":0.2,"costRatio":2.63,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":4.2,"ciLower":1.76,"ciUpper":9.71,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":0.4247,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":31.42,"ciLower":22.63,"ciUpper":41.78,"ciIsUpperBound":false,"passAll3Rate":13.79,"costPerTrialUsd":0.03172,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"claude-code","name":"Claude Code","vendor":"Anthropic","color":"#E07A4F","version":"2.1.300","passRate":16.88,"ciLower":12.44,"ciUpper":22.78,"costPerTrialUsd":0.1827,"costPerPassUsd":1.18,"passAll3Rate":6.2,"tokensPerTrial":9330000,"agentSecondsPerTrial":4413,"nScored":594,"onFrontier":false,"dominatedById":"pi","scoreDeficit":1.13,"costRatio":1.9,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":5.41,"ciLower":2.5,"ciUpper":11.29,"ciIsUpperBound":false,"passAll3Rate":0.9,"costPerTrialUsd":0.3043,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":28.35,"ciLower":19.95,"ciUpper":38.58,"ciIsUpperBound":false,"passAll3Rate":11.49,"costPerTrialUsd":0.02756,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"grok-build","name":"Grok Build","vendor":"SpaceXAI","color":"#111111","version":"1.0.77","passRate":16.85,"ciLower":12.4,"ciUpper":22.69,"costPerTrialUsd":0.2585,"costPerPassUsd":1.687,"passAll3Rate":7.47,"tokensPerTrial":11860000,"agentSecondsPerTrial":5003,"nScored":594,"onFrontier":false,"dominatedById":"pi","scoreDeficit":1.16,"costRatio":2.7,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":4.2,"ciLower":1.76,"ciUpper":9.71,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":0.4382,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":29.5,"ciLower":20.95,"ciUpper":39.79,"ciIsUpperBound":false,"passAll3Rate":14.94,"costPerTrialUsd":0.02913,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"openclaw","name":"OpenClaw","vendor":"OpenClaw","color":"#EF3E36","version":"2026.11.1","passRate":16.84,"ciLower":12.4,"ciUpper":22.75,"costPerTrialUsd":0.1523,"costPerPassUsd":0.9831,"passAll3Rate":9.2,"tokensPerTrial":10140000,"agentSecondsPerTrial":2847,"nScored":594,"onFrontier":false,"dominatedById":"pi","scoreDeficit":1.17,"costRatio":1.59,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":5.71,"ciLower":2.69,"ciUpper":11.68,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":0.2441,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":27.97,"ciLower":19.62,"ciUpper":38.18,"ciIsUpperBound":false,"passAll3Rate":18.39,"costPerTrialUsd":0.0351,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"qwen-code","name":"Qwen Code","vendor":"Alibaba","color":"#EC4899","version":"0.25.2","passRate":12.75,"ciLower":8.8,"ciUpper":18.11,"costPerTrialUsd":0.0594,"costPerPassUsd":0.5266,"passAll3Rate":5.17,"tokensPerTrial":2866000,"agentSecondsPerTrial":2082,"nScored":594,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":0.6,"ciLower":0.08,"ciUpper":4.43,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":0.08761,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":24.9,"ciLower":17.01,"ciUpper":34.92,"ciIsUpperBound":false,"passAll3Rate":10.34,"costPerTrialUsd":0.02341,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"fx","name":"fx","vendor":"Vercel Labs","color":"#000000","version":"0.0.16","passRate":9.54,"ciLower":6.13,"ciUpper":14.6,"costPerTrialUsd":0.1484,"costPerPassUsd":1.764,"passAll3Rate":2.87,"tokensPerTrial":5778000,"agentSecondsPerTrial":2642,"nScored":594,"onFrontier":false,"dominatedById":"qwen-code","scoreDeficit":3.21,"costRatio":2.5,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":0.3,"ciLower":0.02,"ciUpper":3.9,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":0.2498,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":18.77,"ciLower":11.96,"ciUpper":28.23,"ciIsUpperBound":false,"passAll3Rate":5.75,"costPerTrialUsd":0.01912,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]}]},{"sliceId":"deepseek-ai/DeepSeek-V4.1-Flash","label":"DS-V4.1-Flash","org":"DeepSeek","entries":[{"harnessId":"grok-build","name":"Grok Build","vendor":"SpaceXAI","color":"#111111","version":"1.0.77","passRate":70.92,"ciLower":64.15,"ciUpper":76.53,"costPerTrialUsd":0.1067,"costPerPassUsd":0.1518,"passAll3Rate":52.52,"tokensPerTrial":8284000,"agentSecondsPerTrial":910.8,"nScored":1188,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":65.02,"ciLower":55.78,"ciUpper":73.25,"ciIsUpperBound":true,"passAll3Rate":40.09,"costPerTrialUsd":0.1595,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":76.82,"ciLower":66.94,"ciUpper":84.43,"ciIsUpperBound":true,"passAll3Rate":64.94,"costPerTrialUsd":0.03945,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"opencode","name":"OpenCode","vendor":"Anomaly","color":"#3E8E5E","version":"2.2.6","passRate":70.79,"ciLower":64,"ciUpper":76.43,"costPerTrialUsd":0.1071,"costPerPassUsd":0.1529,"passAll3Rate":52.84,"tokensPerTrial":8826000,"agentSecondsPerTrial":798.8,"nScored":1188,"onFrontier":false,"dominatedById":"grok-build","scoreDeficit":0.13,"costRatio":1,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":65.92,"ciLower":56.71,"ciUpper":74.07,"ciIsUpperBound":true,"passAll3Rate":41.89,"costPerTrialUsd":0.1681,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":75.67,"ciLower":65.7,"ciUpper":83.48,"ciIsUpperBound":true,"passAll3Rate":63.79,"costPerTrialUsd":0.02937,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"deepseek-harness","name":"DeepSeek Harness","vendor":"DeepSeek","color":"#4D6BFE","version":"0.2.2-rc.1","passRate":70.57,"ciLower":63.74,"ciUpper":76.27,"costPerTrialUsd":0.1253,"costPerPassUsd":0.1783,"passAll3Rate":53.56,"tokensPerTrial":10040000,"agentSecondsPerTrial":1072,"nScored":1188,"onFrontier":false,"dominatedById":"grok-build","scoreDeficit":0.35,"costRatio":1.17,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":67.57,"ciLower":58.4,"ciUpper":75.56,"ciIsUpperBound":true,"passAll3Rate":45.05,"costPerTrialUsd":0.1934,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":73.56,"ciLower":63.44,"ciUpper":81.68,"ciIsUpperBound":true,"passAll3Rate":62.07,"costPerTrialUsd":0.03847,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"claude-code","name":"Claude Code","vendor":"Anthropic","color":"#E07A4F","version":"2.1.300","passRate":69.48,"ciLower":62.64,"ciUpper":75.24,"costPerTrialUsd":0.1073,"costPerPassUsd":0.1554,"passAll3Rate":50.89,"tokensPerTrial":6718000,"agentSecondsPerTrial":1161,"nScored":1188,"onFrontier":false,"dominatedById":"grok-build","scoreDeficit":1.44,"costRatio":1.01,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":65.02,"ciLower":55.78,"ciUpper":73.25,"ciIsUpperBound":true,"passAll3Rate":41.44,"costPerTrialUsd":0.1627,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":73.95,"ciLower":63.86,"ciUpper":82.02,"ciIsUpperBound":true,"passAll3Rate":60.34,"costPerTrialUsd":0.03646,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"kimi-code","name":"Kimi Code","vendor":"Moonshot","color":"#0EA5E9","version":"2.2.0","passRate":69.37,"ciLower":62.62,"ciUpper":75.05,"costPerTrialUsd":0.1064,"costPerPassUsd":0.1559,"passAll3Rate":49.59,"tokensPerTrial":8200000,"agentSecondsPerTrial":757.8,"nScored":1188,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":60.96,"ciLower":51.67,"ciUpper":69.52,"ciIsUpperBound":true,"passAll3Rate":34.23,"costPerTrialUsd":0.1569,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":77.78,"ciLower":67.98,"ciUpper":85.24,"ciIsUpperBound":true,"passAll3Rate":64.94,"costPerTrialUsd":0.04207,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"codex","name":"Codex","vendor":"OpenAI","color":"#3B7CF6","version":"0.162.1","passRate":68.85,"ciLower":62.02,"ciUpper":74.63,"costPerTrialUsd":0.09412,"costPerPassUsd":0.1381,"passAll3Rate":48.54,"tokensPerTrial":7476000,"agentSecondsPerTrial":851.7,"nScored":1188,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":62.61,"ciLower":53.34,"ciUpper":71.04,"ciIsUpperBound":true,"passAll3Rate":35.59,"costPerTrialUsd":0.1451,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":75.1,"ciLower":65.09,"ciUpper":83,"ciIsUpperBound":true,"passAll3Rate":61.49,"costPerTrialUsd":0.02911,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"mini-swe-agent","name":"mini-swe-agent","vendor":"Princeton/Stanford","color":"#8C6D46","version":"2.4.6","passRate":68.55,"ciLower":61.71,"ciUpper":74.35,"costPerTrialUsd":0.1275,"costPerPassUsd":0.1875,"passAll3Rate":48.94,"tokensPerTrial":9790000,"agentSecondsPerTrial":1227,"nScored":1188,"onFrontier":false,"dominatedById":"codex","scoreDeficit":0.3,"costRatio":1.35,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":62.76,"ciLower":53.5,"ciUpper":71.17,"ciIsUpperBound":true,"passAll3Rate":40.99,"costPerTrialUsd":0.1913,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":74.33,"ciLower":64.27,"ciUpper":82.34,"ciIsUpperBound":true,"passAll3Rate":56.9,"costPerTrialUsd":0.04608,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"fx","name":"fx","vendor":"Vercel Labs","color":"#000000","version":"0.0.16","passRate":67.64,"ciLower":60.82,"ciUpper":73.47,"costPerTrialUsd":0.1058,"costPerPassUsd":0.1589,"passAll3Rate":48.44,"tokensPerTrial":7861000,"agentSecondsPerTrial":855.1,"nScored":1188,"onFrontier":false,"dominatedById":"codex","scoreDeficit":1.21,"costRatio":1.12,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":59.61,"ciLower":50.31,"ciUpper":68.27,"ciIsUpperBound":true,"passAll3Rate":34.23,"costPerTrialUsd":0.1552,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":75.67,"ciLower":65.7,"ciUpper":83.48,"ciIsUpperBound":true,"passAll3Rate":62.64,"costPerTrialUsd":0.04278,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"pi","name":"Pi","vendor":"Earendil","color":"#A855F7","version":"1.2.0","passRate":65.94,"ciLower":59.06,"ciUpper":71.92,"costPerTrialUsd":0.09594,"costPerPassUsd":0.1473,"passAll3Rate":46.75,"tokensPerTrial":5642000,"agentSecondsPerTrial":1314,"nScored":1188,"onFrontier":false,"dominatedById":"codex","scoreDeficit":2.91,"costRatio":1.02,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":58.71,"ciLower":49.41,"ciUpper":67.42,"ciIsUpperBound":true,"passAll3Rate":36.04,"costPerTrialUsd":0.151,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":73.18,"ciLower":63.04,"ciUpper":81.36,"ciIsUpperBound":true,"passAll3Rate":57.47,"costPerTrialUsd":0.02564,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"qwen-code","name":"Qwen Code","vendor":"Alibaba","color":"#EC4899","version":"0.25.2","passRate":65.81,"ciLower":58.92,"ciUpper":71.81,"costPerTrialUsd":0.1064,"costPerPassUsd":0.164,"passAll3Rate":47,"tokensPerTrial":6918000,"agentSecondsPerTrial":739.9,"nScored":1188,"onFrontier":false,"dominatedById":"codex","scoreDeficit":3.04,"costRatio":1.13,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":59.01,"ciLower":49.71,"ciUpper":67.7,"ciIsUpperBound":true,"passAll3Rate":34.23,"costPerTrialUsd":0.1571,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":72.61,"ciLower":62.44,"ciUpper":80.88,"ciIsUpperBound":true,"passAll3Rate":59.77,"costPerTrialUsd":0.04173,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"openclaw","name":"OpenClaw","vendor":"OpenClaw","color":"#EF3E36","version":"2026.11.1","passRate":62.13,"ciLower":55.14,"ciUpper":68.44,"costPerTrialUsd":0.0876,"costPerPassUsd":0.1412,"passAll3Rate":42.9,"tokensPerTrial":6622000,"agentSecondsPerTrial":786.9,"nScored":1188,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":59.31,"ciLower":50.03,"ciUpper":67.96,"ciIsUpperBound":true,"passAll3Rate":30.63,"costPerTrialUsd":0.1241,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":64.94,"ciLower":54.47,"ciUpper":74.14,"ciIsUpperBound":true,"passAll3Rate":55.17,"costPerTrialUsd":0.04108,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"hermes","name":"Hermes Agent","vendor":"Nous Research","color":"#FF9D00","version":"0.22.1","passRate":61.93,"ciLower":54.95,"ciUpper":68.24,"costPerTrialUsd":0.1,"costPerPassUsd":0.1623,"passAll3Rate":43.12,"tokensPerTrial":7482000,"agentSecondsPerTrial":861.4,"nScored":1188,"onFrontier":false,"dominatedById":"openclaw","scoreDeficit":0.2,"costRatio":1.14,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":57.96,"ciLower":48.69,"ciUpper":66.7,"ciIsUpperBound":true,"passAll3Rate":37.39,"costPerTrialUsd":0.1505,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":65.9,"ciLower":55.46,"ciUpper":75,"ciIsUpperBound":true,"passAll3Rate":48.85,"costPerTrialUsd":0.03569,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"oh-my-pi","name":"oh-my-pi","vendor":"Stencil Labs","color":"#F97316","version":"18.18.3","passRate":61.47,"ciLower":54.49,"ciUpper":67.79,"costPerTrialUsd":0.1036,"costPerPassUsd":0.1701,"passAll3Rate":39.45,"tokensPerTrial":7460000,"agentSecondsPerTrial":831.1,"nScored":1188,"onFrontier":false,"dominatedById":"openclaw","scoreDeficit":0.66,"costRatio":1.18,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":56.46,"ciLower":47.17,"ciUpper":65.31,"ciIsUpperBound":true,"passAll3Rate":30.63,"costPerTrialUsd":0.1522,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":66.48,"ciLower":56.05,"ciUpper":75.51,"ciIsUpperBound":true,"passAll3Rate":48.28,"costPerTrialUsd":0.04159,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]}]},{"sliceId":"deepseek-v4.1-flash-1017","label":"DeepSeek-V4.1-Flash-1017","org":"DeepSeek","entries":[{"harnessId":"zcode","name":"ZCode","vendor":"Zhipu","color":"#7c4d3f","version":"0.17.0","passRate":68.95,"ciLower":59.15,"ciUpper":78.75,"costPerTrialUsd":0.152923,"costPerPassUsd":0.221783,"passAll3Rate":55.65,"tokensPerTrial":4658342,"agentSecondsPerTrial":2017,"nScored":248,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":84.44,"ciLower":75.46,"ciUpper":93.43,"ciIsUpperBound":false,"passAll3Rate":75,"costPerTrialUsd":0.087818,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":27.94,"ciLower":14.08,"ciUpper":41.8,"ciIsUpperBound":false,"passAll3Rate":4.41,"costPerTrialUsd":0.32526,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"claude-code","name":"Claude Code","vendor":"Anthropic","color":"#d97757","version":"2.1.308","passRate":68.15,"ciLower":58.31,"ciUpper":77.98,"costPerTrialUsd":0.194042,"costPerPassUsd":0.284748,"passAll3Rate":55.24,"tokensPerTrial":4094323,"agentSecondsPerTrial":2452.9,"nScored":248,"onFrontier":false,"dominatedById":"zcode","scoreDeficit":0.8,"costRatio":1.27,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":82.22,"ciLower":73.19,"ciUpper":91.25,"ciIsUpperBound":false,"passAll3Rate":71.11,"costPerTrialUsd":0.106042,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":30.88,"ciLower":13.8,"ciUpper":47.97,"ciIsUpperBound":false,"passAll3Rate":13.24,"costPerTrialUsd":0.426984,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"codex","name":"Codex CLI","vendor":"OpenAI","color":"#10a37f","version":"0.164.0+baseten-catalog.1","passRate":67.34,"ciLower":57.8,"ciUpper":76.87,"costPerTrialUsd":0.159625,"costPerPassUsd":0.237048,"passAll3Rate":50.4,"tokensPerTrial":4996383,"agentSecondsPerTrial":1836.7,"nScored":248,"onFrontier":false,"dominatedById":"zcode","scoreDeficit":1.61,"costRatio":1.04,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":79.44,"ciLower":70.46,"ciUpper":88.43,"ciIsUpperBound":false,"passAll3Rate":63.33,"costPerTrialUsd":0.071325,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":35.29,"ciLower":16.95,"ciUpper":53.64,"ciIsUpperBound":false,"passAll3Rate":16.18,"costPerTrialUsd":0.393362,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"kimi-code","name":"Kimi Code","vendor":"Moonshot","color":"#a3305e","version":"2.3.1","passRate":66.53,"ciLower":56.15,"ciUpper":76.92,"costPerTrialUsd":0.175439,"costPerPassUsd":0.26369,"passAll3Rate":56.45,"tokensPerTrial":4745137,"agentSecondsPerTrial":2090.1,"nScored":248,"onFrontier":false,"dominatedById":"zcode","scoreDeficit":2.42,"costRatio":1.15,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":83.33,"ciLower":74.39,"ciUpper":92.28,"ciIsUpperBound":false,"passAll3Rate":73.33,"costPerTrialUsd":0.111051,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":22.06,"ciLower":5.85,"ciUpper":38.27,"ciIsUpperBound":false,"passAll3Rate":11.76,"costPerTrialUsd":0.345878,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"grok-build","name":"Grok Build","vendor":"SpaceXAI","color":"#1f2328","version":"1.0.67","passRate":66.13,"ciLower":56.11,"ciUpper":76.15,"costPerTrialUsd":0.187716,"costPerPassUsd":0.283863,"passAll3Rate":52.02,"tokensPerTrial":4211695,"agentSecondsPerTrial":2405.4,"nScored":248,"onFrontier":false,"dominatedById":"zcode","scoreDeficit":2.82,"costRatio":1.23,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":81.11,"ciLower":72.42,"ciUpper":89.8,"ciIsUpperBound":false,"passAll3Rate":66.11,"costPerTrialUsd":0.093202,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":26.47,"ciLower":8.41,"ciUpper":44.53,"ciIsUpperBound":false,"passAll3Rate":14.71,"costPerTrialUsd":0.437899,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"mini-swe-agent","name":"mini-SWE-agent","vendor":"SWE-agent","color":"#8b95a5","version":"2.4.6","passRate":65.32,"ciLower":55.54,"ciUpper":75.1,"costPerTrialUsd":0.142295,"costPerPassUsd":0.217834,"passAll3Rate":48.39,"tokensPerTrial":4628359,"agentSecondsPerTrial":1794.2,"nScored":247,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":82.22,"ciLower":74.49,"ciUpper":89.95,"ciIsUpperBound":false,"passAll3Rate":63.89,"costPerTrialUsd":0.08245,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":20.59,"ciLower":5.3,"ciUpper":35.87,"ciIsUpperBound":false,"passAll3Rate":7.35,"costPerTrialUsd":0.30071,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-creator","name":"DeepSeek Harness: Creator","vendor":"DeepSeek","color":"#06b6d4","version":"0.2.1-rc.3","passRate":64.92,"ciLower":54.2,"ciUpper":75.64,"costPerTrialUsd":0.177554,"costPerPassUsd":0.273499,"passAll3Rate":54.84,"tokensPerTrial":5588588,"agentSecondsPerTrial":2577.4,"nScored":248,"onFrontier":false,"dominatedById":"mini-swe-agent","scoreDeficit":0.4,"costRatio":1.25,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":80,"ciLower":69.84,"ciUpper":90.16,"ciIsUpperBound":false,"passAll3Rate":71.11,"costPerTrialUsd":0.108755,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":25,"ciLower":7.17,"ciUpper":42.83,"ciIsUpperBound":false,"passAll3Rate":11.76,"costPerTrialUsd":0.35967,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-code","name":"DeepSeek Harness: Code","vendor":"DeepSeek","color":"#1d8fe1","version":"0.2.1-rc.3","passRate":64.11,"ciLower":53.51,"ciUpper":74.72,"costPerTrialUsd":0.155784,"costPerPassUsd":0.242984,"passAll3Rate":54.03,"tokensPerTrial":4079928,"agentSecondsPerTrial":2183.8,"nScored":248,"onFrontier":false,"dominatedById":"mini-swe-agent","scoreDeficit":1.21,"costRatio":1.09,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":81.67,"ciLower":72.14,"ciUpper":91.19,"ciIsUpperBound":false,"passAll3Rate":72.78,"costPerTrialUsd":0.106509,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":17.65,"ciLower":3.86,"ciUpper":31.43,"ciIsUpperBound":false,"passAll3Rate":4.41,"costPerTrialUsd":0.286217,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-minimal","name":"DeepSeek Harness: Minimal","vendor":"DeepSeek","color":"#97a9ff","version":"0.2.1-rc.3","passRate":63.71,"ciLower":53.59,"ciUpper":73.83,"costPerTrialUsd":0.162958,"costPerPassUsd":0.255782,"passAll3Rate":49.6,"tokensPerTrial":5128392,"agentSecondsPerTrial":2251.6,"nScored":248,"onFrontier":false,"dominatedById":"mini-swe-agent","scoreDeficit":1.61,"costRatio":1.15,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":77.78,"ciLower":68.28,"ciUpper":87.28,"ciIsUpperBound":false,"passAll3Rate":62.78,"costPerTrialUsd":0.097291,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":26.47,"ciLower":8.91,"ciUpper":44.03,"ciIsUpperBound":false,"passAll3Rate":14.71,"costPerTrialUsd":0.336783,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"cline-cli","name":"Cline CLI","vendor":"Cline","color":"#5b3df5","version":"3.0.73","passRate":63.31,"ciLower":52.81,"ciUpper":73.8,"costPerTrialUsd":0.544558,"costPerPassUsd":0.860194,"passAll3Rate":49.19,"tokensPerTrial":6772510,"agentSecondsPerTrial":2909.3,"nScored":248,"onFrontier":false,"dominatedById":"mini-swe-agent","scoreDeficit":2.01,"costRatio":3.83,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":81.67,"ciLower":72.94,"ciUpper":90.39,"ciIsUpperBound":false,"passAll3Rate":66.11,"costPerTrialUsd":0.105859,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":14.71,"ciLower":0.73,"ciUpper":28.68,"ciIsUpperBound":false,"passAll3Rate":4.41,"costPerTrialUsd":1.70582,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"alma","name":"Alma","vendor":"Alma","color":"#de7cb7","version":"0.4.212","passRate":62.5,"ciLower":52.31,"ciUpper":72.69,"costPerTrialUsd":0.155502,"costPerPassUsd":0.248803,"passAll3Rate":46.77,"tokensPerTrial":4364993,"agentSecondsPerTrial":2261.4,"nScored":248,"onFrontier":false,"dominatedById":"mini-swe-agent","scoreDeficit":2.82,"costRatio":1.09,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":81.67,"ciLower":73.51,"ciUpper":89.82,"ciIsUpperBound":false,"passAll3Rate":63.89,"costPerTrialUsd":0.103722,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":11.76,"ciLower":1.37,"ciUpper":22.16,"ciIsUpperBound":false,"passAll3Rate":1.47,"costPerTrialUsd":0.292569,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"hermes","name":"Hermes Agent","vendor":"Nous Research","color":"#d99a0b","version":"0.22.5","passRate":62.5,"ciLower":52.63,"ciUpper":72.37,"costPerTrialUsd":0.196438,"costPerPassUsd":0.314301,"passAll3Rate":45.56,"tokensPerTrial":4698216,"agentSecondsPerTrial":2129.1,"nScored":248,"onFrontier":false,"dominatedById":"mini-swe-agent","scoreDeficit":2.82,"costRatio":1.38,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":77.22,"ciLower":68.03,"ciUpper":86.41,"ciIsUpperBound":false,"passAll3Rate":59.44,"costPerTrialUsd":0.150504,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":23.53,"ciLower":8.11,"ciUpper":38.95,"ciIsUpperBound":false,"passAll3Rate":8.82,"costPerTrialUsd":0.318028,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-harness","name":"DeepSeek Harness","vendor":"DeepSeek","color":"#4d6bfe","version":"0.2.1-rc.3","passRate":62.1,"ciLower":51.48,"ciUpper":72.71,"costPerTrialUsd":0.181065,"costPerPassUsd":0.291585,"passAll3Rate":50,"tokensPerTrial":5731467,"agentSecondsPerTrial":2273,"nScored":248,"onFrontier":false,"dominatedById":"mini-swe-agent","scoreDeficit":3.22,"costRatio":1.27,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":78.33,"ciLower":68.78,"ciUpper":87.88,"ciIsUpperBound":false,"passAll3Rate":63.89,"costPerTrialUsd":0.104682,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":19.12,"ciLower":2.03,"ciUpper":36.2,"ciIsUpperBound":false,"passAll3Rate":13.24,"costPerTrialUsd":0.383254,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"opencode-1.19","name":"OpenCode 1.19.4","vendor":"OpenCode","color":"#9aab1a","version":"1.19.4","passRate":61.29,"ciLower":51.04,"ciUpper":71.54,"costPerTrialUsd":0.108376,"costPerPassUsd":0.176824,"passAll3Rate":47.58,"tokensPerTrial":3124179,"agentSecondsPerTrial":2107,"nScored":248,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":75,"ciLower":65.03,"ciUpper":84.97,"ciIsUpperBound":false,"passAll3Rate":60.56,"costPerTrialUsd":0.063254,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":25,"ciLower":7.68,"ciUpper":42.32,"ciIsUpperBound":false,"passAll3Rate":13.24,"costPerTrialUsd":0.227816,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"pi","name":"Pi","vendor":"Pi","color":"#e0409f","version":"1.4.0","passRate":61.29,"ciLower":51.11,"ciUpper":71.47,"costPerTrialUsd":0.128765,"costPerPassUsd":0.21009,"passAll3Rate":46.77,"tokensPerTrial":3727726,"agentSecondsPerTrial":2110.3,"nScored":248,"onFrontier":false,"dominatedById":"opencode-1.19","scoreDeficit":0,"costRatio":1.19,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":77.22,"ciLower":67.77,"ciUpper":86.67,"ciIsUpperBound":false,"passAll3Rate":61.67,"costPerTrialUsd":0.079962,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":19.12,"ciLower":4.85,"ciUpper":33.38,"ciIsUpperBound":false,"passAll3Rate":7.35,"costPerTrialUsd":0.257948,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"maka","name":"Maka","vendor":"Maka","color":"#9b7a2e","version":"0.2.2-dev.92.20261019","passRate":60.48,"ciLower":49.94,"ciUpper":71.03,"costPerTrialUsd":0.116825,"costPerPassUsd":0.193151,"passAll3Rate":47.18,"tokensPerTrial":2643230,"agentSecondsPerTrial":1794,"nScored":248,"onFrontier":false,"dominatedById":"opencode-1.19","scoreDeficit":0.81,"costRatio":1.08,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":76.67,"ciLower":66.89,"ciUpper":86.44,"ciIsUpperBound":false,"passAll3Rate":60.56,"costPerTrialUsd":0.086749,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":17.65,"ciLower":2.06,"ciUpper":33.24,"ciIsUpperBound":false,"passAll3Rate":11.76,"costPerTrialUsd":0.19644,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"minimax-code","name":"MiniMax Code","vendor":"MiniMax","color":"#f14564","version":"0.6.13","passRate":60.08,"ciLower":49.85,"ciUpper":70.31,"costPerTrialUsd":0.153668,"costPerPassUsd":0.25577,"passAll3Rate":45.16,"tokensPerTrial":3735118,"agentSecondsPerTrial":2040.4,"nScored":248,"onFrontier":false,"dominatedById":"opencode-1.19","scoreDeficit":1.21,"costRatio":1.42,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":75.56,"ciLower":65.4,"ciUpper":85.71,"ciIsUpperBound":false,"passAll3Rate":61.11,"costPerTrialUsd":0.101788,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":19.12,"ciLower":6.85,"ciUpper":31.39,"ciIsUpperBound":false,"passAll3Rate":2.94,"costPerTrialUsd":0.290999,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"antigravity-cli","name":"Antigravity CLI","vendor":"Google","color":"#2fb344","version":"1.4.6","passRate":58.87,"ciLower":48.42,"ciUpper":69.33,"costPerTrialUsd":0.171634,"costPerPassUsd":0.291543,"passAll3Rate":45.16,"tokensPerTrial":6181885,"agentSecondsPerTrial":2202.9,"nScored":248,"onFrontier":false,"dominatedById":"opencode-1.19","scoreDeficit":2.42,"costRatio":1.58,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":75,"ciLower":64.67,"ciUpper":85.33,"ciIsUpperBound":false,"passAll3Rate":61.11,"costPerTrialUsd":0.120859,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":16.18,"ciLower":4.34,"ciUpper":28.02,"ciIsUpperBound":false,"passAll3Rate":2.94,"costPerTrialUsd":0.306036,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"copilot-cli","name":"GitHub Copilot CLI","vendor":"GitHub","color":"#8957e5","version":"1.0.103","passRate":58.47,"ciLower":47.6,"ciUpper":69.33,"costPerTrialUsd":0.287451,"costPerPassUsd":0.49164,"passAll3Rate":47.18,"tokensPerTrial":2830991,"agentSecondsPerTrial":2288.1,"nScored":248,"onFrontier":false,"dominatedById":"opencode-1.19","scoreDeficit":2.82,"costRatio":2.65,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":74.44,"ciLower":64.06,"ciUpper":84.83,"ciIsUpperBound":false,"passAll3Rate":60.56,"costPerTrialUsd":0.166088,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":16.18,"ciLower":0,"ciUpper":32.42,"ciIsUpperBound":false,"passAll3Rate":11.76,"costPerTrialUsd":0.608705,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"oh-my-pi","name":"Oh My Pi","vendor":"Oh My Pi","color":"#9b51e0","version":"18.12.6","passRate":57.66,"ciLower":47.3,"ciUpper":68.02,"costPerTrialUsd":0.143936,"costPerPassUsd":0.249623,"passAll3Rate":43.15,"tokensPerTrial":3732228,"agentSecondsPerTrial":2113.4,"nScored":248,"onFrontier":false,"dominatedById":"opencode-1.19","scoreDeficit":3.63,"costRatio":1.33,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":73.89,"ciLower":63.92,"ciUpper":83.86,"ciIsUpperBound":false,"passAll3Rate":57.22,"costPerTrialUsd":0.087807,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":14.71,"ciLower":2.06,"ciUpper":27.35,"ciIsUpperBound":false,"passAll3Rate":5.88,"costPerTrialUsd":0.292515,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"exo","name":"Exo","vendor":"Exo","color":"#0f766e","version":"825318306c0e","passRate":54.03,"ciLower":43.27,"ciUpper":64.79,"costPerTrialUsd":0.181438,"costPerPassUsd":0.335796,"passAll3Rate":41.13,"tokensPerTrial":4691300,"agentSecondsPerTrial":2502.5,"nScored":248,"onFrontier":false,"dominatedById":"opencode-1.19","scoreDeficit":7.26,"costRatio":1.67,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":68.89,"ciLower":57.64,"ciUpper":80.14,"ciIsUpperBound":false,"passAll3Rate":54.44,"costPerTrialUsd":0.116613,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":14.71,"ciLower":1.38,"ciUpper":28.03,"ciIsUpperBound":false,"passAll3Rate":5.88,"costPerTrialUsd":0.353032,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]}]},{"sliceId":"glm-5.3","label":"GLM-5.3","org":"Zhipu","entries":[{"harnessId":"zcode","name":"ZCode","vendor":"Zhipu","color":"#7c4d3f","version":"0.17.0","passRate":58.87,"ciLower":48.42,"ciUpper":69.33,"costPerTrialUsd":0.87269,"costPerPassUsd":1.482378,"passAll3Rate":45.16,"tokensPerTrial":2494185,"agentSecondsPerTrial":2508.6,"nScored":248,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":70,"ciLower":59.26,"ciUpper":80.74,"ciIsUpperBound":false,"passAll3Rate":54.44,"costPerTrialUsd":0.580227,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":29.41,"ciLower":9.6,"ciUpper":49.22,"ciIsUpperBound":false,"passAll3Rate":20.59,"costPerTrialUsd":1.646856,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"kimi-code","name":"Kimi Code","vendor":"Moonshot","color":"#a3305e","version":"2.3.1","passRate":58.06,"ciLower":47.09,"ciUpper":69.03,"costPerTrialUsd":0.898636,"costPerPassUsd":1.547651,"passAll3Rate":46.77,"tokensPerTrial":2819153,"agentSecondsPerTrial":2160.4,"nScored":248,"onFrontier":false,"dominatedById":"zcode","scoreDeficit":0.81,"costRatio":1.03,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":70.56,"ciLower":58.67,"ciUpper":82.45,"ciIsUpperBound":false,"passAll3Rate":60.56,"costPerTrialUsd":0.606472,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":25,"ciLower":8.19,"ciUpper":41.81,"ciIsUpperBound":false,"passAll3Rate":10.29,"costPerTrialUsd":1.672012,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"codex","name":"Codex CLI","vendor":"OpenAI","color":"#10a37f","version":"0.164.0+baseten-catalog.1","passRate":57.66,"ciLower":47.3,"ciUpper":68.02,"costPerTrialUsd":0.462957,"costPerPassUsd":0.80289,"passAll3Rate":42.74,"tokensPerTrial":1859311,"agentSecondsPerTrial":1245.1,"nScored":248,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":71.67,"ciLower":60.81,"ciUpper":82.52,"ciIsUpperBound":false,"passAll3Rate":56.67,"costPerTrialUsd":0.379173,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":20.59,"ciLower":7.15,"ciUpper":34.03,"ciIsUpperBound":false,"passAll3Rate":5.88,"costPerTrialUsd":0.684738,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"hermes","name":"Hermes Agent","vendor":"Nous Research","color":"#d99a0b","version":"0.22.5","passRate":57.66,"ciLower":47.24,"ciUpper":68.09,"costPerTrialUsd":0.956106,"costPerPassUsd":1.658142,"passAll3Rate":42.34,"tokensPerTrial":2676012,"agentSecondsPerTrial":2571.8,"nScored":248,"onFrontier":false,"dominatedById":"codex","scoreDeficit":0,"costRatio":2.07,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":68.33,"ciLower":57.06,"ciUpper":79.61,"ciIsUpperBound":false,"passAll3Rate":52.78,"costPerTrialUsd":0.649139,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":29.41,"ciLower":11.47,"ciUpper":47.35,"ciIsUpperBound":false,"passAll3Rate":14.71,"costPerTrialUsd":1.768665,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"maka","name":"Maka","vendor":"Maka","color":"#9b7a2e","version":"0.2.2-dev.92.20261019","passRate":56.45,"ciLower":46.07,"ciUpper":66.84,"costPerTrialUsd":0.834123,"costPerPassUsd":1.477589,"passAll3Rate":38.71,"tokensPerTrial":2088874,"agentSecondsPerTrial":2348,"nScored":248,"onFrontier":false,"dominatedById":"codex","scoreDeficit":1.21,"costRatio":1.8,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":66.67,"ciLower":55.12,"ciUpper":78.22,"ciIsUpperBound":false,"passAll3Rate":50,"costPerTrialUsd":0.552741,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":29.41,"ciLower":12.48,"ciUpper":46.34,"ciIsUpperBound":false,"passAll3Rate":8.82,"costPerTrialUsd":1.578957,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"pi","name":"Pi","vendor":"Pi","color":"#e0409f","version":"1.4.0","passRate":56.45,"ciLower":45.53,"ciUpper":67.37,"costPerTrialUsd":0.875411,"costPerPassUsd":1.550728,"passAll3Rate":45.56,"tokensPerTrial":3158164,"agentSecondsPerTrial":2188.9,"nScored":248,"onFrontier":false,"dominatedById":"codex","scoreDeficit":1.21,"costRatio":1.89,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":68.33,"ciLower":56.23,"ciUpper":80.44,"ciIsUpperBound":false,"passAll3Rate":59.44,"costPerTrialUsd":0.517992,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":25,"ciLower":8.73,"ciUpper":41.27,"ciIsUpperBound":false,"passAll3Rate":8.82,"costPerTrialUsd":1.821521,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"alma","name":"Alma","vendor":"Alma","color":"#de7cb7","version":"0.4.212","passRate":56.45,"ciLower":45.36,"ciUpper":67.55,"costPerTrialUsd":0.914434,"costPerPassUsd":1.619855,"passAll3Rate":45.56,"tokensPerTrial":2929447,"agentSecondsPerTrial":2321.1,"nScored":248,"onFrontier":false,"dominatedById":"codex","scoreDeficit":1.21,"costRatio":1.98,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":70.56,"ciLower":59.08,"ciUpper":82.03,"ciIsUpperBound":false,"passAll3Rate":58.89,"costPerTrialUsd":0.62939,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":19.12,"ciLower":2.03,"ciUpper":36.2,"ciIsUpperBound":false,"passAll3Rate":10.29,"costPerTrialUsd":1.668962,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"grok-build","name":"Grok Build","vendor":"SpaceXAI","color":"#1f2328","version":"1.0.67","passRate":56.45,"ciLower":45.59,"ciUpper":67.31,"costPerTrialUsd":1.020599,"costPerPassUsd":1.807918,"passAll3Rate":43.55,"tokensPerTrial":3273748,"agentSecondsPerTrial":2386.5,"nScored":248,"onFrontier":false,"dominatedById":"codex","scoreDeficit":1.21,"costRatio":2.2,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":66.67,"ciLower":54.81,"ciUpper":78.53,"ciIsUpperBound":false,"passAll3Rate":52.78,"costPerTrialUsd":0.641411,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":29.41,"ciLower":10.05,"ciUpper":48.77,"ciIsUpperBound":false,"passAll3Rate":19.12,"costPerTrialUsd":2.024331,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-minimal","name":"DeepSeek Harness: Minimal","vendor":"DeepSeek","color":"#97a9ff","version":"0.2.1-rc.3","passRate":55.24,"ciLower":45.28,"ciUpper":65.2,"costPerTrialUsd":1.038873,"costPerPassUsd":1.880588,"passAll3Rate":36.29,"tokensPerTrial":3205210,"agentSecondsPerTrial":2662.6,"nScored":248,"onFrontier":false,"dominatedById":"codex","scoreDeficit":2.42,"costRatio":2.24,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":66.67,"ciLower":56.34,"ciUpper":77,"ciIsUpperBound":false,"passAll3Rate":45,"costPerTrialUsd":0.597754,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":25,"ciLower":7.68,"ciUpper":42.32,"ciIsUpperBound":false,"passAll3Rate":13.24,"costPerTrialUsd":2.206541,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"copilot-cli","name":"GitHub Copilot CLI","vendor":"GitHub","color":"#8957e5","version":"1.0.103","passRate":54.84,"ciLower":44.1,"ciUpper":65.58,"costPerTrialUsd":0.964417,"costPerPassUsd":1.758643,"passAll3Rate":41.53,"tokensPerTrial":2224309,"agentSecondsPerTrial":2336.4,"nScored":248,"onFrontier":false,"dominatedById":"codex","scoreDeficit":2.82,"costRatio":2.08,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":67.78,"ciLower":56.37,"ciUpper":79.18,"ciIsUpperBound":false,"passAll3Rate":52.78,"costPerTrialUsd":0.668156,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":20.59,"ciLower":4.19,"ciUpper":36.99,"ciIsUpperBound":false,"passAll3Rate":11.76,"costPerTrialUsd":1.748636,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"oh-my-pi","name":"Oh My Pi","vendor":"Oh My Pi","color":"#9b51e0","version":"18.12.6","passRate":54.44,"ciLower":44.2,"ciUpper":64.67,"costPerTrialUsd":0.945118,"costPerPassUsd":1.736217,"passAll3Rate":37.1,"tokensPerTrial":2695889,"agentSecondsPerTrial":2385.6,"nScored":248,"onFrontier":false,"dominatedById":"codex","scoreDeficit":3.22,"costRatio":2.04,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":65,"ciLower":53.83,"ciUpper":76.17,"ciIsUpperBound":false,"passAll3Rate":47.22,"costPerTrialUsd":0.630518,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":26.47,"ciLower":9.42,"ciUpper":43.52,"ciIsUpperBound":false,"passAll3Rate":10.29,"costPerTrialUsd":1.777882,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"claude-code","name":"Claude Code","vendor":"Anthropic","color":"#d97757","version":"2.1.308","passRate":54.44,"ciLower":43.89,"ciUpper":64.98,"costPerTrialUsd":0.962894,"costPerPassUsd":1.768872,"passAll3Rate":38.71,"tokensPerTrial":3040401,"agentSecondsPerTrial":2566.4,"nScored":248,"onFrontier":false,"dominatedById":"codex","scoreDeficit":3.22,"costRatio":2.08,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":63.89,"ciLower":52.07,"ciUpper":75.7,"ciIsUpperBound":false,"passAll3Rate":48.89,"costPerTrialUsd":0.608314,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":29.41,"ciLower":11.47,"ciUpper":47.35,"ciIsUpperBound":false,"passAll3Rate":11.76,"costPerTrialUsd":1.90149,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"opencode-1.19","name":"OpenCode 1.19.4","vendor":"OpenCode","color":"#9aab1a","version":"1.19.4","passRate":54.03,"ciLower":43.57,"ciUpper":64.49,"costPerTrialUsd":0.875122,"costPerPassUsd":1.619629,"passAll3Rate":37.9,"tokensPerTrial":3036314,"agentSecondsPerTrial":2469,"nScored":248,"onFrontier":false,"dominatedById":"codex","scoreDeficit":3.63,"costRatio":1.89,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":66.11,"ciLower":54.44,"ciUpper":77.78,"ciIsUpperBound":false,"passAll3Rate":51.11,"costPerTrialUsd":0.555646,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":22.06,"ciLower":8.2,"ciUpper":35.92,"ciIsUpperBound":false,"passAll3Rate":2.94,"costPerTrialUsd":1.720794,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-harness","name":"DeepSeek Harness","vendor":"DeepSeek","color":"#4d6bfe","version":"0.2.1-rc.3","passRate":54.03,"ciLower":43.75,"ciUpper":64.31,"costPerTrialUsd":1.080798,"costPerPassUsd":2.000283,"passAll3Rate":36.69,"tokensPerTrial":3816712,"agentSecondsPerTrial":2499.2,"nScored":248,"onFrontier":false,"dominatedById":"codex","scoreDeficit":3.63,"costRatio":2.33,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":62.78,"ciLower":51.48,"ciUpper":74.07,"ciIsUpperBound":false,"passAll3Rate":44.44,"costPerTrialUsd":0.709366,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":30.88,"ciLower":11.84,"ciUpper":49.92,"ciIsUpperBound":false,"passAll3Rate":16.18,"costPerTrialUsd":2.064002,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"mini-swe-agent","name":"mini-SWE-agent","vendor":"SWE-agent","color":"#8b95a5","version":"2.4.6","passRate":54.03,"ciLower":43.69,"ciUpper":64.37,"costPerTrialUsd":1.097822,"costPerPassUsd":2.03179,"passAll3Rate":39.11,"tokensPerTrial":3988731,"agentSecondsPerTrial":2543.4,"nScored":248,"onFrontier":false,"dominatedById":"codex","scoreDeficit":3.63,"costRatio":2.37,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":66.67,"ciLower":55.55,"ciUpper":77.79,"ciIsUpperBound":false,"passAll3Rate":51.11,"costPerTrialUsd":0.798432,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":20.59,"ciLower":5.89,"ciUpper":35.29,"ciIsUpperBound":false,"passAll3Rate":7.35,"costPerTrialUsd":1.890325,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"antigravity-cli","name":"Antigravity CLI","vendor":"Google","color":"#2fb344","version":"1.4.6","passRate":53.63,"ciLower":43.5,"ciUpper":63.76,"costPerTrialUsd":1.317693,"costPerPassUsd":2.457052,"passAll3Rate":35.08,"tokensPerTrial":5560682,"agentSecondsPerTrial":2696.3,"nScored":248,"onFrontier":false,"dominatedById":"codex","scoreDeficit":4.03,"costRatio":2.85,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":62.78,"ciLower":51.59,"ciUpper":73.97,"ciIsUpperBound":false,"passAll3Rate":42.78,"costPerTrialUsd":0.878531,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":29.41,"ciLower":11.47,"ciUpper":47.35,"ciIsUpperBound":false,"passAll3Rate":14.71,"costPerTrialUsd":2.480179,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"cline-cli","name":"Cline CLI","vendor":"Cline","color":"#5b3df5","version":"3.0.73","passRate":53.23,"ciLower":42.51,"ciUpper":63.94,"costPerTrialUsd":0.824401,"costPerPassUsd":1.548875,"passAll3Rate":40.73,"tokensPerTrial":1768547,"agentSecondsPerTrial":2170.9,"nScored":248,"onFrontier":false,"dominatedById":"codex","scoreDeficit":4.43,"costRatio":1.78,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":67.78,"ciLower":56.37,"ciUpper":79.18,"ciIsUpperBound":false,"passAll3Rate":53.89,"costPerTrialUsd":0.506991,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":14.71,"ciLower":2.06,"ciUpper":27.35,"ciIsUpperBound":false,"passAll3Rate":5.88,"costPerTrialUsd":1.664604,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-creator","name":"DeepSeek Harness: Creator","vendor":"DeepSeek","color":"#06b6d4","version":"0.2.1-rc.3","passRate":51.61,"ciLower":40.87,"ciUpper":62.35,"costPerTrialUsd":1.132707,"costPerPassUsd":2.19462,"passAll3Rate":37.5,"tokensPerTrial":3865876,"agentSecondsPerTrial":2566.2,"nScored":248,"onFrontier":false,"dominatedById":"codex","scoreDeficit":6.05,"costRatio":2.45,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":60.56,"ciLower":48.54,"ciUpper":72.57,"ciIsUpperBound":false,"passAll3Rate":45.56,"costPerTrialUsd":0.648943,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":27.94,"ciLower":8.74,"ciUpper":47.14,"ciIsUpperBound":false,"passAll3Rate":16.18,"costPerTrialUsd":2.41326,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"opencode","name":"OpenCode","vendor":"OpenCode","color":"#facc15","version":"2.2.7","passRate":51.21,"ciLower":40.61,"ciUpper":61.81,"costPerTrialUsd":0.779635,"costPerPassUsd":1.522437,"passAll3Rate":35.08,"tokensPerTrial":2288424,"agentSecondsPerTrial":1790.6,"nScored":246,"onFrontier":false,"dominatedById":"codex","scoreDeficit":6.45,"costRatio":1.68,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":62.22,"ciLower":50.71,"ciUpper":73.73,"ciIsUpperBound":false,"passAll3Rate":43.33,"costPerTrialUsd":0.521756,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":22.06,"ciLower":4.29,"ciUpper":39.83,"ciIsUpperBound":false,"passAll3Rate":13.24,"costPerTrialUsd":1.462254,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"minimax-code","name":"MiniMax Code","vendor":"MiniMax","color":"#f14564","version":"0.6.13","passRate":50.4,"ciLower":40.05,"ciUpper":60.76,"costPerTrialUsd":0.888668,"costPerPassUsd":1.763117,"passAll3Rate":33.47,"tokensPerTrial":2473014,"agentSecondsPerTrial":2335,"nScored":248,"onFrontier":false,"dominatedById":"codex","scoreDeficit":7.26,"costRatio":1.92,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":62.22,"ciLower":50.82,"ciUpper":73.63,"ciIsUpperBound":false,"passAll3Rate":43.33,"costPerTrialUsd":0.652397,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":19.12,"ciLower":4.24,"ciUpper":33.99,"ciIsUpperBound":false,"passAll3Rate":7.35,"costPerTrialUsd":1.51409,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"exo","name":"Exo","vendor":"Exo","color":"#0f766e","version":"825318306c0e","passRate":50,"ciLower":39.98,"ciUpper":60.02,"costPerTrialUsd":1.358697,"costPerPassUsd":2.717394,"passAll3Rate":31.05,"tokensPerTrial":4298023,"agentSecondsPerTrial":2596.2,"nScored":248,"onFrontier":false,"dominatedById":"codex","scoreDeficit":7.66,"costRatio":2.93,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":60.56,"ciLower":49.27,"ciUpper":71.84,"ciIsUpperBound":false,"passAll3Rate":41.11,"costPerTrialUsd":0.884801,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":22.06,"ciLower":7.58,"ciUpper":36.54,"ciIsUpperBound":false,"passAll3Rate":4.41,"costPerTrialUsd":2.613128,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-code","name":"DeepSeek Harness: Code","vendor":"DeepSeek","color":"#1d8fe1","version":"0.2.1-rc.3","passRate":48.39,"ciLower":38.32,"ciUpper":58.46,"costPerTrialUsd":1.014306,"costPerPassUsd":2.096232,"passAll3Rate":29.84,"tokensPerTrial":2876310,"agentSecondsPerTrial":2719.1,"nScored":248,"onFrontier":false,"dominatedById":"codex","scoreDeficit":9.27,"costRatio":2.19,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":56.67,"ciLower":45.08,"ciUpper":68.26,"ciIsUpperBound":false,"passAll3Rate":36.67,"costPerTrialUsd":0.603305,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":26.47,"ciLower":9.94,"ciUpper":43,"ciIsUpperBound":false,"passAll3Rate":11.76,"costPerTrialUsd":2.102249,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]}]},{"sliceId":"glm-5.3-flash","label":"GLM-5.3-Flash","org":"Zhipu","entries":[{"harnessId":"fx","name":"fx","vendor":"Vercel Labs","color":"#000000","version":"0.0.16","passRate":57.42,"ciLower":50.46,"ciUpper":64.38,"costPerTrialUsd":0.149406,"costPerPassUsd":0.260199,"passAll3Rate":35.48,"tokensPerTrial":3316000,"agentSecondsPerTrial":1118,"nScored":1188,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":57.06,"ciLower":47.79,"ciUpper":65.86,"ciIsUpperBound":true,"passAll3Rate":31.53,"costPerTrialUsd":0.2149,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":67.43,"ciLower":57.03,"ciUpper":76.35,"ciIsUpperBound":true,"passAll3Rate":48.28,"costPerTrialUsd":0.03716,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"grok-build","name":"Grok Build","vendor":"SpaceXAI","color":"#1f2328","version":"1.0.67 / 1.0.77","passRate":57.21,"ciLower":51.25,"ciUpper":63.17,"costPerTrialUsd":0.157733,"costPerPassUsd":0.275709,"passAll3Rate":36.74,"tokensPerTrial":3099078,"agentSecondsPerTrial":1843.3,"nScored":1436,"onFrontier":false,"dominatedById":"fx","scoreDeficit":0.21,"costRatio":1.06,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":59.31,"ciLower":50.03,"ciUpper":67.96,"ciIsUpperBound":true,"passAll3Rate":35.14,"costPerTrialUsd":0.2208,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":70,"ciLower":58.71,"ciUpper":81.29,"ciIsUpperBound":false,"passAll3Rate":56.67,"costPerTrialUsd":0.129431,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":66.28,"ciLower":55.86,"ciUpper":75.33,"ciIsUpperBound":true,"passAll3Rate":47.13,"costPerTrialUsd":0.03535,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":17.65,"ciLower":2.06,"ciUpper":33.24,"ciIsUpperBound":false,"passAll3Rate":8.82,"costPerTrialUsd":0.308774,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"mini-swe-agent","name":"mini-SWE-agent","vendor":"SWE-agent","color":"#8b95a5","version":"2.4.6","passRate":57.18,"ciLower":51.14,"ciUpper":63.22,"costPerTrialUsd":0.215806,"costPerPassUsd":0.377415,"passAll3Rate":35.51,"tokensPerTrial":5177737,"agentSecondsPerTrial":1935,"nScored":1436,"onFrontier":false,"dominatedById":"fx","scoreDeficit":0.24,"costRatio":1.44,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":58.86,"ciLower":49.57,"ciUpper":67.55,"ciIsUpperBound":true,"passAll3Rate":31.98,"costPerTrialUsd":0.2919,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":68.89,"ciLower":56.91,"ciUpper":80.87,"ciIsUpperBound":false,"passAll3Rate":57.22,"costPerTrialUsd":0.181843,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":65.52,"ciLower":55.07,"ciUpper":74.66,"ciIsUpperBound":true,"passAll3Rate":47.13,"costPerTrialUsd":0.05286,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":22.06,"ciLower":4.79,"ciUpper":39.32,"ciIsUpperBound":false,"passAll3Rate":13.24,"costPerTrialUsd":0.427926,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"pi","name":"Pi","vendor":"Pi","color":"#e0409f","version":"1.4.0 / 1.2.0","passRate":57.02,"ciLower":51.07,"ciUpper":62.97,"costPerTrialUsd":0.112898,"costPerPassUsd":0.197997,"passAll3Rate":34.85,"tokensPerTrial":2295080,"agentSecondsPerTrial":1533.8,"nScored":1436,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":57.06,"ciLower":47.81,"ciUpper":65.85,"ciIsUpperBound":true,"passAll3Rate":31.08,"costPerTrialUsd":0.1559,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":70.56,"ciLower":58.98,"ciUpper":82.14,"ciIsUpperBound":false,"passAll3Rate":57.78,"costPerTrialUsd":0.100567,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":67.05,"ciLower":56.66,"ciUpper":75.99,"ciIsUpperBound":true,"passAll3Rate":50,"costPerTrialUsd":0.02474,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":25,"ciLower":10.44,"ciUpper":39.56,"ciIsUpperBound":false,"passAll3Rate":4.41,"costPerTrialUsd":0.206738,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"opencode","name":"OpenCode","vendor":"OpenCode","color":"#facc15","version":"2.2.7 / 2.2.6","passRate":56.84,"ciLower":50.84,"ciUpper":62.84,"costPerTrialUsd":0.133859,"costPerPassUsd":0.235501,"passAll3Rate":null,"tokensPerTrial":2949145,"agentSecondsPerTrial":1576.5,"nScored":1432,"onFrontier":false,"dominatedById":"pi","scoreDeficit":0.18,"costRatio":1.19,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":57.51,"ciLower":48.23,"ciUpper":66.29,"ciIsUpperBound":true,"passAll3Rate":31.98,"costPerTrialUsd":0.2105,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":67.78,"ciLower":55.55,"ciUpper":80,"ciIsUpperBound":false,"passAll3Rate":57.78,"costPerTrialUsd":0.090009,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":67.24,"ciLower":56.87,"ciUpper":76.15,"ciIsUpperBound":true,"passAll3Rate":48.28,"costPerTrialUsd":0.03146,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":22.06,"ciLower":5.85,"ciUpper":38.27,"ciIsUpperBound":false,"passAll3Rate":null,"costPerTrialUsd":0.257562,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"claude-code","name":"Claude Code","vendor":"Anthropic","color":"#d97757","version":"2.1.308 / 2.1.300","passRate":56.78,"ciLower":50.81,"ciUpper":62.75,"costPerTrialUsd":0.173272,"costPerPassUsd":0.305164,"passAll3Rate":35.69,"tokensPerTrial":3369949,"agentSecondsPerTrial":1942.3,"nScored":1436,"onFrontier":false,"dominatedById":"pi","scoreDeficit":0.24,"costRatio":1.53,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":58.56,"ciLower":49.28,"ciUpper":67.27,"ciIsUpperBound":true,"passAll3Rate":34.68,"costPerTrialUsd":0.2239,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":70.56,"ciLower":59.85,"ciUpper":81.26,"ciIsUpperBound":false,"passAll3Rate":53.89,"costPerTrialUsd":0.144095,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":64.94,"ciLower":54.47,"ciUpper":74.14,"ciIsUpperBound":true,"passAll3Rate":45.4,"costPerTrialUsd":0.04173,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":20.59,"ciLower":5.3,"ciUpper":35.87,"ciIsUpperBound":false,"passAll3Rate":7.35,"costPerTrialUsd":0.379488,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"zcode","name":"ZCode","vendor":"Zhipu","color":"#7c4d3f","version":"0.17.0","passRate":56.59,"ciLower":43,"ciUpper":70.18,"costPerTrialUsd":0.145499,"costPerPassUsd":0.257111,"passAll3Rate":36.81,"tokensPerTrial":2683343,"agentSecondsPerTrial":2148.7,"nScored":248,"onFrontier":false,"dominatedById":"pi","scoreDeficit":0.43,"costRatio":1.29,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":73.33,"ciLower":62.61,"ciUpper":84.06,"ciIsUpperBound":false,"passAll3Rate":59.44,"costPerTrialUsd":0.109353,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":25,"ciLower":8.19,"ciUpper":41.81,"ciIsUpperBound":false,"passAll3Rate":13.24,"costPerTrialUsd":0.290079,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"kimi-code","name":"Kimi Code","vendor":"Moonshot","color":"#a3305e","version":"2.3.1 / 2.2.0","passRate":56.52,"ciLower":50.44,"ciUpper":62.6,"costPerTrialUsd":0.201533,"costPerPassUsd":0.356569,"passAll3Rate":34.35,"tokensPerTrial":4275927,"agentSecondsPerTrial":1891.2,"nScored":1436,"onFrontier":false,"dominatedById":"pi","scoreDeficit":0.5,"costRatio":1.79,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":55.71,"ciLower":46.47,"ciUpper":64.56,"ciIsUpperBound":true,"passAll3Rate":30.18,"costPerTrialUsd":0.2779,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":70,"ciLower":58.92,"ciUpper":81.08,"ciIsUpperBound":false,"passAll3Rate":54.44,"costPerTrialUsd":0.139277,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":65.13,"ciLower":54.67,"ciUpper":74.31,"ciIsUpperBound":true,"passAll3Rate":46.55,"costPerTrialUsd":0.0542,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":32.35,"ciLower":14.64,"ciUpper":50.06,"ciIsUpperBound":false,"passAll3Rate":14.71,"costPerTrialUsd":0.456091,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-harness","name":"DeepSeek Harness","vendor":"DeepSeek","color":"#4d6bfe","version":"0.2.1-rc.3 / 0.2.2-rc.1","passRate":56.43,"ciLower":50.29,"ciUpper":62.57,"costPerTrialUsd":0.226656,"costPerPassUsd":0.401659,"passAll3Rate":38.63,"tokensPerTrial":4392039,"agentSecondsPerTrial":2113.9,"nScored":1436,"onFrontier":false,"dominatedById":"pi","scoreDeficit":0.59,"costRatio":2.01,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":59.31,"ciLower":50.03,"ciUpper":67.96,"ciIsUpperBound":true,"passAll3Rate":38.74,"costPerTrialUsd":0.3395,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":66.11,"ciLower":55.19,"ciUpper":77.03,"ciIsUpperBound":false,"passAll3Rate":48.33,"costPerTrialUsd":0.165785,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":61.3,"ciLower":50.81,"ciUpper":70.83,"ciIsUpperBound":true,"passAll3Rate":44.25,"costPerTrialUsd":0.04192,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":25,"ciLower":5.29,"ciUpper":44.71,"ciIsUpperBound":false,"passAll3Rate":19.12,"costPerTrialUsd":0.458476,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"hermes","name":"Hermes Agent","vendor":"Nous Research","color":"#d99a0b","version":"0.22.5 / 0.22.1","passRate":56.27,"ciLower":50.15,"ciUpper":62.39,"costPerTrialUsd":0.173115,"costPerPassUsd":0.307651,"passAll3Rate":31.8,"tokensPerTrial":3543227,"agentSecondsPerTrial":1779.1,"nScored":1436,"onFrontier":false,"dominatedById":"pi","scoreDeficit":0.75,"costRatio":1.53,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":55.41,"ciLower":46.15,"ciUpper":64.31,"ciIsUpperBound":true,"passAll3Rate":26.13,"costPerTrialUsd":0.2346,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":72.22,"ciLower":62.22,"ciUpper":82.22,"ciIsUpperBound":false,"passAll3Rate":54.44,"costPerTrialUsd":0.129786,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":64.94,"ciLower":54.5,"ciUpper":74.12,"ciIsUpperBound":true,"passAll3Rate":44.83,"costPerTrialUsd":0.04265,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":30.88,"ciLower":12.31,"ciUpper":49.45,"ciIsUpperBound":false,"passAll3Rate":16.18,"costPerTrialUsd":0.384135,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"copilot-cli","name":"GitHub Copilot CLI","vendor":"GitHub","color":"#8957e5","version":"1.0.103","passRate":55.24,"ciLower":41.6,"ciUpper":68.88,"costPerTrialUsd":0.162975,"costPerPassUsd":0.295031,"passAll3Rate":32.99,"tokensPerTrial":1659031,"agentSecondsPerTrial":2346.3,"nScored":248,"onFrontier":false,"dominatedById":"pi","scoreDeficit":1.78,"costRatio":1.44,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":72.22,"ciLower":62.47,"ciUpper":81.98,"ciIsUpperBound":false,"passAll3Rate":53.89,"costPerTrialUsd":0.127504,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":23.53,"ciLower":6.48,"ciUpper":40.58,"ciIsUpperBound":false,"passAll3Rate":10.29,"costPerTrialUsd":0.311644,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-creator","name":"DeepSeek Harness: Creator","vendor":"DeepSeek","color":"#06b6d4","version":"0.2.1-rc.3","passRate":54.6,"ciLower":40.82,"ciUpper":68.38,"costPerTrialUsd":0.230342,"costPerPassUsd":0.421872,"passAll3Rate":32.73,"tokensPerTrial":5413443,"agentSecondsPerTrial":2519,"nScored":248,"onFrontier":false,"dominatedById":"pi","scoreDeficit":2.42,"costRatio":2.04,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":64.44,"ciLower":53.05,"ciUpper":75.83,"ciIsUpperBound":false,"passAll3Rate":47.22,"costPerTrialUsd":0.157881,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":26.47,"ciLower":9.42,"ciUpper":43.52,"ciIsUpperBound":false,"passAll3Rate":13.24,"costPerTrialUsd":0.499568,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"minimax-code","name":"MiniMax Code","vendor":"MiniMax","color":"#f14564","version":"0.6.13","passRate":54.37,"ciLower":41.94,"ciUpper":66.8,"costPerTrialUsd":0.163273,"costPerPassUsd":0.3003,"passAll3Rate":27.96,"tokensPerTrial":2702780,"agentSecondsPerTrial":2422.1,"nScored":248,"onFrontier":false,"dominatedById":"pi","scoreDeficit":2.65,"costRatio":1.45,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":66.67,"ciLower":55.44,"ciUpper":77.9,"ciIsUpperBound":false,"passAll3Rate":50.56,"costPerTrialUsd":0.128466,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":25,"ciLower":10.44,"ciUpper":39.56,"ciIsUpperBound":false,"passAll3Rate":4.41,"costPerTrialUsd":0.310284,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"antigravity-cli","name":"Antigravity CLI","vendor":"Google","color":"#2fb344","version":"1.4.6","passRate":54.37,"ciLower":41.68,"ciUpper":67.06,"costPerTrialUsd":0.226483,"costPerPassUsd":0.416559,"passAll3Rate":28.2,"tokensPerTrial":5257180,"agentSecondsPerTrial":2435.9,"nScored":248,"onFrontier":false,"dominatedById":"pi","scoreDeficit":2.65,"costRatio":2.01,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":66.67,"ciLower":55.99,"ciUpper":77.34,"ciIsUpperBound":false,"passAll3Rate":48.33,"costPerTrialUsd":0.175864,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":25,"ciLower":9.85,"ciUpper":40.15,"ciIsUpperBound":false,"passAll3Rate":5.88,"costPerTrialUsd":0.43659,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-minimal","name":"DeepSeek Harness: Minimal","vendor":"DeepSeek","color":"#97a9ff","version":"0.2.1-rc.3","passRate":54.37,"ciLower":40.18,"ciUpper":68.56,"costPerTrialUsd":0.24499,"costPerPassUsd":0.450598,"passAll3Rate":36.6,"tokensPerTrial":5665807,"agentSecondsPerTrial":2602.1,"nScored":248,"onFrontier":false,"dominatedById":"pi","scoreDeficit":2.65,"costRatio":2.17,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":66.67,"ciLower":55.55,"ciUpper":77.79,"ciIsUpperBound":false,"passAll3Rate":50,"costPerTrialUsd":0.17503,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":25,"ciLower":7.17,"ciUpper":42.83,"ciIsUpperBound":false,"passAll3Rate":17.65,"costPerTrialUsd":0.512513,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-code","name":"DeepSeek Harness: Code","vendor":"DeepSeek","color":"#1d8fe1","version":"0.2.1-rc.3","passRate":54.18,"ciLower":40.32,"ciUpper":68.04,"costPerTrialUsd":0.186691,"costPerPassUsd":0.344575,"passAll3Rate":31.99,"tokensPerTrial":3872181,"agentSecondsPerTrial":2323,"nScored":248,"onFrontier":false,"dominatedById":"pi","scoreDeficit":2.84,"costRatio":1.65,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":66.11,"ciLower":55.53,"ciUpper":76.69,"ciIsUpperBound":false,"passAll3Rate":45,"costPerTrialUsd":0.129621,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":25,"ciLower":7.68,"ciUpper":42.32,"ciIsUpperBound":false,"passAll3Rate":13.24,"costPerTrialUsd":0.400503,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"openclaw","name":"OpenClaw","vendor":"OpenClaw","color":"#EF3E36","version":"2026.11.1","passRate":53.52,"ciLower":46.46,"ciUpper":60.58,"costPerTrialUsd":0.159236,"costPerPassUsd":0.297526,"passAll3Rate":32.83,"tokensPerTrial":3898000,"agentSecondsPerTrial":1144,"nScored":1188,"onFrontier":false,"dominatedById":"pi","scoreDeficit":3.5,"costRatio":1.41,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":54.95,"ciLower":45.7,"ciUpper":63.87,"ciIsUpperBound":true,"passAll3Rate":29.28,"costPerTrialUsd":0.224,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":59.96,"ciLower":49.45,"ciUpper":69.62,"ciIsUpperBound":true,"passAll3Rate":44.83,"costPerTrialUsd":0.046,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"oh-my-pi","name":"Oh My Pi","vendor":"Oh My Pi","color":"#9b51e0","version":"18.12.6 / 18.18.3","passRate":52.59,"ciLower":46.58,"ciUpper":58.6,"costPerTrialUsd":0.155821,"costPerPassUsd":0.296294,"passAll3Rate":30.56,"tokensPerTrial":2888173,"agentSecondsPerTrial":1689.8,"nScored":1436,"onFrontier":false,"dominatedById":"pi","scoreDeficit":4.43,"costRatio":1.38,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":51.05,"ciLower":41.93,"ciUpper":60.09,"ciIsUpperBound":true,"passAll3Rate":25.68,"costPerTrialUsd":0.2436,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":68.89,"ciLower":58.08,"ciUpper":79.7,"ciIsUpperBound":false,"passAll3Rate":51.67,"costPerTrialUsd":0.102294,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":61.69,"ciLower":51.21,"ciUpper":71.18,"ciIsUpperBound":true,"passAll3Rate":45.4,"costPerTrialUsd":0.03844,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":29.41,"ciLower":14.71,"ciUpper":44.11,"ciIsUpperBound":false,"passAll3Rate":7.35,"costPerTrialUsd":0.306483,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"opencode-1.19","name":"OpenCode 1.19.4","vendor":"OpenCode","color":"#9aab1a","version":"1.19.4","passRate":52.54,"ciLower":39.77,"ciUpper":65.31,"costPerTrialUsd":0.148325,"costPerPassUsd":0.282309,"passAll3Rate":31.64,"tokensPerTrial":2924034,"agentSecondsPerTrial":2347.1,"nScored":248,"onFrontier":false,"dominatedById":"pi","scoreDeficit":4.48,"costRatio":1.31,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":70,"ciLower":59.26,"ciUpper":80.74,"ciIsUpperBound":false,"passAll3Rate":52.78,"costPerTrialUsd":0.120826,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":20.59,"ciLower":5.3,"ciUpper":35.87,"ciIsUpperBound":false,"passAll3Rate":8.82,"costPerTrialUsd":0.270964,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"cline-cli","name":"Cline CLI","vendor":"Cline","color":"#5b3df5","version":"3.0.73","passRate":51.74,"ciLower":38.03,"ciUpper":65.45,"costPerTrialUsd":0.135146,"costPerPassUsd":0.261202,"passAll3Rate":34.76,"tokensPerTrial":1709247,"agentSecondsPerTrial":2165.9,"nScored":248,"onFrontier":false,"dominatedById":"pi","scoreDeficit":5.28,"costRatio":1.2,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":70.56,"ciLower":60.19,"ciUpper":80.92,"ciIsUpperBound":false,"passAll3Rate":53.33,"costPerTrialUsd":0.106618,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":19.12,"ciLower":2.03,"ciUpper":36.2,"ciIsUpperBound":false,"passAll3Rate":13.24,"costPerTrialUsd":0.256083,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"alma","name":"Alma","vendor":"Alma","color":"#de7cb7","version":"0.4.212","passRate":51.26,"ciLower":38.94,"ciUpper":63.58,"costPerTrialUsd":0.158796,"costPerPassUsd":0.309785,"passAll3Rate":33.25,"tokensPerTrial":2927739,"agentSecondsPerTrial":2070.5,"nScored":248,"onFrontier":false,"dominatedById":"pi","scoreDeficit":5.76,"costRatio":1.41,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":75,"ciLower":64.67,"ciUpper":85.33,"ciIsUpperBound":false,"passAll3Rate":60.56,"costPerTrialUsd":0.120579,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":16.18,"ciLower":1.66,"ciUpper":30.7,"ciIsUpperBound":false,"passAll3Rate":7.35,"costPerTrialUsd":0.313324,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"qwen-code","name":"Qwen Code","vendor":"Alibaba","color":"#EC4899","version":"0.25.2","passRate":50.62,"ciLower":43.54,"ciUpper":57.7,"costPerTrialUsd":0.192546,"costPerPassUsd":0.380375,"passAll3Rate":29.55,"tokensPerTrial":2820000,"agentSecondsPerTrial":1154,"nScored":1188,"onFrontier":false,"dominatedById":"pi","scoreDeficit":6.4,"costRatio":1.71,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":50.6,"ciLower":41.47,"ciUpper":59.7,"ciIsUpperBound":true,"passAll3Rate":25.23,"costPerTrialUsd":0.2749,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":59.96,"ciLower":49.46,"ciUpper":69.62,"ciIsUpperBound":true,"passAll3Rate":43.1,"costPerTrialUsd":0.05053,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"maka","name":"Maka","vendor":"Maka","color":"#9b7a2e","version":"0.2.2-dev.92.20261019","passRate":49.97,"ciLower":38.31,"ciUpper":61.63,"costPerTrialUsd":0.132985,"costPerPassUsd":0.26613,"passAll3Rate":29.49,"tokensPerTrial":2118877,"agentSecondsPerTrial":2268.5,"nScored":248,"onFrontier":false,"dominatedById":"pi","scoreDeficit":7.05,"costRatio":1.18,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":71.11,"ciLower":60.9,"ciUpper":81.32,"ciIsUpperBound":false,"passAll3Rate":52.22,"costPerTrialUsd":0.113236,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":16.18,"ciLower":2.93,"ciUpper":29.42,"ciIsUpperBound":false,"passAll3Rate":5.88,"costPerTrialUsd":0.229957,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"exo","name":"Exo","vendor":"Exo","color":"#0f766e","version":"825318306c0e","passRate":48.17,"ciLower":35.21,"ciUpper":61.13,"costPerTrialUsd":0.195849,"costPerPassUsd":0.406579,"passAll3Rate":30.45,"tokensPerTrial":3709324,"agentSecondsPerTrial":2410.9,"nScored":248,"onFrontier":false,"dominatedById":"pi","scoreDeficit":8.85,"costRatio":1.73,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":62.78,"ciLower":51.7,"ciUpper":73.86,"ciIsUpperBound":false,"passAll3Rate":43.33,"costPerTrialUsd":0.157682,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":17.65,"ciLower":2.06,"ciUpper":33.24,"ciIsUpperBound":false,"passAll3Rate":11.76,"costPerTrialUsd":0.362701,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"codex","name":"Codex CLI","vendor":"OpenAI","color":"#10a37f","version":"0.164.0+baseten-catalog.1 / 0.162.1","passRate":40.63,"ciLower":35.01,"ciUpper":46.25,"costPerTrialUsd":0.145489,"costPerPassUsd":0.358083,"passAll3Rate":20.42,"tokensPerTrial":3523421,"agentSecondsPerTrial":1345.1,"nScored":1436,"onFrontier":false,"dominatedById":"pi","scoreDeficit":16.39,"costRatio":1.29,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":30.93,"ciLower":23.13,"ciUpper":40.01,"ciIsUpperBound":true,"passAll3Rate":9.01,"costPerTrialUsd":0.2283,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1 (FrontierHarness)","passRate":76.67,"ciLower":67.4,"ciUpper":85.93,"ciIsUpperBound":false,"passAll3Rate":58.89,"costPerTrialUsd":0.115247,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1 (Harness LB)","passRate":63.79,"ciLower":53.34,"ciUpper":73.08,"ciIsUpperBound":true,"passAll3Rate":41.38,"costPerTrialUsd":0.06813,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":13.24,"ciLower":1.26,"ciUpper":25.21,"ciIsUpperBound":false,"passAll3Rate":5.88,"costPerTrialUsd":0.183835,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]}]},{"sliceId":"kimi-k3","label":"Kimi-K3","org":"Moonshot","entries":[{"harnessId":"mini-swe-agent","name":"mini-SWE-agent","vendor":"SWE-agent","color":"#8b95a5","version":"2.4.6","passRate":64.92,"ciLower":54.32,"ciUpper":75.52,"costPerTrialUsd":1.313267,"costPerPassUsd":2.022921,"passAll3Rate":55.24,"tokensPerTrial":1783126,"agentSecondsPerTrial":1988.3,"nScored":248,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":85.56,"ciLower":77.1,"ciUpper":94.01,"ciIsUpperBound":false,"passAll3Rate":76.11,"costPerTrialUsd":0.866328,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":10.29,"ciLower":2.95,"ciUpper":17.64,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":2.496341,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"alma","name":"Alma","vendor":"Alma","color":"#de7cb7","version":"0.4.212","passRate":64.11,"ciLower":53.04,"ciUpper":75.19,"costPerTrialUsd":1.012647,"costPerPassUsd":1.579475,"passAll3Rate":56.85,"tokensPerTrial":1286703,"agentSecondsPerTrial":1595.7,"nScored":248,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":86.11,"ciLower":77.79,"ciUpper":94.43,"ciIsUpperBound":false,"passAll3Rate":77.78,"costPerTrialUsd":0.625836,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":5.88,"ciLower":0,"ciUpper":14.82,"ciIsUpperBound":false,"passAll3Rate":1.47,"costPerTrialUsd":2.036559,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"zcode","name":"ZCode","vendor":"Zhipu","color":"#7c4d3f","version":"0.17.0","passRate":64.11,"ciLower":53.63,"ciUpper":74.6,"costPerTrialUsd":1.063214,"costPerPassUsd":1.658346,"passAll3Rate":52.82,"tokensPerTrial":1387309,"agentSecondsPerTrial":2070.4,"nScored":248,"onFrontier":false,"dominatedById":"alma","scoreDeficit":0,"costRatio":1.05,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":82.78,"ciLower":73.59,"ciUpper":91.97,"ciIsUpperBound":false,"passAll3Rate":72.22,"costPerTrialUsd":0.62799,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":14.71,"ciLower":4.36,"ciUpper":25.05,"ciIsUpperBound":false,"passAll3Rate":1.47,"costPerTrialUsd":2.215278,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"codex","name":"Codex CLI","vendor":"OpenAI","color":"#10a37f","version":"0.164.0+baseten-catalog.1","passRate":63.71,"ciLower":53.1,"ciUpper":74.32,"costPerTrialUsd":1.133738,"costPerPassUsd":1.779538,"passAll3Rate":52.82,"tokensPerTrial":1459942,"agentSecondsPerTrial":1847.7,"nScored":248,"onFrontier":false,"dominatedById":"alma","scoreDeficit":0.4,"costRatio":1.12,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":82.78,"ciLower":73.72,"ciUpper":91.83,"ciIsUpperBound":false,"passAll3Rate":72.22,"costPerTrialUsd":0.612503,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":13.24,"ciLower":2.03,"ciUpper":24.45,"ciIsUpperBound":false,"passAll3Rate":1.47,"costPerTrialUsd":2.513479,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"minimax-code","name":"MiniMax Code","vendor":"MiniMax","color":"#f14564","version":"0.6.13","passRate":62.5,"ciLower":51.47,"ciUpper":73.53,"costPerTrialUsd":1.060538,"costPerPassUsd":1.696861,"passAll3Rate":53.63,"tokensPerTrial":1380200,"agentSecondsPerTrial":1723.3,"nScored":248,"onFrontier":false,"dominatedById":"alma","scoreDeficit":1.61,"costRatio":1.05,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":85,"ciLower":76.7,"ciUpper":93.3,"ciIsUpperBound":false,"passAll3Rate":73.89,"costPerTrialUsd":0.743526,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":2.94,"ciLower":0,"ciUpper":6.89,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":1.899688,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"grok-build","name":"Grok Build","vendor":"SpaceXAI","color":"#1f2328","version":"1.0.67","passRate":62.1,"ciLower":51.19,"ciUpper":73.01,"costPerTrialUsd":1.122643,"costPerPassUsd":1.807893,"passAll3Rate":53.23,"tokensPerTrial":1391997,"agentSecondsPerTrial":1885.1,"nScored":248,"onFrontier":false,"dominatedById":"alma","scoreDeficit":2.01,"costRatio":1.11,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":82.78,"ciLower":73.86,"ciUpper":91.7,"ciIsUpperBound":false,"passAll3Rate":72.78,"costPerTrialUsd":0.753958,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":7.35,"ciLower":0,"ciUpper":16.52,"ciIsUpperBound":false,"passAll3Rate":1.47,"costPerTrialUsd":2.098574,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"claude-code","name":"Claude Code","vendor":"Anthropic","color":"#d97757","version":"2.1.308","passRate":62.1,"ciLower":51.25,"ciUpper":72.95,"costPerTrialUsd":1.207152,"costPerPassUsd":1.943985,"passAll3Rate":52.42,"tokensPerTrial":1393431,"agentSecondsPerTrial":2153.7,"nScored":248,"onFrontier":false,"dominatedById":"alma","scoreDeficit":2.01,"costRatio":1.19,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":82.78,"ciLower":73.72,"ciUpper":91.83,"ciIsUpperBound":false,"passAll3Rate":72.22,"costPerTrialUsd":0.701766,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":7.35,"ciLower":0.37,"ciUpper":14.34,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":2.544938,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"maka","name":"Maka","vendor":"Maka","color":"#9b7a2e","version":"0.2.2-dev.92.20261019","passRate":61.69,"ciLower":50.96,"ciUpper":72.43,"costPerTrialUsd":1.372669,"costPerPassUsd":2.22498,"passAll3Rate":51.61,"tokensPerTrial":1351060,"agentSecondsPerTrial":1979,"nScored":248,"onFrontier":false,"dominatedById":"alma","scoreDeficit":2.42,"costRatio":1.36,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":81.11,"ciLower":71.62,"ciUpper":90.6,"ciIsUpperBound":false,"passAll3Rate":71.11,"costPerTrialUsd":0.834604,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":10.29,"ciLower":1.83,"ciUpper":18.76,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":2.796957,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"opencode","name":"OpenCode","vendor":"OpenCode","color":"#facc15","version":"2.2.7","passRate":61.29,"ciLower":50.04,"ciUpper":72.54,"costPerTrialUsd":0.989249,"costPerPassUsd":1.614038,"passAll3Rate":54.44,"tokensPerTrial":1180951,"agentSecondsPerTrial":1007.9,"nScored":245,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":83.33,"ciLower":74.12,"ciUpper":92.55,"ciIsUpperBound":false,"passAll3Rate":75,"costPerTrialUsd":0.613919,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":2.94,"ciLower":0,"ciUpper":6.89,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":1.982769,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"opencode-1.19","name":"OpenCode 1.19.4","vendor":"OpenCode","color":"#9aab1a","version":"1.19.4","passRate":61.29,"ciLower":50.27,"ciUpper":72.31,"costPerTrialUsd":1.067954,"costPerPassUsd":1.742451,"passAll3Rate":52.42,"tokensPerTrial":1421040,"agentSecondsPerTrial":2020.1,"nScored":248,"onFrontier":false,"dominatedById":"opencode","scoreDeficit":0,"costRatio":1.08,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":81.11,"ciLower":71.12,"ciUpper":91.1,"ciIsUpperBound":false,"passAll3Rate":72.22,"costPerTrialUsd":0.673145,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":8.82,"ciLower":1.62,"ciUpper":16.03,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":2.113038,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"pi","name":"Pi","vendor":"Pi","color":"#e0409f","version":"1.4.0","passRate":60.89,"ciLower":49.93,"ciUpper":71.85,"costPerTrialUsd":0.937952,"costPerPassUsd":1.540477,"passAll3Rate":50.81,"tokensPerTrial":1133690,"agentSecondsPerTrial":1850.1,"nScored":248,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":83.33,"ciLower":75.09,"ciUpper":91.57,"ciIsUpperBound":false,"passAll3Rate":70,"costPerTrialUsd":0.57307,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":1.47,"ciLower":0,"ciUpper":4.35,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":1.903816,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"cline-cli","name":"Cline CLI","vendor":"Cline","color":"#5b3df5","version":"3.0.73","passRate":60.89,"ciLower":49.75,"ciUpper":72.02,"costPerTrialUsd":1.062741,"costPerPassUsd":1.745429,"passAll3Rate":50.81,"tokensPerTrial":1136177,"agentSecondsPerTrial":1788.4,"nScored":248,"onFrontier":false,"dominatedById":"pi","scoreDeficit":0,"costRatio":1.13,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":83.33,"ciLower":74.66,"ciUpper":92,"ciIsUpperBound":false,"passAll3Rate":70,"costPerTrialUsd":0.627496,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":1.47,"ciLower":0,"ciUpper":4.35,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":2.214862,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"hermes","name":"Hermes Agent","vendor":"Nous Research","color":"#d99a0b","version":"0.22.5","passRate":60.89,"ciLower":49.75,"ciUpper":72.02,"costPerTrialUsd":1.108774,"costPerPassUsd":1.821033,"passAll3Rate":53.23,"tokensPerTrial":1363642,"agentSecondsPerTrial":1732.2,"nScored":248,"onFrontier":false,"dominatedById":"pi","scoreDeficit":0,"costRatio":1.18,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":81.11,"ciLower":71.12,"ciUpper":91.1,"ciIsUpperBound":false,"passAll3Rate":73.33,"costPerTrialUsd":0.714974,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":7.35,"ciLower":0.37,"ciUpper":14.34,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":2.151185,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"kimi-code","name":"Kimi Code","vendor":"Moonshot","color":"#a3305e","version":"2.3.1","passRate":60.08,"ciLower":48.96,"ciUpper":71.2,"costPerTrialUsd":1.105567,"costPerPassUsd":1.840138,"passAll3Rate":50.4,"tokensPerTrial":1508668,"agentSecondsPerTrial":1917.8,"nScored":248,"onFrontier":false,"dominatedById":"pi","scoreDeficit":0.81,"costRatio":1.18,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":81.67,"ciLower":72.4,"ciUpper":90.93,"ciIsUpperBound":false,"passAll3Rate":69.44,"costPerTrialUsd":0.688086,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":2.94,"ciLower":0,"ciUpper":6.89,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":2.210663,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-minimal","name":"DeepSeek Harness: Minimal","vendor":"DeepSeek","color":"#97a9ff","version":"0.2.1-rc.3","passRate":59.27,"ciLower":48.11,"ciUpper":70.44,"costPerTrialUsd":1.004814,"costPerPassUsd":1.695196,"passAll3Rate":50.4,"tokensPerTrial":1262414,"agentSecondsPerTrial":1860.8,"nScored":248,"onFrontier":false,"dominatedById":"pi","scoreDeficit":1.62,"costRatio":1.07,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":78.33,"ciLower":67.7,"ciUpper":88.96,"ciIsUpperBound":false,"passAll3Rate":69.44,"costPerTrialUsd":0.585159,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":8.82,"ciLower":0.48,"ciUpper":17.16,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":2.115665,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"copilot-cli","name":"GitHub Copilot CLI","vendor":"GitHub","color":"#8957e5","version":"1.0.103","passRate":58.06,"ciLower":47.27,"ciUpper":68.86,"costPerTrialUsd":1.04264,"costPerPassUsd":1.795658,"passAll3Rate":46.77,"tokensPerTrial":1484316,"agentSecondsPerTrial":2010.2,"nScored":248,"onFrontier":false,"dominatedById":"pi","scoreDeficit":2.83,"costRatio":1.11,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":77.22,"ciLower":67.39,"ciUpper":87.05,"ciIsUpperBound":false,"passAll3Rate":64.44,"costPerTrialUsd":0.707836,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":7.35,"ciLower":0,"ciUpper":15.51,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":1.928884,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"oh-my-pi","name":"Oh My Pi","vendor":"Oh My Pi","color":"#9b51e0","version":"18.12.6","passRate":57.66,"ciLower":46.7,"ciUpper":68.62,"costPerTrialUsd":0.838698,"costPerPassUsd":1.454525,"passAll3Rate":46.77,"tokensPerTrial":942405,"agentSecondsPerTrial":2070.1,"nScored":248,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":77.22,"ciLower":67.15,"ciUpper":87.29,"ciIsUpperBound":false,"passAll3Rate":64.44,"costPerTrialUsd":0.48123,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":5.88,"ciLower":0,"ciUpper":12.56,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":1.784935,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-creator","name":"DeepSeek Harness: Creator","vendor":"DeepSeek","color":"#06b6d4","version":"0.2.1-rc.3","passRate":57.66,"ciLower":46.82,"ciUpper":68.5,"costPerTrialUsd":1.297337,"costPerPassUsd":2.249927,"passAll3Rate":46.77,"tokensPerTrial":1817453,"agentSecondsPerTrial":2095.5,"nScored":248,"onFrontier":false,"dominatedById":"oh-my-pi","scoreDeficit":0,"costRatio":1.55,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":76.11,"ciLower":65.67,"ciUpper":86.55,"ciIsUpperBound":false,"passAll3Rate":64.44,"costPerTrialUsd":0.678288,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":8.82,"ciLower":1.62,"ciUpper":16.03,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":2.935996,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"antigravity-cli","name":"Antigravity CLI","vendor":"Google","color":"#2fb344","version":"1.4.6","passRate":56.85,"ciLower":45.63,"ciUpper":68.08,"costPerTrialUsd":1.550352,"costPerPassUsd":2.72686,"passAll3Rate":47.58,"tokensPerTrial":2591243,"agentSecondsPerTrial":2059.7,"nScored":248,"onFrontier":false,"dominatedById":"oh-my-pi","scoreDeficit":0.81,"costRatio":1.85,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":76.67,"ciLower":66.06,"ciUpper":87.27,"ciIsUpperBound":false,"passAll3Rate":65.56,"costPerTrialUsd":0.955067,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":4.41,"ciLower":0,"ciUpper":9.08,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":3.126107,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-code","name":"DeepSeek Harness: Code","vendor":"DeepSeek","color":"#1d8fe1","version":"0.2.1-rc.3","passRate":56.05,"ciLower":45.03,"ciUpper":67.07,"costPerTrialUsd":1.181657,"costPerPassUsd":2.10828,"passAll3Rate":45.56,"tokensPerTrial":1510554,"agentSecondsPerTrial":1948.8,"nScored":248,"onFrontier":false,"dominatedById":"oh-my-pi","scoreDeficit":1.61,"costRatio":1.41,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":74.44,"ciLower":63.6,"ciUpper":85.29,"ciIsUpperBound":false,"passAll3Rate":62.78,"costPerTrialUsd":0.658698,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":7.35,"ciLower":0.37,"ciUpper":14.34,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":2.56596,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-harness","name":"DeepSeek Harness","vendor":"DeepSeek","color":"#4d6bfe","version":"0.2.1-rc.3","passRate":54.44,"ciLower":43.42,"ciUpper":65.45,"costPerTrialUsd":1.059913,"costPerPassUsd":1.947099,"passAll3Rate":43.15,"tokensPerTrial":1323938,"agentSecondsPerTrial":1868,"nScored":248,"onFrontier":false,"dominatedById":"oh-my-pi","scoreDeficit":3.22,"costRatio":1.26,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":72.22,"ciLower":61.08,"ciUpper":83.37,"ciIsUpperBound":false,"passAll3Rate":59.44,"costPerTrialUsd":0.634892,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":7.35,"ciLower":0.37,"ciUpper":14.34,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":2.184966,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"exo","name":"Exo","vendor":"Exo","color":"#0f766e","version":"825318306c0e","passRate":43.55,"ciLower":33.93,"ciUpper":53.17,"costPerTrialUsd":0.974986,"costPerPassUsd":2.238857,"passAll3Rate":24.6,"tokensPerTrial":904261,"agentSecondsPerTrial":1553.3,"nScored":248,"onFrontier":false,"dominatedById":"oh-my-pi","scoreDeficit":14.11,"costRatio":1.16,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":57.22,"ciLower":46.7,"ciUpper":67.74,"ciIsUpperBound":false,"passAll3Rate":33.89,"costPerTrialUsd":0.660109,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":7.35,"ciLower":0.37,"ciUpper":14.34,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":1.808485,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]}]},{"sliceId":"muse-spark-open","label":"Muse Spark Open","org":"Meta","entries":[{"harnessId":"opencode","name":"OpenCode","vendor":"OpenCode","color":"#facc15","version":"2.2.7","passRate":61.29,"ciLower":49.76,"ciUpper":72.82,"costPerTrialUsd":0.718426,"costPerPassUsd":1.172169,"passAll3Rate":null,"tokensPerTrial":2295855,"agentSecondsPerTrial":1054.1,"nScored":231,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":84.44,"ciLower":75.33,"ciUpper":93.56,"ciIsUpperBound":false,"passAll3Rate":null,"costPerTrialUsd":0.481135,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":0,"ciLower":0,"ciUpper":18.43,"ciIsUpperBound":false,"passAll3Rate":null,"costPerTrialUsd":1.34655,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"grok-build","name":"Grok Build","vendor":"SpaceXAI","color":"#1f2328","version":"1.0.67","passRate":59.68,"ciLower":48.85,"ciUpper":70.51,"costPerTrialUsd":1.058757,"costPerPassUsd":1.774133,"passAll3Rate":48.79,"tokensPerTrial":3801860,"agentSecondsPerTrial":1754.5,"nScored":248,"onFrontier":false,"dominatedById":"opencode","scoreDeficit":1.61,"costRatio":1.47,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":80,"ciLower":70.71,"ciUpper":89.29,"ciIsUpperBound":false,"passAll3Rate":67.22,"costPerTrialUsd":0.58152,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":5.88,"ciLower":0,"ciUpper":12.56,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":2.322034,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"oh-my-pi","name":"Oh My Pi","vendor":"Oh My Pi","color":"#9b51e0","version":"18.12.6","passRate":58.87,"ciLower":47.65,"ciUpper":70.09,"costPerTrialUsd":0.713614,"costPerPassUsd":1.212166,"passAll3Rate":50.4,"tokensPerTrial":2548190,"agentSecondsPerTrial":1524.3,"nScored":248,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":78.33,"ciLower":67.82,"ciUpper":88.85,"ciIsUpperBound":false,"passAll3Rate":69.44,"costPerTrialUsd":0.480039,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":7.35,"ciLower":0,"ciUpper":15.51,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":1.331903,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"hermes","name":"Hermes Agent","vendor":"Nous Research","color":"#d99a0b","version":"0.22.5","passRate":57.66,"ciLower":46.76,"ciUpper":68.56,"costPerTrialUsd":0.972033,"costPerPassUsd":1.685764,"passAll3Rate":46.37,"tokensPerTrial":2820354,"agentSecondsPerTrial":1835.2,"nScored":248,"onFrontier":false,"dominatedById":"oh-my-pi","scoreDeficit":1.21,"costRatio":1.36,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":76.11,"ciLower":65.79,"ciUpper":86.44,"ciIsUpperBound":false,"passAll3Rate":63.33,"costPerTrialUsd":0.556201,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":8.82,"ciLower":0,"ciUpper":18.16,"ciIsUpperBound":false,"passAll3Rate":1.47,"costPerTrialUsd":2.072766,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"codex","name":"Codex CLI","vendor":"OpenAI","color":"#10a37f","version":"0.164.0+baseten-catalog.1","passRate":57.66,"ciLower":46.7,"ciUpper":68.62,"costPerTrialUsd":0.972227,"costPerPassUsd":1.6861,"passAll3Rate":46.77,"tokensPerTrial":4308835,"agentSecondsPerTrial":1534.6,"nScored":248,"onFrontier":false,"dominatedById":"oh-my-pi","scoreDeficit":1.21,"costRatio":1.36,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":76.11,"ciLower":65.79,"ciUpper":86.44,"ciIsUpperBound":false,"passAll3Rate":63.89,"costPerTrialUsd":0.669072,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":8.82,"ciLower":0,"ciUpper":19.07,"ciIsUpperBound":false,"passAll3Rate":1.47,"costPerTrialUsd":1.774694,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"zcode","name":"ZCode","vendor":"Zhipu","color":"#7c4d3f","version":"0.17.0","passRate":57.66,"ciLower":46.64,"ciUpper":68.68,"costPerTrialUsd":1.032025,"costPerPassUsd":1.789806,"passAll3Rate":48.39,"tokensPerTrial":2789899,"agentSecondsPerTrial":1870.1,"nScored":248,"onFrontier":false,"dominatedById":"oh-my-pi","scoreDeficit":1.21,"costRatio":1.45,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":78.33,"ciLower":68.66,"ciUpper":88.01,"ciIsUpperBound":false,"passAll3Rate":66.67,"costPerTrialUsd":0.508462,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":2.94,"ciLower":0,"ciUpper":6.89,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":2.417928,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"alma","name":"Alma","vendor":"Alma","color":"#de7cb7","version":"0.4.212","passRate":57.26,"ciLower":46.25,"ciUpper":68.26,"costPerTrialUsd":1.364028,"costPerPassUsd":2.382246,"passAll3Rate":null,"tokensPerTrial":3491569,"agentSecondsPerTrial":1988.9,"nScored":246,"onFrontier":false,"dominatedById":"oh-my-pi","scoreDeficit":1.61,"costRatio":1.91,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":76.67,"ciLower":66.65,"ciUpper":86.69,"ciIsUpperBound":false,"passAll3Rate":null,"costPerTrialUsd":0.636982,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":5.88,"ciLower":0,"ciUpper":14.82,"ciIsUpperBound":false,"passAll3Rate":1.47,"costPerTrialUsd":3.28856,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"kimi-code","name":"Kimi Code","vendor":"Moonshot","color":"#a3305e","version":"2.3.1","passRate":56.45,"ciLower":45.71,"ciUpper":67.2,"costPerTrialUsd":0.842012,"costPerPassUsd":1.491564,"passAll3Rate":42.74,"tokensPerTrial":3293607,"agentSecondsPerTrial":1510.4,"nScored":248,"onFrontier":false,"dominatedById":"oh-my-pi","scoreDeficit":2.42,"costRatio":1.18,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":75,"ciLower":65.03,"ciUpper":84.97,"ciIsUpperBound":false,"passAll3Rate":58.33,"costPerTrialUsd":0.663391,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":7.35,"ciLower":0,"ciUpper":16.52,"ciIsUpperBound":false,"passAll3Rate":1.47,"costPerTrialUsd":1.314829,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"minimax-code","name":"MiniMax Code","vendor":"MiniMax","color":"#f14564","version":"0.6.13","passRate":55.91,"ciLower":45.14,"ciUpper":66.69,"costPerTrialUsd":0.924192,"costPerPassUsd":1.652882,"passAll3Rate":43.15,"tokensPerTrial":2797678,"agentSecondsPerTrial":1796.2,"nScored":247,"onFrontier":false,"dominatedById":"oh-my-pi","scoreDeficit":2.96,"costRatio":1.3,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":74.26,"ciLower":63.87,"ciUpper":84.65,"ciIsUpperBound":false,"passAll3Rate":59.44,"costPerTrialUsd":0.546316,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":7.35,"ciLower":0.37,"ciUpper":14.34,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":1.924452,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"exo","name":"Exo","vendor":"Exo","color":"#0f766e","version":"825318306c0e","passRate":55.24,"ciLower":44.08,"ciUpper":66.4,"costPerTrialUsd":0.738169,"costPerPassUsd":1.336248,"passAll3Rate":45.16,"tokensPerTrial":3029737,"agentSecondsPerTrial":1155.8,"nScored":248,"onFrontier":false,"dominatedById":"oh-my-pi","scoreDeficit":3.63,"costRatio":1.03,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":73.89,"ciLower":62.88,"ciUpper":84.9,"ciIsUpperBound":false,"passAll3Rate":62.22,"costPerTrialUsd":0.497058,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":5.88,"ciLower":0,"ciUpper":12.56,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":1.376404,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"cline-cli","name":"Cline CLI","vendor":"Cline","color":"#5b3df5","version":"3.0.73","passRate":55.24,"ciLower":44.37,"ciUpper":66.12,"costPerTrialUsd":0.865614,"costPerPassUsd":1.566951,"passAll3Rate":null,"tokensPerTrial":2760533,"agentSecondsPerTrial":1552,"nScored":246,"onFrontier":false,"dominatedById":"oh-my-pi","scoreDeficit":3.63,"costRatio":1.21,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":75,"ciLower":65.03,"ciUpper":84.97,"ciIsUpperBound":false,"passAll3Rate":null,"costPerTrialUsd":0.589829,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":2.94,"ciLower":0,"ciUpper":6.89,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":1.595632,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"opencode-1.19","name":"OpenCode 1.19.4","vendor":"OpenCode","color":"#9aab1a","version":"1.19.4","passRate":54.44,"ciLower":43.66,"ciUpper":65.22,"costPerTrialUsd":0.96655,"costPerPassUsd":1.775588,"passAll3Rate":41.53,"tokensPerTrial":4333214,"agentSecondsPerTrial":1821.4,"nScored":247,"onFrontier":false,"dominatedById":"oh-my-pi","scoreDeficit":4.43,"costRatio":1.35,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":73.33,"ciLower":63.19,"ciUpper":83.47,"ciIsUpperBound":false,"passAll3Rate":57.22,"costPerTrialUsd":0.552822,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":4.41,"ciLower":0,"ciUpper":10.69,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":2.061713,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"copilot-cli","name":"GitHub Copilot CLI","vendor":"GitHub","color":"#8957e5","version":"1.0.103","passRate":54.44,"ciLower":43.19,"ciUpper":65.68,"costPerTrialUsd":1.609158,"costPerPassUsd":2.956083,"passAll3Rate":44.76,"tokensPerTrial":3009953,"agentSecondsPerTrial":2052,"nScored":248,"onFrontier":false,"dominatedById":"oh-my-pi","scoreDeficit":4.43,"costRatio":2.25,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":72.22,"ciLower":61.19,"ciUpper":83.26,"ciIsUpperBound":false,"passAll3Rate":59.44,"costPerTrialUsd":0.719817,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":7.35,"ciLower":0,"ciUpper":19.06,"ciIsUpperBound":false,"passAll3Rate":5.88,"costPerTrialUsd":3.963297,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"mini-swe-agent","name":"mini-SWE-agent","vendor":"SWE-agent","color":"#8b95a5","version":"2.4.6","passRate":54.17,"ciLower":42.91,"ciUpper":65.43,"costPerTrialUsd":0.944351,"costPerPassUsd":1.743417,"passAll3Rate":45.97,"tokensPerTrial":3702499,"agentSecondsPerTrial":1708.9,"nScored":245,"onFrontier":false,"dominatedById":"oh-my-pi","scoreDeficit":4.7,"costRatio":1.32,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":72.41,"ciLower":61,"ciUpper":83.81,"ciIsUpperBound":false,"passAll3Rate":63.33,"costPerTrialUsd":0.732483,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":5.88,"ciLower":0,"ciUpper":12.56,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":1.505181,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"maka","name":"Maka","vendor":"Maka","color":"#9b7a2e","version":"0.2.2-dev.92.20261019","passRate":54.03,"ciLower":43.27,"ciUpper":64.79,"costPerTrialUsd":0.812842,"costPerPassUsd":1.504364,"passAll3Rate":41.94,"tokensPerTrial":2106711,"agentSecondsPerTrial":1571,"nScored":248,"onFrontier":false,"dominatedById":"oh-my-pi","scoreDeficit":4.84,"costRatio":1.14,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":69.44,"ciLower":58.12,"ciUpper":80.77,"ciIsUpperBound":false,"passAll3Rate":57.22,"costPerTrialUsd":0.549354,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":13.24,"ciLower":2.03,"ciUpper":24.45,"ciIsUpperBound":false,"passAll3Rate":1.47,"costPerTrialUsd":1.510312,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-minimal","name":"DeepSeek Harness: Minimal","vendor":"DeepSeek","color":"#97a9ff","version":"0.2.1-rc.3","passRate":53.76,"ciLower":42.92,"ciUpper":64.6,"costPerTrialUsd":0.825103,"costPerPassUsd":1.534692,"passAll3Rate":41.53,"tokensPerTrial":3174159,"agentSecondsPerTrial":1483.1,"nScored":247,"onFrontier":false,"dominatedById":"oh-my-pi","scoreDeficit":5.11,"costRatio":1.16,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":71.11,"ciLower":60.1,"ciUpper":82.12,"ciIsUpperBound":false,"passAll3Rate":57.22,"costPerTrialUsd":0.600874,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":7.84,"ciLower":0.5,"ciUpper":15.18,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":1.41865,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-code","name":"DeepSeek Harness: Code","vendor":"DeepSeek","color":"#1d8fe1","version":"0.2.1-rc.3","passRate":53.23,"ciLower":42.16,"ciUpper":64.29,"costPerTrialUsd":1.100494,"costPerPassUsd":2.067595,"passAll3Rate":41.53,"tokensPerTrial":4599966,"agentSecondsPerTrial":1881.5,"nScored":246,"onFrontier":false,"dominatedById":"oh-my-pi","scoreDeficit":5.64,"costRatio":1.54,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":70.56,"ciLower":59.19,"ciUpper":81.92,"ciIsUpperBound":false,"passAll3Rate":57.22,"costPerTrialUsd":0.599879,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":7.35,"ciLower":0,"ciUpper":15.51,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":2.425651,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"claude-code","name":"Claude Code","vendor":"Anthropic","color":"#d97757","version":"2.1.308","passRate":52.96,"ciLower":42.4,"ciUpper":63.51,"costPerTrialUsd":1.984232,"costPerPassUsd":3.746875,"passAll3Rate":37.1,"tokensPerTrial":9817486,"agentSecondsPerTrial":2634.5,"nScored":247,"onFrontier":false,"dominatedById":"oh-my-pi","scoreDeficit":5.91,"costRatio":2.78,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":72.41,"ciLower":62.85,"ciUpper":81.96,"ciIsUpperBound":false,"passAll3Rate":51.11,"costPerTrialUsd":0.848835,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":1.47,"ciLower":0,"ciUpper":4.35,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":4.989695,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-creator","name":"DeepSeek Harness: Creator","vendor":"DeepSeek","color":"#06b6d4","version":"0.2.1-rc.3","passRate":52.42,"ciLower":41.22,"ciUpper":63.61,"costPerTrialUsd":0.701504,"costPerPassUsd":1.338254,"passAll3Rate":41.53,"tokensPerTrial":2874107,"agentSecondsPerTrial":1535.7,"nScored":248,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":67.78,"ciLower":55.65,"ciUpper":79.9,"ciIsUpperBound":false,"passAll3Rate":56.67,"costPerTrialUsd":0.528452,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":11.76,"ciLower":0.55,"ciUpper":22.97,"ciIsUpperBound":false,"passAll3Rate":1.47,"costPerTrialUsd":1.159584,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"pi","name":"Pi","vendor":"Pi","color":"#e0409f","version":"1.4.0","passRate":52.02,"ciLower":41.55,"ciUpper":62.48,"costPerTrialUsd":0.763227,"costPerPassUsd":1.467289,"passAll3Rate":36.69,"tokensPerTrial":2399680,"agentSecondsPerTrial":1442.1,"nScored":247,"onFrontier":false,"dominatedById":"deepseek-creator","scoreDeficit":0.4,"costRatio":1.09,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":68.89,"ciLower":58.31,"ciUpper":79.47,"ciIsUpperBound":false,"passAll3Rate":50.56,"costPerTrialUsd":0.398899,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":7.35,"ciLower":0.37,"ciUpper":14.34,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":1.727625,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"antigravity-cli","name":"Antigravity CLI","vendor":"Google","color":"#2fb344","version":"1.4.6","passRate":50.81,"ciLower":39.94,"ciUpper":61.67,"costPerTrialUsd":1.179751,"costPerPassUsd":2.32205,"passAll3Rate":38.31,"tokensPerTrial":4757037,"agentSecondsPerTrial":1630.2,"nScored":246,"onFrontier":false,"dominatedById":"deepseek-creator","scoreDeficit":1.61,"costRatio":1.68,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":68.33,"ciLower":57.17,"ciUpper":79.5,"ciIsUpperBound":false,"passAll3Rate":52.78,"costPerTrialUsd":0.84686,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":4.41,"ciLower":0,"ciUpper":9.08,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":2.060932,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]},{"harnessId":"deepseek-harness","name":"DeepSeek Harness","vendor":"DeepSeek","color":"#4d6bfe","version":"0.2.1-rc.3","passRate":49.87,"ciLower":38.56,"ciUpper":61.17,"costPerTrialUsd":0.882062,"costPerPassUsd":1.768879,"passAll3Rate":40.32,"tokensPerTrial":2781108,"agentSecondsPerTrial":1598.4,"nScored":247,"onFrontier":false,"dominatedById":"deepseek-creator","scoreDeficit":2.55,"costRatio":1.26,"benchmarks":[{"benchmarkId":"tb21-frontierharness","name":"Terminal-Bench 2.1","passRate":67.04,"ciLower":54.9,"ciUpper":79.18,"ciIsUpperBound":false,"passAll3Rate":55.56,"costPerTrialUsd":0.560016,"nTasks":45,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"},{"benchmarkId":"terminal-bench-4.0","name":"Terminal-Bench 4.0","passRate":4.41,"ciLower":0,"ciUpper":9.08,"ciIsUpperBound":false,"passAll3Rate":0,"costPerTrialUsd":1.734539,"nTasks":17,"sourceUrl":"https://frontierharness.org/leaderboard/v1.1/"}]}]},{"sliceId":"ricursiveAI/Ripple-65B-Preview","label":"Ripple-65B","org":"Ricursive","entries":[{"harnessId":"kimi-code","name":"Kimi Code","vendor":"Moonshot","color":"#0EA5E9","version":"2.2.0","passRate":56.43,"ciLower":49.48,"ciUpper":62.97,"costPerTrialUsd":1.932,"costPerPassUsd":3.477,"passAll3Rate":37.45,"tokensPerTrial":18600000,"agentSecondsPerTrial":2018,"nScored":594,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":49.25,"ciLower":40.13,"ciUpper":58.42,"ciIsUpperBound":false,"passAll3Rate":24.32,"costPerTrialUsd":3.108,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":63.6,"ciLower":53.12,"ciUpper":72.94,"ciIsUpperBound":false,"passAll3Rate":50.57,"costPerTrialUsd":0.4306,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"codex","name":"Codex","vendor":"OpenAI","color":"#3B7CF6","version":"0.162.1","passRate":55.93,"ciLower":48.99,"ciUpper":62.49,"costPerTrialUsd":1.584,"costPerPassUsd":2.878,"passAll3Rate":36.67,"tokensPerTrial":15370000,"agentSecondsPerTrial":1783,"nScored":594,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":48.65,"ciLower":39.55,"ciUpper":57.84,"ciIsUpperBound":false,"passAll3Rate":21.62,"costPerTrialUsd":2.543,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":63.22,"ciLower":52.73,"ciUpper":72.59,"ciIsUpperBound":false,"passAll3Rate":51.72,"costPerTrialUsd":0.3611,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"opencode","name":"OpenCode","vendor":"Anomaly","color":"#3E8E5E","version":"2.2.6","passRate":53.86,"ciLower":46.96,"ciUpper":60.48,"costPerTrialUsd":1.643,"costPerPassUsd":3.109,"passAll3Rate":32.32,"tokensPerTrial":15960000,"agentSecondsPerTrial":1773,"nScored":594,"onFrontier":false,"dominatedById":"codex","scoreDeficit":2.07,"costRatio":1.04,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":45.65,"ciLower":36.68,"ciUpper":54.9,"ciIsUpperBound":false,"passAll3Rate":19.82,"costPerTrialUsd":2.632,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":62.07,"ciLower":51.57,"ciUpper":71.55,"ciIsUpperBound":false,"passAll3Rate":44.83,"costPerTrialUsd":0.3813,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"hermes","name":"Hermes Agent","vendor":"Nous Research","color":"#FF9D00","version":"0.22.1","passRate":51.76,"ciLower":44.83,"ciUpper":58.55,"costPerTrialUsd":1.949,"costPerPassUsd":3.795,"passAll3Rate":29.33,"tokensPerTrial":18560000,"agentSecondsPerTrial":2370,"nScored":594,"onFrontier":false,"dominatedById":"codex","scoreDeficit":4.17,"costRatio":1.23,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":48.35,"ciLower":39.26,"ciUpper":57.54,"ciIsUpperBound":false,"passAll3Rate":20.72,"costPerTrialUsd":3.132,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":55.17,"ciLower":44.72,"ciUpper":65.18,"ciIsUpperBound":false,"passAll3Rate":37.93,"costPerTrialUsd":0.4387,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"pi","name":"Pi","vendor":"Earendil","color":"#A855F7","version":"1.2.0","passRate":51.55,"ciLower":44.69,"ciUpper":58.25,"costPerTrialUsd":1.623,"costPerPassUsd":3.214,"passAll3Rate":30.4,"tokensPerTrial":14600000,"agentSecondsPerTrial":2297,"nScored":594,"onFrontier":false,"dominatedById":"codex","scoreDeficit":4.38,"costRatio":1.02,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":42.94,"ciLower":34.12,"ciUpper":52.24,"ciIsUpperBound":false,"passAll3Rate":17.12,"costPerTrialUsd":2.7,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":60.15,"ciLower":49.65,"ciUpper":69.8,"ciIsUpperBound":false,"passAll3Rate":43.68,"costPerTrialUsd":0.2497,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"grok-build","name":"Grok Build","vendor":"SpaceXAI","color":"#111111","version":"1.0.77","passRate":50.79,"ciLower":43.9,"ciUpper":57.59,"costPerTrialUsd":2.119,"costPerPassUsd":4.225,"passAll3Rate":28.3,"tokensPerTrial":21440000,"agentSecondsPerTrial":2034,"nScored":594,"onFrontier":false,"dominatedById":"codex","scoreDeficit":5.14,"costRatio":1.34,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":45.65,"ciLower":36.68,"ciUpper":54.9,"ciIsUpperBound":false,"passAll3Rate":19.82,"costPerTrialUsd":3.436,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":55.94,"ciLower":45.48,"ciUpper":65.9,"ciIsUpperBound":false,"passAll3Rate":36.78,"costPerTrialUsd":0.44,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"deepseek-harness","name":"DeepSeek Harness","vendor":"DeepSeek","color":"#4D6BFE","version":"0.2.2-rc.1","passRate":48.2,"ciLower":41.48,"ciUpper":54.9,"costPerTrialUsd":1.724,"costPerPassUsd":3.685,"passAll3Rate":30.2,"tokensPerTrial":14610000,"agentSecondsPerTrial":2333,"nScored":594,"onFrontier":false,"dominatedById":"codex","scoreDeficit":7.73,"costRatio":1.09,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":36.64,"ciLower":28.26,"ciUpper":45.91,"ciIsUpperBound":false,"passAll3Rate":14.41,"costPerTrialUsd":2.749,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":59.77,"ciLower":49.26,"ciUpper":69.45,"ciIsUpperBound":false,"passAll3Rate":45.98,"costPerTrialUsd":0.4179,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"mini-swe-agent","name":"mini-swe-agent","vendor":"Princeton/Stanford","color":"#8C6D46","version":"2.4.6","passRate":47.37,"ciLower":40.8,"ciUpper":53.91,"costPerTrialUsd":1.831,"costPerPassUsd":4.029,"passAll3Rate":32.29,"tokensPerTrial":17530000,"agentSecondsPerTrial":1977,"nScored":594,"onFrontier":false,"dominatedById":"codex","scoreDeficit":8.56,"costRatio":1.16,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":31.53,"ciLower":23.63,"ciUpper":40.67,"ciIsUpperBound":false,"passAll3Rate":11.71,"costPerTrialUsd":2.787,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":63.22,"ciLower":52.73,"ciUpper":72.59,"ciIsUpperBound":false,"passAll3Rate":52.87,"costPerTrialUsd":0.6126,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"claude-code","name":"Claude Code","vendor":"Anthropic","color":"#E07A4F","version":"2.1.300","passRate":47.24,"ciLower":40.46,"ciUpper":54.1,"costPerTrialUsd":2.075,"costPerPassUsd":4.466,"passAll3Rate":29.7,"tokensPerTrial":19030000,"agentSecondsPerTrial":2459,"nScored":594,"onFrontier":false,"dominatedById":"codex","scoreDeficit":8.69,"costRatio":1.31,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":40.84,"ciLower":32.15,"ciUpper":50.14,"ciIsUpperBound":false,"passAll3Rate":18.02,"costPerTrialUsd":3.445,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":53.64,"ciLower":43.23,"ciUpper":63.74,"ciIsUpperBound":false,"passAll3Rate":41.38,"costPerTrialUsd":0.3275,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"fx","name":"fx","vendor":"Vercel Labs","color":"#000000","version":"0.0.16","passRate":44.78,"ciLower":38.09,"ciUpper":51.68,"costPerTrialUsd":2,"costPerPassUsd":4.551,"passAll3Rate":22.44,"tokensPerTrial":19350000,"agentSecondsPerTrial":1841,"nScored":594,"onFrontier":false,"dominatedById":"codex","scoreDeficit":11.15,"costRatio":1.26,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":37.84,"ciLower":29.37,"ciUpper":47.12,"ciIsUpperBound":false,"passAll3Rate":8.11,"costPerTrialUsd":3.202,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":51.72,"ciLower":41.37,"ciUpper":61.93,"ciIsUpperBound":false,"passAll3Rate":36.78,"costPerTrialUsd":0.4659,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"oh-my-pi","name":"oh-my-pi","vendor":"Stencil Labs","color":"#F97316","version":"18.18.3","passRate":44.07,"ciLower":37.42,"ciUpper":50.95,"costPerTrialUsd":2.609,"costPerPassUsd":6.053,"passAll3Rate":20.84,"tokensPerTrial":23210000,"agentSecondsPerTrial":2735,"nScored":594,"onFrontier":false,"dominatedById":"codex","scoreDeficit":11.86,"costRatio":1.65,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":36.04,"ciLower":27.71,"ciUpper":45.3,"ciIsUpperBound":false,"passAll3Rate":7.21,"costPerTrialUsd":4.315,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":52.11,"ciLower":41.74,"ciUpper":62.29,"ciIsUpperBound":false,"passAll3Rate":34.48,"costPerTrialUsd":0.4324,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"qwen-code","name":"Qwen Code","vendor":"Alibaba","color":"#EC4899","version":"0.25.2","passRate":41.79,"ciLower":35.3,"ciUpper":48.59,"costPerTrialUsd":1.743,"costPerPassUsd":4.314,"passAll3Rate":21.87,"tokensPerTrial":14880000,"agentSecondsPerTrial":2158,"nScored":594,"onFrontier":false,"dominatedById":"codex","scoreDeficit":14.14,"costRatio":1.1,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":30.33,"ciLower":22.56,"ciUpper":39.42,"ciIsUpperBound":false,"passAll3Rate":8.11,"costPerTrialUsd":2.676,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":53.26,"ciLower":42.86,"ciUpper":63.38,"ciIsUpperBound":false,"passAll3Rate":35.63,"costPerTrialUsd":0.5532,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]},{"harnessId":"openclaw","name":"OpenClaw","vendor":"OpenClaw","color":"#EF3E36","version":"2026.11.1","passRate":34.43,"ciLower":28.33,"ciUpper":41.24,"costPerTrialUsd":1.047,"costPerPassUsd":3.157,"passAll3Rate":16.17,"tokensPerTrial":10090000,"agentSecondsPerTrial":1361,"nScored":594,"onFrontier":true,"dominatedById":null,"scoreDeficit":null,"costRatio":null,"benchmarks":[{"benchmarkId":"deep-swe","name":"DeepSWE v1.1","passRate":24.02,"ciLower":17.03,"ciUpper":32.76,"ciIsUpperBound":false,"passAll3Rate":3.6,"costPerTrialUsd":1.586,"nTasks":111,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"},{"benchmarkId":"tb21-harness-lb","name":"Terminal-Bench 2.1","passRate":44.83,"ciLower":34.82,"ciUpper":55.28,"ciIsUpperBound":false,"passAll3Rate":28.74,"costPerTrialUsd":0.3593,"nTasks":87,"sourceUrl":"https://huggingface.co/spaces/joelniklaus/harness-leaderboard"}]}]}]}},"lifecycle":{"auditedAt":"2026-11-21T22:00:15.273Z","audits":[{"benchmarkId":"frontiercode_2_0","benchmarkVersion":"2.0","domainId":"coding","harnessRegime":"standardized","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":49.13,"top5Score":28.66,"top5Median":44.41,"rNorm":1.029,"saturationIndex":0.3472,"separability":0.472,"looKendallTau":0.9594,"looCohortCount":155,"looDroppedEligibleCount":0,"itemRestTau":0.707,"ageMonths":3.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Pairwise 95% separability 47% < 50%"]},{"benchmarkId":"terminalbench_4_0","benchmarkVersion":"4.0","domainId":"coding","harnessRegime":"standardized","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":67.17,"top5Score":57.07,"top5Median":64.14,"rNorm":0.421,"saturationIndex":0.8372,"separability":0.472,"looKendallTau":0.945,"looCohortCount":155,"looDroppedEligibleCount":0,"itemRestTau":0.7484,"ageMonths":4.2,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["High small-N Popoviciu uncertainty (S=0.84, top-1 below 85% ceiling)","Pairwise 95% separability 47% < 50%"]},{"benchmarkId":"scicode_aa","benchmarkVersion":"1.0","domainId":"coding","harnessRegime":"standardized","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":67.71,"top5Score":59.03,"top5Median":61.81,"rNorm":0.331,"saturationIndex":0.8959,"separability":0,"looKendallTau":0.975,"looCohortCount":155,"looDroppedEligibleCount":0,"itemRestTau":0.6085,"ageMonths":6.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["High small-N Popoviciu uncertainty (S=0.90, top-1 below 85% ceiling)","Pairwise 95% separability 0% < 35% (small-N Popoviciu bound below 85% ceiling)"]},{"benchmarkId":"epoch_mirrorcode","benchmarkVersion":"2026.11","domainId":"coding","harnessRegime":"standardized","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":97.78,"top5Score":54.44,"top5Median":64.44,"rNorm":3.958,"saturationIndex":0,"separability":0.25,"looKendallTau":0.9622,"looCohortCount":147,"looDroppedEligibleCount":8,"itemRestTau":0.5619,"ageMonths":4.9,"upstreamLagDays":4,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Pairwise 95% separability 25% < 35% (low spread without saturation pressure)"]},{"benchmarkId":"epoch_ale_bench","benchmarkVersion":"2026.11","domainId":"coding","harnessRegime":"standardized","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":2951.32,"top5Score":1688.35,"top5Median":2374.7,"rNorm":null,"saturationIndex":null,"separability":null,"looKendallTau":0.9553,"looCohortCount":126,"looDroppedEligibleCount":29,"itemRestTau":0.67,"ageMonths":3.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Uncertainty / separability unavailable"]},{"benchmarkId":"epoch_frontierswe","benchmarkVersion":"2026.11","domainId":"coding","harnessRegime":"standardized","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":8,"top1Score":69.94,"top5Score":29.5,"top5Median":61.2,"rNorm":1.502,"saturationIndex":0.1048,"separability":0.536,"looKendallTau":0.9968,"looCohortCount":155,"looDroppedEligibleCount":0,"itemRestTau":0.8182,"ageMonths":2.6,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"epoch_cursorbench","benchmarkVersion":"2026.11","domainId":"coding","harnessRegime":"standardized","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":5,"top1Score":65.4,"top5Score":40.9,"top5Median":48.6,"rNorm":null,"saturationIndex":null,"separability":null,"looKendallTau":0.9868,"looCohortCount":147,"looDroppedEligibleCount":8,"itemRestTau":0.7357,"ageMonths":2.4,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Uncertainty / separability unavailable"]},{"benchmarkId":"lmarena_agent","benchmarkVersion":"agent_capability","domainId":"coding","harnessRegime":"standardized","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":1488.82,"top5Score":1192.17,"top5Median":1325.18,"rNorm":9.62,"saturationIndex":0,"separability":0.806,"looKendallTau":0.9624,"looCohortCount":152,"looDroppedEligibleCount":3,"itemRestTau":0.7468,"ageMonths":1.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"surge_sudo_l7","benchmarkVersion":"1.0","domainId":"coding","harnessRegime":"standardized","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":8,"top1Score":52.2,"top5Score":16.7,"top5Median":38.3,"rNorm":1.578,"saturationIndex":0.083,"separability":0.536,"looKendallTau":0.9923,"looCohortCount":155,"looDroppedEligibleCount":0,"itemRestTau":0.7586,"ageMonths":1.4,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"vals_code_migration","benchmarkVersion":"1.1","domainId":"coding","harnessRegime":"standardized","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":86.88,"top5Score":67.52,"top5Median":72.36,"rNorm":6.138,"saturationIndex":0,"separability":0.833,"looKendallTau":0.9896,"looCohortCount":155,"looDroppedEligibleCount":0,"itemRestTau":0.6381,"ageMonths":2.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"vals_sre_bench","benchmarkVersion":"1","domainId":"coding","harnessRegime":"standardized","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":8,"top1Score":56.87,"top5Score":9.16,"top5Median":27.1,"rNorm":13.446,"saturationIndex":0,"separability":0.857,"looKendallTau":0.9718,"looCohortCount":155,"looDroppedEligibleCount":0,"itemRestTau":0.5985,"ageMonths":2.2,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"livebench_long_agentic","benchmarkVersion":"20261106","domainId":"coding","harnessRegime":"standardized","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":82.9,"top5Score":46.6,"top5Median":74.7,"rNorm":1.18,"saturationIndex":0.2488,"separability":0.417,"looKendallTau":0.9943,"looCohortCount":155,"looDroppedEligibleCount":0,"itemRestTau":0.678,"ageMonths":1.2,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Pairwise 95% separability 42% < 50%"]},{"benchmarkId":"tbench_4_1","benchmarkVersion":"4.1","domainId":"harnesses","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":65.76,"top5Score":46.97,"top5Median":62.12,"rNorm":0.777,"saturationIndex":0.5467,"separability":0.528,"looKendallTau":0.9247,"looCohortCount":84,"looDroppedEligibleCount":26,"itemRestTau":0.6134,"ageMonths":2.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Akhtar saturation index 0.55 approaching ceiling"]},{"benchmarkId":"vulcanbench_swe_v4","benchmarkVersion":"4.0","domainId":"harnesses","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":5,"top1Score":93.85,"top5Score":88.36,"top5Median":90,"rNorm":6.356,"saturationIndex":0,"separability":0.6,"looKendallTau":0.9091,"looCohortCount":65,"looDroppedEligibleCount":45,"itemRestTau":0.3251,"ageMonths":1.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"aa_agent_frontiercode","benchmarkVersion":"2.0","domainId":"harnesses","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":6,"top1Score":48.74,"top5Score":24.39,"top5Median":38.04,"rNorm":1.251,"saturationIndex":0.2091,"separability":0.6,"looKendallTau":0.9603,"looCohortCount":102,"looDroppedEligibleCount":8,"itemRestTau":0.4152,"ageMonths":1.2,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"aa_agent_swe_atlas","benchmarkVersion":"1.0","domainId":"harnesses","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":6,"top1Score":66.4,"top5Score":59.95,"top5Median":61.56,"rNorm":0.316,"saturationIndex":0.9049,"separability":0,"looKendallTau":0.9623,"looCohortCount":110,"looDroppedEligibleCount":0,"itemRestTau":0.3238,"ageMonths":1.2,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["High small-N Popoviciu uncertainty (S=0.90, top-1 below 85% ceiling)","Pairwise 95% separability 0% < 35% (small-N Popoviciu bound below 85% ceiling)"]},{"benchmarkId":"lmarena_creative_writing","benchmarkVersion":"text_style_control","domainId":"writing","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":1528.73,"top5Score":1470.84,"top5Median":1486.75,"rNorm":5.6,"saturationIndex":0,"separability":0.75,"looKendallTau":0.7741,"looCohortCount":80,"looDroppedEligibleCount":27,"itemRestTau":0.3531,"ageMonths":5.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"lechmazur_writing","benchmarkVersion":"2026.11","domainId":"writing","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":4.3,"top5Score":2.37,"top5Median":3.55,"rNorm":35.067,"saturationIndex":0,"separability":0.917,"looKendallTau":0.8799,"looCohortCount":93,"looDroppedEligibleCount":14,"itemRestTau":0.44,"ageMonths":7.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"eqbench_creative_writing","benchmarkVersion":"3.0","domainId":"writing","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":16.63,"top5Score":16.17,"top5Median":16.38,"rNorm":0.099,"saturationIndex":0.9903,"separability":0,"looKendallTau":0.9785,"looCohortCount":96,"looDroppedEligibleCount":11,"itemRestTau":0.5616,"ageMonths":6.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["High small-N Popoviciu uncertainty (S=0.99, top-1 below 85% ceiling)","Pairwise 95% separability 0% < 35% (small-N Popoviciu bound below 85% ceiling)"]},{"benchmarkId":"eqbench_longform_writing","benchmarkVersion":"3.0","domainId":"writing","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":86.3,"top5Score":81.8,"top5Median":82.8,"rNorm":0.153,"saturationIndex":0.977,"separability":0,"looKendallTau":0.9785,"looCohortCount":96,"looDroppedEligibleCount":11,"itemRestTau":0.5364,"ageMonths":9,"upstreamLagDays":null,"triggerSince":"2026-11-21","consecutiveTriggerDays":1,"status":"warning","reasons":["Akhtar saturation index 0.98 >= 0.70","Pairwise 95% separability 0% < 35% with saturation pressure","Day 1 of 14 sustained confirmation window"]},{"benchmarkId":"surge_complex_constraints","benchmarkVersion":"1.1","domainId":"writing","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":8,"top1Score":73.7,"top5Score":60.8,"top5Median":67.9,"rNorm":0.576,"saturationIndex":0.7174,"separability":0.214,"looKendallTau":0.9846,"looCohortCount":100,"looDroppedEligibleCount":7,"itemRestTau":0.1538,"ageMonths":5.6,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["High small-N Popoviciu uncertainty (S=0.72, top-1 below 85% ceiling)","Pairwise 95% separability 21% < 35% (small-N Popoviciu bound below 85% ceiling)"]},{"benchmarkId":"vals_vibe_code","benchmarkVersion":"1.1","domainId":"design_code","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":93.77,"top5Score":86.66,"top5Median":89.9,"rNorm":2.79,"saturationIndex":0.0004,"separability":0.333,"looKendallTau":0.9703,"looCohortCount":105,"looDroppedEligibleCount":20,"itemRestTau":0.7737,"ageMonths":4.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Pairwise 95% separability 33% < 35% (low spread without saturation pressure)"]},{"benchmarkId":"vals_swebench_multimodal","benchmarkVersion":"1","domainId":"design_code","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":8,"top1Score":70.21,"top5Score":44.17,"top5Median":47.29,"rNorm":8.441,"saturationIndex":0,"separability":0.464,"looKendallTau":0.9946,"looCohortCount":125,"looDroppedEligibleCount":0,"itemRestTau":0.5897,"ageMonths":3.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Pairwise 95% separability 46% < 50%"]},{"benchmarkId":"lmarena_webdev","benchmarkVersion":"webdev","domainId":"design_code","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":1823.64,"top5Score":1722.63,"top5Median":1766.18,"rNorm":9.249,"saturationIndex":0,"separability":0.917,"looKendallTau":0.9322,"looCohortCount":111,"looDroppedEligibleCount":14,"itemRestTau":0.7953,"ageMonths":6.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"designarena_website","benchmarkVersion":"public","domainId":"design_code","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":1374,"top5Score":1326,"top5Median":1336,"rNorm":null,"saturationIndex":null,"separability":null,"looKendallTau":0.9866,"looCohortCount":125,"looDroppedEligibleCount":0,"itemRestTau":0.7977,"ageMonths":5.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Uncertainty / separability unavailable"]},{"benchmarkId":"designarena_svg","benchmarkVersion":"public","domainId":"design_code","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":1414,"top5Score":1306,"top5Median":1319,"rNorm":null,"saturationIndex":null,"separability":null,"looKendallTau":0.9701,"looCohortCount":125,"looDroppedEligibleCount":0,"itemRestTau":0.7691,"ageMonths":4.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Uncertainty / separability unavailable"]},{"benchmarkId":"designarena_3d","benchmarkVersion":"public","domainId":"design_code","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":1521,"top5Score":1420,"top5Median":1489,"rNorm":null,"saturationIndex":null,"separability":null,"looKendallTau":0.9719,"looCohortCount":125,"looDroppedEligibleCount":0,"itemRestTau":0.8147,"ageMonths":4.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Uncertainty / separability unavailable"]},{"benchmarkId":"designarena_gamedev","benchmarkVersion":"public","domainId":"design_code","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":1539,"top5Score":1382,"top5Median":1410,"rNorm":null,"saturationIndex":null,"separability":null,"looKendallTau":0.9664,"looCohortCount":124,"looDroppedEligibleCount":1,"itemRestTau":0.8638,"ageMonths":4.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Uncertainty / separability unavailable"]},{"benchmarkId":"aa_code2video","benchmarkVersion":"1.0","domainId":"design_code","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":8,"top1Score":1535.57,"top5Score":1041.34,"top5Median":1259.4,"rNorm":29.999,"saturationIndex":0,"separability":0.786,"looKendallTau":0.9905,"looCohortCount":125,"looDroppedEligibleCount":0,"itemRestTau":0.6833,"ageMonths":2.2,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"mlcr_aa","benchmarkVersion":"1.0","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":43.89,"top5Score":30.56,"top5Median":38.33,"rNorm":0.547,"saturationIndex":0.7413,"separability":0.25,"looKendallTau":0.9865,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.4635,"ageMonths":4.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["High small-N Popoviciu uncertainty (S=0.74, top-1 below 85% ceiling)","Pairwise 95% separability 25% < 35% (small-N Popoviciu bound below 85% ceiling)"]},{"benchmarkId":"aa_omniscience","benchmarkVersion":"1.1","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":79.13,"top5Score":45.73,"top5Median":60.47,"rNorm":3.845,"saturationIndex":0,"separability":0.889,"looKendallTau":0.9758,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.6515,"ageMonths":4.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"gdp_pdf","benchmarkVersion":"1.0","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":33.4,"top5Score":28.8,"top5Median":31.4,"rNorm":0.222,"saturationIndex":0.9519,"separability":0.25,"looKendallTau":0.9896,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.5406,"ageMonths":3.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["High small-N Popoviciu uncertainty (S=0.95, top-1 below 85% ceiling)","Pairwise 95% separability 25% < 35% (small-N Popoviciu bound below 85% ceiling)"]},{"benchmarkId":"hle_diamond","benchmarkVersion":"1.0","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":80.2,"top5Score":44.9,"top5Median":71.3,"rNorm":3.114,"saturationIndex":0.0001,"separability":0.917,"looKendallTau":0.9706,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.7942,"ageMonths":4.2,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"critpt_v2_hard","benchmarkVersion":"2.0","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":76.5,"top5Score":49.5,"top5Median":69,"rNorm":1.032,"saturationIndex":0.3449,"separability":0.472,"looKendallTau":0.9816,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.7169,"ageMonths":3.2,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Pairwise 95% separability 47% < 50%"]},{"benchmarkId":"terminalbench_science_aa","benchmarkVersion":"0.1","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":78.57,"top5Score":34.29,"top5Median":70,"rNorm":2.034,"saturationIndex":0.016,"separability":0.694,"looKendallTau":0.9666,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.7475,"ageMonths":2.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"aa_analyst_agent","benchmarkVersion":"1.0","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":8,"top1Score":62.5,"top5Score":46.25,"top5Median":55,"rNorm":0.699,"saturationIndex":0.6135,"separability":0.5,"looKendallTau":0.9964,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.4525,"ageMonths":3.4,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Akhtar saturation index 0.61 approaching ceiling"]},{"benchmarkId":"aa_mka","benchmarkVersion":"1.0","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":8,"top1Score":56.56,"top5Score":43.86,"top5Median":52.67,"rNorm":0.681,"saturationIndex":0.629,"separability":0.429,"looKendallTau":0.9997,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.8954,"ageMonths":0.4,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Akhtar saturation index 0.63 approaching ceiling","Pairwise 95% separability 43% < 50%"]},{"benchmarkId":"aa_vials","benchmarkVersion":"1.0","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":8,"top1Score":67.08,"top5Score":45.34,"top5Median":63.98,"rNorm":1.131,"saturationIndex":0.2784,"separability":0.571,"looKendallTau":0.9989,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.7273,"ageMonths":1.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"aa_briefcase","benchmarkVersion":"1.1","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":1813.08,"top5Score":1673.56,"top5Median":1698.69,"rNorm":18.315,"saturationIndex":0,"separability":0.917,"looKendallTau":0.9983,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.4771,"ageMonths":5.1,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"gdpval_aa","benchmarkVersion":"2.1","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":8,"top1Score":1857.09,"top5Score":1732.68,"top5Median":1808.69,"rNorm":7.008,"saturationIndex":0,"separability":0.821,"looKendallTau":0.9971,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.4667,"ageMonths":5.2,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"epoch_simpleqa_verified","benchmarkVersion":"2026.11","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":94,"top5Score":60.3,"top5Median":77.7,"rNorm":19.545,"saturationIndex":0,"separability":0.833,"looKendallTau":0.9375,"looCohortCount":165,"looDroppedEligibleCount":5,"itemRestTau":0.5038,"ageMonths":14.4,"upstreamLagDays":2,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"epoch_apex_agents","benchmarkVersion":"2026.11","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":8,"top1Score":82.9,"top5Score":66.1,"top5Median":72.2,"rNorm":2.394,"saturationIndex":0.0032,"separability":0.571,"looKendallTau":0.9934,"looCohortCount":168,"looDroppedEligibleCount":2,"itemRestTau":0.619,"ageMonths":10,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"lmarena_factuality","benchmarkVersion":"text_factuality","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":1522.23,"top5Score":1478.82,"top5Median":1493.67,"rNorm":7.934,"saturationIndex":0,"separability":0.806,"looKendallTau":0.957,"looCohortCount":149,"looDroppedEligibleCount":21,"itemRestTau":0.4932,"ageMonths":3.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"vals_legal_research","benchmarkVersion":"1","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":55.77,"top5Score":48.08,"top5Median":55.29,"rNorm":1.57,"saturationIndex":0.0849,"separability":0.111,"looKendallTau":0.9794,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.4188,"ageMonths":4.2,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Pairwise 95% separability 11% < 35% (low spread without saturation pressure)"]},{"benchmarkId":"vals_finance_agent","benchmarkVersion":"2.1","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":69.47,"top5Score":65.43,"top5Median":67.58,"rNorm":1.428,"saturationIndex":0.1302,"separability":0.278,"looKendallTau":0.995,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.2851,"ageMonths":3.2,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Pairwise 95% separability 28% < 35% (low spread without saturation pressure)"]},{"benchmarkId":"vals_emb","benchmarkVersion":"1.1","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":77.38,"top5Score":71.98,"top5Median":76.03,"rNorm":1.668,"saturationIndex":0.0619,"separability":0.167,"looKendallTau":0.9981,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.7169,"ageMonths":3.2,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Pairwise 95% separability 17% < 35% (low spread without saturation pressure)"]},{"benchmarkId":"vals_tax_agent","benchmarkVersion":"1","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":52.3,"top5Score":43.33,"top5Median":49.66,"rNorm":1.77,"saturationIndex":0.0436,"separability":0.389,"looKendallTau":0.9593,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.3725,"ageMonths":2.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Pairwise 95% separability 39% < 50%"]},{"benchmarkId":"vals_mystery_mechanism","benchmarkVersion":"1","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":53.15,"top5Score":30.63,"top5Median":45.49,"rNorm":4.928,"saturationIndex":0,"separability":0.639,"looKendallTau":0.994,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.7028,"ageMonths":2.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"vals_medcode","benchmarkVersion":"1","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":63.57,"top5Score":49.55,"top5Median":52.73,"rNorm":4.757,"saturationIndex":0,"separability":0.722,"looKendallTau":0.9671,"looCohortCount":166,"looDroppedEligibleCount":4,"itemRestTau":0.4444,"ageMonths":3.2,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"surge_gdp_xlsx","benchmarkVersion":"1.0","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":43.1,"top5Score":22.6,"top5Median":29.7,"rNorm":0.912,"saturationIndex":0.435,"separability":0.25,"looKendallTau":0.9968,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.678,"ageMonths":1.7,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Pairwise 95% separability 25% < 35% (low spread without saturation pressure)"]},{"benchmarkId":"surge_dayjob_finance","benchmarkVersion":"1.0","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":50.5,"top5Score":26.5,"top5Median":43.7,"rNorm":1.075,"saturationIndex":0.3151,"separability":0.611,"looKendallTau":0.9887,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.6444,"ageMonths":1.9,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"surge_dayjob_law","benchmarkVersion":"1.0","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":8,"top1Score":41.4,"top5Score":20.6,"top5Median":26.2,"rNorm":0.944,"saturationIndex":0.4099,"separability":0.536,"looKendallTau":0.9969,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.3474,"ageMonths":0.1,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"surge_dayjob_healthcare","benchmarkVersion":"1.0","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":48.3,"top5Score":27.6,"top5Median":34.4,"rNorm":0.819,"saturationIndex":0.5112,"separability":0.611,"looKendallTau":0.9916,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.7018,"ageMonths":1.9,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Akhtar saturation index 0.51 approaching ceiling"]},{"benchmarkId":"surge_riemann","benchmarkVersion":"1.0","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":84,"top5Score":60.8,"top5Median":83.2,"rNorm":0.839,"saturationIndex":0.4947,"separability":0.444,"looKendallTau":0.996,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.5446,"ageMonths":8,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"warning","reasons":["Pairwise 95% separability 44% < 50%"]},{"benchmarkId":"scale_prbench_finance","benchmarkVersion":"1.0","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":6,"top1Score":59.54,"top5Score":47.54,"top5Median":52.29,"rNorm":13.611,"saturationIndex":0,"separability":0.933,"looKendallTau":0.9857,"looCohortCount":170,"looDroppedEligibleCount":0,"itemRestTau":0.4425,"ageMonths":12.3,"upstreamLagDays":2,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"scale_prbench_legal","benchmarkVersion":"1.0","domainId":"research","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":7,"top1Score":61.56,"top5Score":47.87,"top5Median":52.56,"rNorm":15.251,"saturationIndex":0,"separability":0.857,"looKendallTau":0.9638,"looCohortCount":169,"looDroppedEligibleCount":1,"itemRestTau":0.2411,"ageMonths":12.3,"upstreamLagDays":2,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"cwe_bench_v2","benchmarkVersion":"2.0","domainId":"cyber","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":9,"top1Score":75,"top5Score":57.6,"top5Median":66.8,"rNorm":4.87,"saturationIndex":0,"separability":0.778,"looKendallTau":null,"looCohortCount":0,"looDroppedEligibleCount":17,"itemRestTau":null,"ageMonths":0.6,"upstreamLagDays":null,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"cyberarena_mc_v1","benchmarkVersion":"mc-v1","domainId":"cyber","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":8,"top1Score":76.89,"top5Score":65.52,"top5Median":65.9,"rNorm":5.854,"saturationIndex":0,"separability":0.643,"looKendallTau":null,"looCohortCount":0,"looDroppedEligibleCount":11,"itemRestTau":null,"ageMonths":1.2,"upstreamLagDays":0,"triggerSince":null,"consecutiveTriggerDays":0,"status":"healthy","reasons":["Unsaturated and discriminating across active frontier labs"]},{"benchmarkId":"aikido_arena","benchmarkVersion":"1.0","domainId":"cyber","harnessRegime":"native","lifecycleState":"ACTIVE","top5LabsCount":5,"coveredLabsCount":6,"top1Score":86.89,"top5Score":78.4,"top5Median":84.16,"rNorm":0.371,"saturationIndex":0.8714,"separability":0,"looKendallTau":0.963,"looCohortCount":28,"looDroppedEligibleCount":0,"itemRestTau":0.3333,"ageMonths":1.7,"upstreamLagDays":9,"triggerSince":"2026-11-21","consecutiveTriggerDays":1,"status":"warning","reasons":["Akhtar saturation index 0.87 >= 0.70","Pairwise 95% separability 0% < 35% with saturation pressure","Day 1 of 14 sustained confirmation window"]}],"crossDomainTau":[{"domainA":"coding","regimeA":"standardized","domainB":"writing","regimeB":"native","overlapCount":79,"kendallTau":0.5404},{"domainA":"coding","regimeA":"standardized","domainB":"design_code","regimeB":"native","overlapCount":83,"kendallTau":0.7497},{"domainA":"coding","regimeA":"standardized","domainB":"research","regimeB":"native","overlapCount":89,"kendallTau":0.6083},{"domainA":"coding","regimeA":"standardized","domainB":"cyber","regimeB":"native","overlapCount":29,"kendallTau":0.601},{"domainA":"coding","regimeA":"standardized","domainB":"harnesses","regimeB":"native","overlapCount":42,"kendallTau":0.6818},{"domainA":"writing","regimeA":"native","domainB":"design_code","regimeB":"native","overlapCount":85,"kendallTau":0.5809},{"domainA":"writing","regimeA":"native","domainB":"research","regimeB":"native","overlapCount":88,"kendallTau":0.6466},{"domainA":"writing","regimeA":"native","domainB":"cyber","regimeB":"native","overlapCount":27,"kendallTau":0.4644},{"domainA":"writing","regimeA":"native","domainB":"harnesses","regimeB":"native","overlapCount":39,"kendallTau":0.4224},{"domainA":"design_code","regimeA":"native","domainB":"research","regimeB":"native","overlapCount":91,"kendallTau":0.5964},{"domainA":"design_code","regimeA":"native","domainB":"cyber","regimeB":"native","overlapCount":29,"kendallTau":0.5172},{"domainA":"design_code","regimeA":"native","domainB":"harnesses","regimeB":"native","overlapCount":42,"kendallTau":0.6167},{"domainA":"research","regimeA":"native","domainB":"cyber","regimeB":"native","overlapCount":29,"kendallTau":0.5517},{"domainA":"research","regimeA":"native","domainB":"harnesses","regimeB":"native","overlapCount":42,"kendallTau":0.5145},{"domainA":"cyber","regimeA":"native","domainB":"harnesses","regimeB":"native","overlapCount":27,"kendallTau":0.5385}]}}