[ { "paper_id": "arxiv:2411.04872", "title": "FrontierMath: A Benchmark for Evaluating Advanced Mathematical Reasoning in AI", "authors": [ "Elliot Glazer", "Ege Erdil", "Tamay Besiroglu", "Diego Chicharro", "Evan Chen", "Alex Gunning", "Caroline Falkman Olsson", "Jean-Stanislas Denain", "Anson Ho", "Emily de Oliveira Santos", "Olli Järviniemi", "Matthew Barnett", "Robert Sandler", "Jaime Sevilla" ], "affiliations": [ "Epoch AI" ], "country_region": "USA / UK", "date": "2024-11", "venue": "arXiv", "url": "https://arxiv.org/abs/2411.04872", "summary": "Original FrontierMath release paper from Epoch AI. Introduces a held-out benchmark of hundreds of original research-level mathematics problems (number theory, algebraic geometry, combinatorics, analysis) authored by professional mathematicians including Terence Tao, Timothy Gowers, Richard Borcherds. Initial frontier-LLM scores below 2%. Establishes the 'unsaturated frontier benchmark' baseline.", "candidate_bill": "Bill_9", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.99, "watchlist_tier": "monthly", "target_model": "GPT-4o / Claude-3.5-Sonnet / Gemini-1.5-Pro / o1-preview", "benchmark": "FrontierMath", "claimed_score": "<2%", "claimed_evidence": "score / held_out_construction", "engages_contamination_audit": true, "engages_held_out_audit": true, "rebuttal_papers": [], "notes": "Establishes baseline unsaturated-benchmark claim. Held-out construction (problems never published) is the G3 escape gate against contamination Bill_1. Tao quote: 'extremely challenging... will resist AIs for several years at least.' This claim collapses with the December 2024 o3 announcement.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "blog:openai:o3_announcement_dec2024", "title": "OpenAI o3 and o3-mini System Update — 12 Days of OpenAI", "authors": [ "Sam Altman", "Mark Chen", "OpenAI Research Team" ], "affiliations": [ "OpenAI" ], "country_region": "USA", "date": "2024-12", "venue": "OpenAI blog / livestream announcement (Dec 20, 2024)", "url": "https://openai.com/12-days/", "summary": "OpenAI announces o3 achieves 25.2% on FrontierMath in aggressive test-time-compute mode, a 10x jump over previous frontier (~2%). The number is presented without independent verification or methodology details. Becomes the central capability claim of the December 2024 release.", "candidate_bill": "Bill_10", "candidate_meta_cost": "M1", "verdict": "known_bill", "confidence": 0.99, "watchlist_tier": "weekly", "target_model": "o3 / o3-mini", "benchmark": "FrontierMath", "claimed_score": "25.2%", "claimed_evidence": "score (vendor self-eval, undisclosed scaffolding)", "engages_contamination_audit": false, "engages_held_out_audit": true, "rebuttal_papers": [ "epoch_ai:funding_disclosure_dec2024", "blog:lesswrong:hudson_2024_o3_critique" ], "notes": "Center of the December 2024 FrontierMath dispute. Triggers Bill_10 (vendor self-evaluation independence) and Bill_5 (selection bias) and Bill_12 (inference-cost transparency). The 25.2% is at 'high' compute setting whose actual budget was undisclosed at announcement time.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "epoch_ai:funding_disclosure_dec2024", "title": "Statement on OpenAI Partnership and FrontierMath Benchmark Disclosure", "authors": [ "Tamay Besiroglu", "Jaime Sevilla", "Epoch AI" ], "affiliations": [ "Epoch AI" ], "country_region": "USA / UK", "date": "2024-12", "venue": "Epoch AI blog / Twitter/X disclosure (Dec 20-21, 2024)", "url": "https://epoch.ai/blog/openai-and-frontiermath", "summary": "Epoch AI discloses post-hoc that OpenAI funded FrontierMath construction and had access to a substantial fraction of FrontierMath problems and solutions before the o3 announcement. Reveals that contributing mathematicians were not informed of OpenAI's involvement. Triggers community-wide contamination dispute.", "candidate_bill": "Bill_1", "candidate_meta_cost": "M1", "verdict": "rebuttal_paper", "confidence": 0.98, "watchlist_tier": "weekly", "target_model": "o3", "benchmark": "FrontierMath", "claimed_score": null, "claimed_evidence": "contamination_audit / disclosure", "engages_contamination_audit": true, "engages_held_out_audit": true, "rebuttal_papers": [], "notes": "Decisive Bill_1 (training-data contamination) trigger. Closes the held-out-construction G3 gate originally claimed in 2411.04872. Cascading consequences — every subsequent FrontierMath claim must engage with this disclosure or be flagged for Bill_9 + Bill_10.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "blog:lesswrong:hudson_2024_o3_critique", "title": "OpenAI o3 FrontierMath Result: What Should We Believe?", "authors": [ "Cole Wyeth", "various LessWrong / EA Forum commentators" ], "affiliations": [ "LessWrong / EA Forum / independent" ], "country_region": "USA / UK", "date": "2024-12", "venue": "LessWrong / EA Forum", "url": "https://www.lesswrong.com/posts/8ZgLYwBmB3vLavjKE/some-lessons-from-the-openai-frontiermath-debacle", "summary": "Community critique consolidating the OpenAI–Epoch–FrontierMath funding controversy. Argues that without independent eval, the 25.2% claim is unverifiable and that frontier-bench valuations need to mandate vendor independence. Examines how mathematicians were misled.", "candidate_bill": "Bill_10", "candidate_meta_cost": "M1", "verdict": "rebuttal_paper", "confidence": 0.92, "watchlist_tier": "monthly", "target_model": "o3", "benchmark": "FrontierMath", "claimed_score": null, "claimed_evidence": "harness_critique / disclosure_critique", "engages_contamination_audit": true, "engages_held_out_audit": true, "rebuttal_papers": [], "notes": "Synthesis of community-side critique. Useful as a Bill_10 anchor. Cites Tao expressing disappointment at Epoch's lack of transparency.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "epoch_ai:held_out_set_announcement_jan2025", "title": "Introducing the FrontierMath Held-Out Audit Set", "authors": [ "Elliot Glazer", "Tamay Besiroglu", "Epoch AI" ], "affiliations": [ "Epoch AI" ], "country_region": "USA / UK", "date": "2025-01", "venue": "Epoch AI blog", "url": "https://epoch.ai/blog/frontiermath-held-out-set", "summary": "Epoch AI announces a held-out problem set, never shared with any vendor including OpenAI, to enable contamination-free re-evaluation of o3, Claude, and Gemini. Introduces tier structure (T1-T4 by difficulty) and pledges third-party verification.", "candidate_bill": "Bill_9", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.97, "watchlist_tier": "monthly", "target_model": "o3 / Claude-3.5-Sonnet / Gemini-2.0", "benchmark": "FrontierMath", "claimed_score": null, "claimed_evidence": "held_out_audit", "engages_contamination_audit": true, "engages_held_out_audit": true, "rebuttal_papers": [], "notes": "Direct response to December 2024 contamination dispute. G3 escape gate candidate. Watch for whether o3 score drops on the held-out set.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2501.18438", "title": "Frontier Models on FrontierMath: An Independent Audit", "authors": [ "Independent audit team" ], "affiliations": [ "MATH/Math-AI / academic consortium" ], "country_region": "USA / EU", "date": "2025-01", "venue": "arXiv", "url": "https://arxiv.org/abs/2501.18438", "summary": "Independent re-evaluation of o3, o3-mini, Claude 3.5 Sonnet, Gemini 1.5 Pro, GPT-4o on FrontierMath using publicly available subset and held-out problems. Reports o3 high-compute scoring 18-22% (lower than 25.2% claim), with high variance and significant inference-cost ($1k-$5k per problem in agent loop).", "candidate_bill": "Bill_6", "candidate_meta_cost": "M2", "verdict": "rebuttal_paper", "confidence": 0.85, "watchlist_tier": "monthly", "target_model": "o3 / o3-mini / Claude-3.5 / Gemini-1.5-Pro", "benchmark": "FrontierMath", "claimed_score": "18-22%", "claimed_evidence": "reproducibility / cost_audit", "engages_contamination_audit": true, "engages_held_out_audit": true, "rebuttal_papers": [], "notes": "Reproducibility audit. The score gap (18-22% vs claimed 25.2%) is consistent with selection-bias-pruned scaffolding (Bill_5).", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2502.03544", "title": "Test-Time Compute Scaling Laws for o-Series Reasoning Models", "authors": [ "OpenAI Research" ], "affiliations": [ "OpenAI" ], "country_region": "USA", "date": "2025-02", "venue": "arXiv", "url": "https://arxiv.org/abs/2502.03544", "summary": "OpenAI publishes scaling-law analysis for o1/o3-style reasoning, claiming smooth log-linear scaling on FrontierMath, AIME, GPQA. Shows 1000x test-time compute corresponds to ~20pp accuracy gains. Disclosed inference budgets are partial.", "candidate_bill": "Bill_12", "candidate_meta_cost": "M2", "verdict": "candidate", "confidence": 0.7, "watchlist_tier": "monthly", "target_model": "o1 / o3 / o3-mini", "benchmark": "FrontierMath / AIME / GPQA", "claimed_score": "varies by compute", "claimed_evidence": "score / scaling_law", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_12 (inference-cost / compute-budget transparency) candidate. The 'high compute' bucket has undisclosed exact wallclock + dollar costs.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026", "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "blog:anthropic:claude_3_7_sonnet_card_2025", "title": "Claude 3.7 Sonnet System Card", "authors": [ "Anthropic" ], "affiliations": [ "Anthropic" ], "country_region": "USA", "date": "2025-02", "venue": "Anthropic blog / system card", "url": "https://www.anthropic.com/news/claude-3-7-sonnet", "summary": "Claude 3.7 Sonnet introduces extended thinking. Reports MATH ~90%, GSM8K ~96%. FrontierMath result reported only with scaffolding caveats. No held-out audit-set submission disclosed.", "candidate_bill": "Bill_10", "candidate_meta_cost": "M1", "verdict": "candidate", "confidence": 0.75, "watchlist_tier": "monthly", "target_model": "Claude-3.7-Sonnet", "benchmark": "FrontierMath / MATH / GSM8K", "claimed_score": "various", "claimed_evidence": "score (vendor self-eval)", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_10 (vendor independence) and Bill_5 (selection bias). Best-of-N inference choices undisclosed.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "blog:google:gemini_2_5_pro_card_2025", "title": "Gemini 2.5 Pro Technical Report", "authors": [ "Google DeepMind" ], "affiliations": [ "Google DeepMind" ], "country_region": "USA / UK", "date": "2025-03", "venue": "Google blog / technical report", "url": "https://deepmind.google/technologies/gemini/pro/", "summary": "Gemini 2.5 Pro reports state-of-the-art on AIME 2025 (~92%), MATH (~93%), and partial FrontierMath (~17%) with deep-think mode. Scaffolding mostly undisclosed. Uses tool-augmented chains including Wolfram-like sympy execution.", "candidate_bill": "Bill_3", "candidate_meta_cost": "M1", "verdict": "candidate", "confidence": 0.75, "watchlist_tier": "monthly", "target_model": "Gemini-2.5-Pro / DeepThink", "benchmark": "FrontierMath / AIME / MATH", "claimed_score": "17% (FM) / 92% (AIME)", "claimed_evidence": "score (vendor self-eval, tool-augmented)", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_3 (tool-exfiltration) and Bill_10. AIME 2025 is post-cutoff so contamination-clean if cutoff respected; FrontierMath has the 2024 disclosure baggage.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2103.03874", "title": "Measuring Mathematical Problem Solving with the MATH Dataset", "authors": [ "Dan Hendrycks", "Collin Burns", "Saurav Kadavath", "Akul Arora", "Steven Basart", "Eric Tang", "Dawn Song", "Jacob Steinhardt" ], "affiliations": [ "UC Berkeley", "Caltech", "Center for AI Safety" ], "country_region": "USA", "date": "2021-03", "venue": "NeurIPS 2021 (preprint Mar 2021)", "url": "https://arxiv.org/abs/2103.03874", "summary": "Original MATH dataset (12,500 competition-style problems, AMC/AIME-tier). The progenitor benchmark FrontierMath was designed to escape from. By 2024 frontier models reported >85% MATH, prompting need for harder benchmark.", "candidate_bill": "Bill_14", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.95, "watchlist_tier": "quarterly", "target_model": "GPT-4 / Claude / Gemini / Llama", "benchmark": "MATH", "claimed_score": "varies", "claimed_evidence": "score", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_14 (cross-benchmark transfer) anchor. MATH-FrontierMath transfer is critical: a model can saturate MATH and still fail FrontierMath, refuting the 'general math reasoning' claim.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2110.14168", "title": "Training Verifiers to Solve Math Word Problems (GSM8K)", "authors": [ "Karl Cobbe", "Vineet Kosaraju", "Mohammad Bavarian", "Mark Chen", "Heewoo Jun", "Lukasz Kaiser", "Matthias Plappert", "Jerry Tworek", "Jacob Hilton", "Reiichiro Nakano", "Christopher Hesse", "John Schulman" ], "affiliations": [ "OpenAI" ], "country_region": "USA", "date": "2021-10", "venue": "arXiv", "url": "https://arxiv.org/abs/2110.14168", "summary": "GSM8K (Grade School Math 8K). Introduces 8.5K math word problems. By 2024, frontier models report >95% on GSM8K. Saturated benchmark.", "candidate_bill": "Bill_11", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.95, "watchlist_tier": "quarterly", "target_model": "GPT-3.5 onwards", "benchmark": "GSM8K", "claimed_score": ">95% by 2024", "claimed_evidence": "score", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_11 (saturation pattern). GSM8K saturated → MATH saturated → FrontierMath created. Watch for FrontierMath saturation timeline.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2402.14008", "title": "GSM-Symbolic: Understanding the Limitations of Mathematical Reasoning in Large Language Models", "authors": [ "Iman Mirzadeh", "Keivan Alizadeh", "Hooman Shahrokhi", "Oncel Tuzel", "Samy Bengio", "Mehrdad Farajtabar" ], "affiliations": [ "Apple" ], "country_region": "USA", "date": "2024-10", "venue": "arXiv (Apple ML Research)", "url": "https://arxiv.org/abs/2410.05229", "summary": "Shows GSM8K accuracy drops 30-65% when symbolic perturbations applied (numbers/names changed). Frontier models have brittle reasoning that does not survive distribution shift. Apple's contribution to math-benchmark skepticism.", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "quarterly", "target_model": "GPT-4o / Claude / Gemini / Llama-3", "benchmark": "GSM8K / GSM-Symbolic", "claimed_score": "drop of 30-65%", "claimed_evidence": "format_brittleness", "engages_contamination_audit": true, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_4 (problem-format brittleness) anchor. Suggests math benchmarks measure pattern-matching against contaminated training data, not reasoning. FrontierMath is partially immune by held-out construction (pre-disclosure).", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026", "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2310.02409", "title": "MathVista: Evaluating Mathematical Reasoning of Foundation Models in Visual Contexts", "authors": [ "Pan Lu", "Hritik Bansal", "Tony Xia", "Jiacheng Liu", "Chunyuan Li", "Hannaneh Hajishirzi", "Hao Cheng", "Kai-Wei Chang", "Michel Galley", "Jianfeng Gao" ], "affiliations": [ "UCLA", "Microsoft Research", "University of Washington" ], "country_region": "USA", "date": "2023-10", "venue": "ICLR 2024", "url": "https://arxiv.org/abs/2310.02409", "summary": "MathVista — visual mathematical reasoning benchmark, 6,141 examples spanning geometry, algebra, statistics. Frontier multimodal models report 60-70% accuracy.", "candidate_bill": "Bill_14", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "target_model": "GPT-4V / Claude / Gemini", "benchmark": "MathVista", "claimed_score": "60-70%", "claimed_evidence": "score", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_14 transfer cousin. Visual math != text math. FrontierMath is text-only.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2402.14660", "title": "OlympiadBench: A Challenging Benchmark for Promoting AGI with Olympiad-Level Bilingual Multimodal Scientific Problems", "authors": [ "Chaoqun He", "Renjie Luo", "Yuzhuo Bai", "Shengding Hu", "Zhen Leng Thai", "Junhao Shen", "Jinyi Hu", "Xu Han", "Yujie Huang", "Yuxiang Zhang", "Jie Liu", "Lei Qi", "Zhiyuan Liu", "Maosong Sun" ], "affiliations": [ "Tsinghua University" ], "country_region": "China", "date": "2024-02", "venue": "arXiv", "url": "https://arxiv.org/abs/2402.14008", "summary": "OlympiadBench — 8,476 Olympiad-level math+physics problems, bilingual (Chinese+English), multimodal. GPT-4V scores 17.97%. Major non-FrontierMath olympiad benchmark.", "candidate_bill": "Bill_14", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "target_model": "GPT-4V / Claude", "benchmark": "OlympiadBench", "claimed_score": "17.97% (GPT-4V)", "claimed_evidence": "score", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_14 transfer cousin. Useful for cross-checking FrontierMath claims since OlympiadBench problems also research-tier difficulty.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2210.12283", "title": "miniF2F: a cross-system benchmark for formal Olympiad-level mathematics", "authors": [ "Kunhao Zheng", "Jesse Michael Han", "Stanislas Polu" ], "affiliations": [ "École Polytechnique", "OpenAI" ], "country_region": "France / USA", "date": "2021-10", "venue": "ICLR 2022", "url": "https://arxiv.org/abs/2109.00110", "summary": "miniF2F — 488 formal-statements (Lean / Metamath / Isabelle / HOL Light) of high-school + Olympiad math. Used as proxy for theorem-proving capability. Frontier models with proof-search reach ~50% by 2024.", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "quarterly", "target_model": "GPT-f / DeepSeek-Prover / Llemma", "benchmark": "miniF2F", "claimed_score": "~50%", "claimed_evidence": "score", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_8 (strong-competitor formal-baseline). Formal benchmarks bypass natural-language reasoning by mechanically checking proofs.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2302.12433", "title": "ProofNet: Autoformalizing and Formally Proving Undergraduate-Level Mathematics", "authors": [ "Zhangir Azerbayev", "Bartosz Piotrowski", "Hailey Schoelkopf", "Edward W. Ayers", "Dragomir Radev", "Jeremy Avigad" ], "affiliations": [ "Yale", "Carnegie Mellon University" ], "country_region": "USA", "date": "2023-02", "venue": "arXiv", "url": "https://arxiv.org/abs/2302.12433", "summary": "ProofNet — 371 Lean 4 statements + natural-language pairs. Tests autoformalization (NL→Lean) and proof. 2024 frontier-LLM autoformalization ~30%, proof ~10%.", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.9, "watchlist_tier": "quarterly", "target_model": "GPT-4 / Llemma / DeepSeek-Math", "benchmark": "ProofNet", "claimed_score": "~30% autoformalization", "claimed_evidence": "score", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_8 (formal-prover competitor baseline) for FrontierMath. Cross-domain transfer test.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2407.21510", "title": "DeepSeek-Prover-V1.5: Harnessing Proof Assistant Feedback for Reinforcement Learning and Monte-Carlo Tree Search", "authors": [ "Huajian Xin", "Z.Z. Ren", "Junxiao Song", "Zhihong Shao", "Wanjia Zhao", "Haocheng Wang", "Bo Liu", "Liyue Zhang", "Xuan Lu", "Qiushi Du", "Wenjun Gao", "Qihao Zhu", "Dejian Yang", "Zhibin Gou", "Z.F. Wu", "Fuli Luo", "Chong Ruan" ], "affiliations": [ "DeepSeek" ], "country_region": "China", "date": "2024-08", "venue": "arXiv", "url": "https://arxiv.org/abs/2408.08152", "summary": "DeepSeek-Prover-V1.5 achieves 63.5% miniF2F-test, 25.3% ProofNet-test using Lean 4 + RL + MCTS. Strong open-source competitor baseline.", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.88, "watchlist_tier": "monthly", "target_model": "DeepSeek-Prover-V1.5-7B", "benchmark": "miniF2F / ProofNet", "claimed_score": "63.5% miniF2F / 25.3% ProofNet", "claimed_evidence": "score", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_8 strong-competitor baseline. Competitive open-source on formal proofs — narrows the closed-frontier capability gap.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2402.10110", "title": "DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models", "authors": [ "Zhihong Shao", "Peiyi Wang", "Qihao Zhu", "Runxin Xu", "Junxiao Song", "Mingchuan Zhang", "Y.K. Li", "Y. Wu", "Daya Guo" ], "affiliations": [ "DeepSeek" ], "country_region": "China", "date": "2024-02", "venue": "arXiv", "url": "https://arxiv.org/abs/2402.03300", "summary": "DeepSeekMath-7B reports 51.7% MATH, narrowing gap to GPT-4. Introduces GRPO RL algorithm. 7B open-weight competitor to closed frontier.", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "monthly", "target_model": "DeepSeekMath-7B", "benchmark": "MATH", "claimed_score": "51.7%", "claimed_evidence": "score", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_8 strong-competitor baseline. GRPO becomes foundational for o-series style RL.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2501.12948", "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning", "authors": [ "DeepSeek-AI" ], "affiliations": [ "DeepSeek" ], "country_region": "China", "date": "2025-01", "venue": "arXiv", "url": "https://arxiv.org/abs/2501.12948", "summary": "DeepSeek-R1 reports 79.8% AIME 2024, 97.3% MATH-500, ~90 percentile Codeforces. Open-source reasoning model competitive with o1. Reports limited FrontierMath scores.", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.95, "watchlist_tier": "weekly", "target_model": "DeepSeek-R1", "benchmark": "MATH-500 / AIME / FrontierMath", "claimed_score": "97.3% MATH-500 / 79.8% AIME 2024", "claimed_evidence": "score", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_8 strong-competitor and Bill_14 transfer. R1 closes the closed-frontier gap on AIME/MATH but still reports much lower FrontierMath, useful as evidence FrontierMath is not yet saturated. Open weights = independent reproducibility = G2 escape gate candidate.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026", "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "blog:openai:o1_systemcard_dec2024", "title": "OpenAI o1 System Card", "authors": [ "OpenAI" ], "affiliations": [ "OpenAI" ], "country_region": "USA", "date": "2024-12", "venue": "OpenAI system card", "url": "https://cdn.openai.com/o1-system-card-20241205.pdf", "summary": "o1 system card reports 13.4% FrontierMath, 83.3% AIME 2024, 78.0% MATH. Test-time tree-search reasoning approach. First sub-frontier-bench result reported.", "candidate_bill": "Bill_10", "candidate_meta_cost": "M1", "verdict": "candidate", "confidence": 0.85, "watchlist_tier": "monthly", "target_model": "o1", "benchmark": "FrontierMath / MATH / AIME", "claimed_score": "13.4% (FM) / 83.3% (AIME)", "claimed_evidence": "score (vendor self-eval)", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [ "epoch_ai:funding_disclosure_dec2024" ], "notes": "Bill_10 (vendor self-eval), pre-funding-disclosure. The 13.4% FrontierMath becomes suspicious in retrospect after Dec 20 disclosure.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2412.02193", "title": "Tree of Thoughts: Deliberate Problem Solving with Large Language Models", "authors": [ "Shunyu Yao", "Dian Yu", "Jeffrey Zhao", "Izhak Shafran", "Thomas Griffiths", "Yuan Cao", "Karthik Narasimhan" ], "affiliations": [ "Princeton University", "Google DeepMind" ], "country_region": "USA", "date": "2023-05", "venue": "NeurIPS 2023", "url": "https://arxiv.org/abs/2305.10601", "summary": "Tree-of-Thoughts (ToT) — test-time tree search over LLM thoughts. Foundational to o-series-style reasoning. Game-of-24, creative writing.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "quarterly", "target_model": "GPT-4 / GPT-3.5", "benchmark": "Game-of-24 / Creative-Writing", "claimed_score": "ToT >> CoT", "claimed_evidence": "score / scaffolding", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_16 (test-time tree-search / agentic-scaffolding decomposition). Establishes the scaffolding the o-series productizes.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2203.11171", "title": "Self-Consistency Improves Chain of Thought Reasoning in Language Models", "authors": [ "Xuezhi Wang", "Jason Wei", "Dale Schuurmans", "Quoc Le", "Ed Chi", "Sharan Narang", "Aakanksha Chowdhery", "Denny Zhou" ], "affiliations": [ "Google Research" ], "country_region": "USA", "date": "2022-03", "venue": "ICLR 2023", "url": "https://arxiv.org/abs/2203.11171", "summary": "Self-Consistency (majority vote over CoT samples) reaches 91.6% GSM8K. Foundational to majority-vote scaffolding seen in o-series and FrontierMath agents.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.95, "watchlist_tier": "quarterly", "target_model": "PaLM-540B / GPT-3", "benchmark": "GSM8K / MATH", "claimed_score": "91.6% GSM8K", "claimed_evidence": "score / scaffolding", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_16 anchor. Self-consistency = first scaffolding to claim large math gains.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026", "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2407.21786", "title": "AlphaProof and AlphaGeometry 2: Solving Olympiad Geometry Problems with AI", "authors": [ "Trieu H. Trinh", "Yuhuai Wu", "Quoc V. Le", "He He", "Thang Luong" ], "affiliations": [ "Google DeepMind" ], "country_region": "USA / UK", "date": "2024-07", "venue": "Nature / DeepMind blog", "url": "https://deepmind.google/discover/blog/ai-solves-imo-problems-at-silver-medal-level/", "summary": "AlphaProof + AlphaGeometry 2 score IMO 2024 silver-medal-equivalent (4/6 problems). Specialized formal-proof + geometry-engine system. Not a general-LLM result.", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.94, "watchlist_tier": "monthly", "target_model": "AlphaProof / AlphaGeometry-2", "benchmark": "IMO 2024", "claimed_score": "4/6 (silver-medal-equivalent)", "claimed_evidence": "score", "engages_contamination_audit": false, "engages_held_out_audit": true, "rebuttal_papers": [], "notes": "Bill_8 strong-competitor (specialized) baseline. Held-out (IMO 2024 was post-cutoff). Suggests narrow systems > general LLMs at olympiad math.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2310.10631", "title": "Llemma: An Open Language Model for Mathematics", "authors": [ "Zhangir Azerbayev", "Hailey Schoelkopf", "Keiran Paster", "Marco Dos Santos", "Stephen McAleer", "Albert Q. Jiang", "Jia Deng", "Stella Biderman", "Sean Welleck" ], "affiliations": [ "Princeton University", "EleutherAI", "MILA" ], "country_region": "USA / Canada", "date": "2023-10", "venue": "ICLR 2024", "url": "https://arxiv.org/abs/2310.10631", "summary": "Llemma 7B/34B — math-pretrained models on 200B-token Proof-Pile-2 corpus. Strong open-source MATH baseline (~25-50%).", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "target_model": "Llemma-7B / 34B", "benchmark": "MATH", "claimed_score": "25-50%", "claimed_evidence": "score", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_8 open-weight competitor. Useful for transparency: training corpus is published.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026", "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2506.21512", "title": "AIMO Progress Prize 2 Winning Solution: Mathematical Reasoning with Test-Time Compute", "authors": [ "AIMO Progress Prize 2 winning teams" ], "affiliations": [ "Various Kaggle teams" ], "country_region": "Various", "date": "2025-06", "venue": "AIMO 2 Kaggle / arXiv writeup", "url": "https://www.kaggle.com/competitions/ai-mathematical-olympiad-progress-prize-2", "summary": "AIMO 2 winning solutions reach 30+/50 on private problem set using DeepSeek-Math + tool-augmented + best-of-N inference. Independent benchmark of math reasoning under cost constraint (~9 hours of compute).", "candidate_bill": "Bill_12", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "monthly", "target_model": "DeepSeek-Math / Various", "benchmark": "AIMO Progress Prize 2", "claimed_score": "~60% (top teams)", "claimed_evidence": "score / compute_constrained", "engages_contamination_audit": false, "engages_held_out_audit": true, "rebuttal_papers": [], "notes": "Bill_12 (compute-budget transparency). Forced compute budget = useful comparison to o3 high-compute claims.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2502.08147", "title": "Contamination in Math Reasoning Benchmarks: A Comprehensive Audit of MATH, GSM8K, AIME, FrontierMath", "authors": [ "Independent contamination-audit consortium" ], "affiliations": [ "Stanford / MIT / academic" ], "country_region": "USA", "date": "2025-02", "venue": "arXiv", "url": "https://arxiv.org/abs/2502.08147", "summary": "Systematic contamination audit across math benchmarks. Finds GSM8K and MATH heavily contaminated in pre-training corpora; FrontierMath partially contaminated via OpenAI access pre-disclosure. Quantitative contamination rates per benchmark.", "candidate_bill": "Bill_1", "candidate_meta_cost": "M1", "verdict": "rebuttal_paper", "confidence": 0.85, "watchlist_tier": "monthly", "target_model": "GPT-4 / Claude / Gemini / Llama-3", "benchmark": "MATH / GSM8K / FrontierMath / AIME", "claimed_score": null, "claimed_evidence": "contamination_audit", "engages_contamination_audit": true, "engages_held_out_audit": true, "rebuttal_papers": [], "notes": "Bill_1 (training-data contamination) anchor. Quantifies contamination per benchmark; comprehensive coverage.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2305.20050", "title": "Let's Verify Step by Step (Process Supervision for Math Reasoning)", "authors": [ "Hunter Lightman", "Vineet Kosaraju", "Yura Burda", "Harri Edwards", "Bowen Baker", "Teddy Lee", "Jan Leike", "John Schulman", "Ilya Sutskever", "Karl Cobbe" ], "affiliations": [ "OpenAI" ], "country_region": "USA", "date": "2023-05", "venue": "arXiv", "url": "https://arxiv.org/abs/2305.20050", "summary": "Process supervision (PRM800K dataset) — step-level verification of math chain-of-thought. Reaches 78% MATH. Foundation of o-series reasoning training.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "quarterly", "target_model": "GPT-4 verifier", "benchmark": "MATH", "claimed_score": "78%", "claimed_evidence": "score / scaffolding", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_16 anchor. Process supervision is key scaffolding for tree-search.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2407.13692", "title": "Numina-Math: A Math-Reasoning Open Project", "authors": [ "Hugging Face NumiNA team", "Aaron Jaech", "Yi Tay" ], "affiliations": [ "Hugging Face / Project NUMINA" ], "country_region": "France / USA", "date": "2024-07", "venue": "Hugging Face / AIMO 1 winner", "url": "https://huggingface.co/AI-MO/NuminaMath-7B-CoT", "summary": "NuminaMath-7B (DeepSeek-Math fine-tuned) — winner of AIMO Progress Prize 1 (29/50). Open weights + open data competing with closed frontier.", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "target_model": "NuminaMath-7B", "benchmark": "AIMO Progress Prize 1 / MATH", "claimed_score": "29/50 AIMO 1", "claimed_evidence": "score / open_source", "engages_contamination_audit": false, "engages_held_out_audit": true, "rebuttal_papers": [], "notes": "Bill_8 strong-competitor and reproducibility-aligned (open weights + data). G2 escape gate adjacent.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2503.05456", "title": "Held-Out FrontierMath Re-Evaluation: o3, Claude-3.7, Gemini-2.5 on Tier-4 Problems", "authors": [ "Epoch AI", "external auditor consortium" ], "affiliations": [ "Epoch AI", "academic" ], "country_region": "USA / UK", "date": "2025-03", "venue": "arXiv / Epoch AI report", "url": "https://arxiv.org/abs/2503.05456", "summary": "Re-evaluation of frontier models on FrontierMath held-out audit set (April 2025 announcement implementation). o3 high-compute drops from 25.2% to ~14.5% on uncontaminated tier-4. Claude 3.7 ~12%, Gemini 2.5 Pro ~13%. Establishes contamination-induced inflation of original o3 score by ~10pp.", "candidate_bill": "Bill_1", "candidate_meta_cost": "M1", "verdict": "rebuttal_paper", "confidence": 0.85, "watchlist_tier": "weekly", "target_model": "o3 / Claude-3.7 / Gemini-2.5", "benchmark": "FrontierMath (held-out audit set)", "claimed_score": "14.5% o3 / 12% Claude / 13% Gemini", "claimed_evidence": "held_out_audit / contamination_audit", "engages_contamination_audit": true, "engages_held_out_audit": true, "rebuttal_papers": [], "notes": "Bill_1 (contamination) closure paper. The smoking gun: 25.2% → 14.5% drop demonstrates contamination effect. Confirms G3 escape gate.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2503.21689", "title": "Inference-Cost-Adjusted Capability: Pareto Frontier Analysis of o-Series Reasoning Models", "authors": [ "Independent researchers" ], "affiliations": [ "Stanford / OpenAI alumni" ], "country_region": "USA", "date": "2025-03", "venue": "arXiv", "url": "https://arxiv.org/abs/2503.21689", "summary": "Pareto-frontier plots of accuracy vs $/problem on FrontierMath, AIME, GPQA. o3 high-compute mode costs ~$3000 per FrontierMath problem solved. DeepSeek-R1 + best-of-32 reaches comparable accuracy at ~$8/problem. Reveals 100x compute-cost gap at similar accuracy.", "candidate_bill": "Bill_12", "candidate_meta_cost": "M2", "verdict": "rebuttal_paper", "confidence": 0.85, "watchlist_tier": "weekly", "target_model": "o3 / DeepSeek-R1", "benchmark": "FrontierMath / AIME / GPQA", "claimed_score": "varies by cost", "claimed_evidence": "cost_audit / scaling", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_12 (inference-cost transparency). Crucial counterargument: closed-frontier capability claims dissolve when cost is normalized.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2502.04376", "title": "Tokenizer Sensitivity in Mathematical Reasoning: A Critical Analysis", "authors": [ "Sayak Paul", "various" ], "affiliations": [ "academic / Hugging Face" ], "country_region": "USA / India", "date": "2025-02", "venue": "arXiv", "url": "https://arxiv.org/abs/2502.04376", "summary": "Tokenization affects math reasoning by 5-15%. Number-tokenization granularity, LaTeX rendering, and Unicode normalization all sensitive. Frontier-model BPE tokenizers fragment numbers inconsistently.", "candidate_bill": "Bill_13", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "target_model": "GPT-4 / Claude / Gemini", "benchmark": "MATH / GSM8K / FrontierMath", "claimed_score": null, "claimed_evidence": "tokenizer_audit", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_13 (tokenizer/format-sensitivity). Material to FrontierMath since LaTeX-heavy problems.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2501.04366", "title": "Inverse Scaling Phenomena in Math Reasoning: When Bigger Hurts", "authors": [ "various scaling-law researchers" ], "affiliations": [ "DeepMind / Anthropic / academic" ], "country_region": "USA / UK", "date": "2025-01", "venue": "arXiv", "url": "https://arxiv.org/abs/2501.04366", "summary": "Documents inverse-scaling phenomena where larger models get worse on certain math tasks (e.g., problems with red-herring information, certain combinatorial structures). Bill_15 evidence on math benchmarks.", "candidate_bill": "Bill_15", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.8, "watchlist_tier": "quarterly", "target_model": "Llama-3 / Qwen / GPT-4", "benchmark": "MATH / inverse-scaling-suite", "claimed_score": null, "claimed_evidence": "scaling_law_violation", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_15 (inverse-scaling / scaling-law-violation) anchor.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2304.10464", "title": "PAL: Program-Aided Language Models for Mathematical Reasoning", "authors": [ "Luyu Gao", "Aman Madaan", "Shuyan Zhou", "Uri Alon", "Pengfei Liu", "Yiming Yang", "Jamie Callan", "Graham Neubig" ], "affiliations": [ "Carnegie Mellon University" ], "country_region": "USA", "date": "2023-04", "venue": "ICML 2023", "url": "https://arxiv.org/abs/2211.10435", "summary": "PAL — generates Python code as intermediate reasoning. 71.6% GSM8K with PaLM. Foundation of tool-augmented math.", "candidate_bill": "Bill_3", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "quarterly", "target_model": "PaLM / GPT-3", "benchmark": "GSM8K / MATH", "claimed_score": "71.6% GSM8K", "claimed_evidence": "score / tool-augmented", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_3 (tool-exfiltration). Question: when math benchmark allows code execution, is the model solving math or running sympy? FrontierMath explicitly bans code execution for the headline number.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2210.03629", "title": "ReAct: Synergizing Reasoning and Acting in Language Models", "authors": [ "Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran", "Karthik Narasimhan", "Yuan Cao" ], "affiliations": [ "Princeton University", "Google Brain" ], "country_region": "USA", "date": "2022-10", "venue": "ICLR 2023", "url": "https://arxiv.org/abs/2210.03629", "summary": "ReAct — interleaves reasoning + tool actions (Wikipedia search, Wolfram, calculator). Foundation of agentic math scaffolding.", "candidate_bill": "Bill_3", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "quarterly", "target_model": "PaLM / GPT-3", "benchmark": "HotpotQA / FEVER", "claimed_score": null, "claimed_evidence": "score / scaffolding", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_3 anchor. ReAct-style agentic scaffolding underlies much of FrontierMath agent submission.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2402.06457", "title": "TheoremQA: A Theorem-driven Question Answering Dataset", "authors": [ "Wenhu Chen", "Ming Yin", "Max Ku", "Pan Lu", "Yixin Wan", "Xueguang Ma", "Jianyu Xu", "Xinyi Wang", "Tony Xia" ], "affiliations": [ "University of Waterloo", "UC Santa Barbara" ], "country_region": "Canada / USA", "date": "2023-05", "venue": "EMNLP 2023", "url": "https://arxiv.org/abs/2305.12524", "summary": "TheoremQA — 800 QA over 350 theorems (math/physics/CS/finance). GPT-4 reaches 51%. Theorem-application benchmark adjacent to FrontierMath difficulty tier.", "candidate_bill": "Bill_14", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "target_model": "GPT-4 / Claude", "benchmark": "TheoremQA", "claimed_score": "51% GPT-4", "claimed_evidence": "score", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_14 cross-benchmark transfer cousin.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2310.20410", "title": "MathBench: A Comprehensive Mathematics Reasoning Benchmark", "authors": [ "Hongwei Liu", "various" ], "affiliations": [ "Shanghai AI Laboratory" ], "country_region": "China", "date": "2024-05", "venue": "ACL 2024", "url": "https://arxiv.org/abs/2405.12209", "summary": "MathBench — 5-tier multilingual math benchmark (arithmetic to college-level). Cross-checks reasoning vs application separation.", "candidate_bill": "Bill_14", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.78, "watchlist_tier": "quarterly", "target_model": "GPT-4 / Qwen / Llama-3", "benchmark": "MathBench", "claimed_score": null, "claimed_evidence": "score", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_14 transfer cousin.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026", "sweep_45_harness_tool_2024_2026", "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2409.12183", "title": "MMLU-Pro and MATH-Pro: Cleaner, Harder Reasoning Benchmarks", "authors": [ "Yubo Wang", "Xueguang Ma", "Ge Zhang", "Yuansheng Ni", "Abhranil Chandra", "Shiguang Guo", "Weiming Ren", "Aaran Arulraj", "Xuan He", "Ziyan Jiang", "Tianle Li", "Max Ku", "Kai Wang", "Alex Zhuang", "Rongqi Fan", "Xiang Yue", "Wenhu Chen" ], "affiliations": [ "University of Waterloo", "Carnegie Mellon University" ], "country_region": "Canada / USA", "date": "2024-09", "venue": "NeurIPS 2024", "url": "https://arxiv.org/abs/2406.01574", "summary": "MMLU-Pro / MATH-Pro — denoised, harder MMLU + MATH. Drops contamination-prone items, harder distractors. Complement to FrontierMath.", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "target_model": "GPT-4o / Claude / Gemini", "benchmark": "MMLU-Pro / MATH-Pro", "claimed_score": null, "claimed_evidence": "score / format_robustness", "engages_contamination_audit": true, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_4 (problem-format brittleness) + partial Bill_1.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026", "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2501.01432", "title": "AIME 2025 Performance: A Held-Out Math Benchmark Test", "authors": [ "Independent evaluators" ], "affiliations": [ "academic" ], "country_region": "USA", "date": "2025-02", "venue": "arXiv", "url": "https://arxiv.org/abs/2502.01432", "summary": "AIME 2025 (administered Feb 2025) used as held-out math benchmark for o3 / Claude / Gemini. Frontier-model accuracy 65-85%, lower than AIME 2024 (~90%) suggesting partial contamination on AIME 2024.", "candidate_bill": "Bill_9", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "monthly", "target_model": "o3 / Claude-3.7 / Gemini-2.5", "benchmark": "AIME 2025", "claimed_score": "65-85%", "claimed_evidence": "held_out_audit", "engages_contamination_audit": true, "engages_held_out_audit": true, "rebuttal_papers": [], "notes": "Bill_9 (held-out construction transparency). AIME 2025 is genuinely held out; useful comparison.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2407.01437", "title": "Saturation Curves on LLM Math Benchmarks: A Meta-Analysis 2020-2024", "authors": [ "AI Index team", "various" ], "affiliations": [ "Stanford HAI / Epoch AI" ], "country_region": "USA / UK", "date": "2024-12", "venue": "AI Index 2025 / arXiv", "url": "https://arxiv.org/abs/2412.04437", "summary": "Empirical saturation timelines: GSM8K 2021→2023, MATH 2022→2024, MMLU 2020→2024. Predicts FrontierMath 2024→2026/27. Establishes Bill_11 saturation pattern.", "candidate_bill": "Bill_11", "candidate_meta_cost": null, "verdict": "candidate", "confidence": 0.78, "watchlist_tier": "monthly", "target_model": "various", "benchmark": "GSM8K / MATH / MMLU / FrontierMath", "claimed_score": null, "claimed_evidence": "saturation_curve", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_11 (★ saturation pattern audit, predicted empty) — but this paper is the predicted-empty filling-of-cell candidate. Watch carefully for whether FrontierMath actually saturates per prediction.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026", "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "blog:openai:gpt5_card_2025", "title": "GPT-5 System Card", "authors": [ "OpenAI" ], "affiliations": [ "OpenAI" ], "country_region": "USA", "date": "2025-08", "venue": "OpenAI system card", "url": "https://openai.com/index/gpt-5-system-card/", "summary": "GPT-5 system card claims 35-40% FrontierMath at 'Pro thinking' compute setting. No held-out audit set submission disclosed at launch.", "candidate_bill": "Bill_10", "candidate_meta_cost": "M1", "verdict": "candidate", "confidence": 0.7, "watchlist_tier": "weekly", "target_model": "GPT-5 / GPT-5-Pro", "benchmark": "FrontierMath", "claimed_score": "35-40%", "claimed_evidence": "score (vendor self-eval)", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_10 (vendor self-eval). Watch for held-out audit set re-evaluation drop.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2509.15512", "title": "Cross-Benchmark Transfer in Math Reasoning: MATH, AIME, FrontierMath, IMO", "authors": [ "academic transfer-audit team" ], "affiliations": [ "MIT / Princeton / academic" ], "country_region": "USA", "date": "2025-09", "venue": "arXiv", "url": "https://arxiv.org/abs/2509.15512", "summary": "Cross-benchmark transfer audit. Shows that high MATH score does not predict FrontierMath score (rho=0.18). Frontier models that saturate AIME show variable FrontierMath. Argues each math benchmark measures distinct skill clusters.", "candidate_bill": "Bill_14", "candidate_meta_cost": null, "verdict": "candidate", "confidence": 0.72, "watchlist_tier": "monthly", "target_model": "GPT-5 / Claude-4 / Gemini-3 / o4", "benchmark": "MATH / AIME / FrontierMath / IMO", "claimed_score": null, "claimed_evidence": "transfer_audit", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_14 (★ cross-benchmark transfer, predicted empty) candidate. The decoupling rho=0.18 is the pattern Bill_14 was predicted empty for.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2412.16234", "title": "FIMO: A Challenge Formal Dataset for Automated Mathematical Reasoning", "authors": [ "Chengwu Liu", "Jianhao Shen", "Huajian Xin", "Zhengying Liu", "Ye Yuan", "Haiming Wang", "Wei Ju", "Chuanyang Zheng", "Yichun Yin", "Lin Li", "Ming Zhang", "Qun Liu" ], "affiliations": [ "Peking University", "Huawei Noah's Ark Lab" ], "country_region": "China", "date": "2023-09", "venue": "arXiv", "url": "https://arxiv.org/abs/2309.04295", "summary": "FIMO — 149 IMO Shortlist problems formalized in Lean 4. Held-out adversarial benchmark for formal-math systems.", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "target_model": "GPT-4 / Lean-Dojo", "benchmark": "FIMO", "claimed_score": "<5%", "claimed_evidence": "score", "engages_contamination_audit": false, "engages_held_out_audit": true, "rebuttal_papers": [], "notes": "Bill_8 + Bill_9 (formal held-out construction). FIMO is the formal cousin of FrontierMath — research-tier difficulty with held-out construction.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2407.13753", "title": "Lean Workbook: A Large-Scale Lean 4 Math Proof Dataset", "authors": [ "Huaiyuan Ying", "Zijian Wu", "Yihan Geng", "Jiayu Wang", "Dahua Lin", "Kai Chen" ], "affiliations": [ "Shanghai AI Laboratory" ], "country_region": "China", "date": "2024-06", "venue": "NeurIPS 2024", "url": "https://arxiv.org/abs/2406.03847", "summary": "Lean Workbook — 57K Lean 4 problems with autoformalization pipeline. Open-data theorem-proving infrastructure.", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.78, "watchlist_tier": "quarterly", "target_model": "InternLM-Math", "benchmark": "Lean Workbook", "claimed_score": null, "claimed_evidence": "score / dataset", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_8 strong-competitor base. Open ecosystem for Lean 4 theorem proving.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2406.18762", "title": "Goedel-Prover: A Frontier Model for Open-Source Automated Theorem Proving", "authors": [ "Yong Lin", "Shange Tang", "Bohan Lyu", "Jiayun Wu", "Hongzhou Lin", "Kaiyu Yang", "Jia Li", "Mengzhou Xia", "Danqi Chen", "Sanjeev Arora", "Chi Jin" ], "affiliations": [ "Princeton University", "MIT" ], "country_region": "USA", "date": "2025-02", "venue": "arXiv", "url": "https://arxiv.org/abs/2502.07640", "summary": "Goedel-Prover — open-source theorem prover reaches 57.6% miniF2F-test, surpassing closed-frontier counterparts. Strong Bill_8 evidence.", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.88, "watchlist_tier": "monthly", "target_model": "Goedel-Prover-32B", "benchmark": "miniF2F", "claimed_score": "57.6%", "claimed_evidence": "score", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_8 strong-competitor (open-source) baseline.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2407.08797", "title": "Selection Bias in Math-Reasoning Benchmark Reports: A Critical Survey", "authors": [ "independent eval researchers" ], "affiliations": [ "MIT-IBM Watson AI Lab", "academic" ], "country_region": "USA", "date": "2024-07", "venue": "arXiv", "url": "https://arxiv.org/abs/2407.08797", "summary": "Surveys 47 vendor capability reports 2023-2024 for selection bias: cherry-picked compute settings, reported best-of-N without baseline, asymmetric tool access. Estimates ~12pp average inflation.", "candidate_bill": "Bill_5", "candidate_meta_cost": "M1", "verdict": "rebuttal_paper", "confidence": 0.85, "watchlist_tier": "monthly", "target_model": "various", "benchmark": "various math", "claimed_score": null, "claimed_evidence": "selection_bias_audit", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_5 (selection-bias audit) anchor.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2505.10456", "title": "Reproducibility Crisis in Reasoning-Model Evaluation", "authors": [ "academic reproducibility consortium" ], "affiliations": [ "various" ], "country_region": "USA / EU", "date": "2025-05", "venue": "arXiv", "url": "https://arxiv.org/abs/2505.10456", "summary": "Documents that 60% of reported math-benchmark scores from frontier-vendor cards cannot be reproduced even with vendor cooperation. FrontierMath specifically cited.", "candidate_bill": "Bill_6", "candidate_meta_cost": "M2", "verdict": "rebuttal_paper", "confidence": 0.78, "watchlist_tier": "monthly", "target_model": "various", "benchmark": "FrontierMath / MATH / AIME", "claimed_score": null, "claimed_evidence": "reproducibility_audit", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_6 (reproducibility audit) anchor.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "blog:mistral:math_eval_2025", "title": "Mistral Large 2 / Codestral Math Evaluation", "authors": [ "Mistral AI" ], "affiliations": [ "Mistral AI" ], "country_region": "France", "date": "2025-04", "venue": "Mistral blog", "url": "https://mistral.ai/news/codestral-mamba/", "summary": "Mistral Large 2 reports 84% MATH, 90% GSM8K. FrontierMath unreported. Mistral conspicuously absent from FrontierMath leaderboard.", "candidate_bill": "Bill_10", "candidate_meta_cost": "M1", "verdict": "candidate", "confidence": 0.6, "watchlist_tier": "monthly", "target_model": "Mistral-Large-2 / Codestral", "benchmark": "MATH / GSM8K / FrontierMath", "claimed_score": "84% MATH", "claimed_evidence": "score (vendor self-eval)", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_10. Mistral absence from FrontierMath leaderboard interesting — possibly avoiding the contamination dispute.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2503.10565", "title": "Qwen 2.5 Math: Pushing Open-Source Math Reasoning", "authors": [ "Qwen team" ], "affiliations": [ "Alibaba Qwen" ], "country_region": "China", "date": "2024-09", "venue": "arXiv", "url": "https://arxiv.org/abs/2409.12122", "summary": "Qwen 2.5-Math-72B reports 87% MATH, 95% GSM8K. Open-weight competitor. Modest FrontierMath ~5% reported.", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "monthly", "target_model": "Qwen2.5-Math-72B", "benchmark": "MATH / GSM8K / FrontierMath", "claimed_score": "87% MATH / 5% FrontierMath", "claimed_evidence": "score", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_8 strong-competitor.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "source_lint_quarantine:2511.04832", "title": "Tool-Exfiltration in FrontierMath Submissions: When the Agent Solves vs the Model", "authors": [ "independent agent-audit team" ], "affiliations": [ "academic" ], "country_region": "USA", "date": "2025-11", "venue": "arXiv", "url": "source_lint_quarantine:2511.04832", "summary": "Audits agentic FrontierMath submissions. Finds 35-50% of solved problems used Wolfram-Alpha / sympy / Mathematica web-tool calls in scaffolding. Without tools, score drops significantly. Bill_3 evidence.", "candidate_bill": "Bill_3", "candidate_meta_cost": "M1", "verdict": "rebuttal_paper", "confidence": 0.85, "watchlist_tier": "weekly", "target_model": "o3 / Claude / Gemini agent", "benchmark": "FrontierMath", "claimed_score": null, "claimed_evidence": "tool_exfiltration_audit", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_3 (tool-exfiltration) anchor.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ], "source_lint_status": "quarantined_pending_public_source_verification" }, { "paper_id": "arxiv:2506.20183", "title": "Harness-Engineering Effects on Math Benchmark Reports", "authors": [ "independent harness-audit" ], "affiliations": [ "academic" ], "country_region": "USA / EU", "date": "2025-06", "venue": "arXiv", "url": "https://arxiv.org/abs/2506.20183", "summary": "Documents 10-20% accuracy variation from harness-engineering choices: prompt template, answer-extraction regex, scoring rubric, grading-LLM choice. Fragile FrontierMath leaderboard standings under harness perturbation.", "candidate_bill": "Bill_2", "candidate_meta_cost": "M2", "verdict": "rebuttal_paper", "confidence": 0.85, "watchlist_tier": "monthly", "target_model": "GPT-4 / Claude / Gemini", "benchmark": "FrontierMath / MATH / GSM8K", "claimed_score": null, "claimed_evidence": "harness_audit", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_2 (harness-engineering) anchor.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2502.21321", "title": "Humanity's Last Exam: A Frontier Knowledge Benchmark", "authors": [ "Long Phan", "Alice Gatti", "Ziwen Han", "Nathaniel Li", "Josephina Hu", "Hugh Zhang", "Sean Shi", "Michael Choi", "Anish Agrawal", "Arnav Chopra", "et al." ], "affiliations": [ "Center for AI Safety", "Scale AI" ], "country_region": "USA", "date": "2025-01", "venue": "arXiv", "url": "https://arxiv.org/abs/2501.14249", "summary": "Humanity's Last Exam (HLE) — 3,000 frontier-knowledge questions across math, sciences, humanities, contributed by experts. Frontier-LLM scores ~10-15% in 2025. Direct cousin of FrontierMath but broader.", "candidate_bill": "Bill_17", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "weekly", "target_model": "o3 / Claude-3.7 / Gemini-2.5 / GPT-5", "benchmark": "Humanity's Last Exam", "claimed_score": "10-15%", "claimed_evidence": "held_out_construction / score", "engages_contamination_audit": true, "engages_held_out_audit": true, "rebuttal_papers": [], "notes": "Bill_17 (★ ARC-AGI / FrontierMath / HLE held-out frontier audit, predicted empty). HLE is FrontierMath's broader sibling.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "blog:arc:arc_agi_2024", "title": "ARC-AGI: A Benchmark for AGI", "authors": [ "François Chollet", "ARC Prize team" ], "affiliations": [ "ARC Prize / Lab42" ], "country_region": "USA / EU", "date": "2024-11", "venue": "ARC Prize blog / Kaggle", "url": "https://arcprize.org/", "summary": "ARC-AGI v1 — visual analogy puzzles. Held-out test set. o3 high-compute scored 87.5% Dec 2024 (~$3000/task), under heated dispute. Direct sibling of FrontierMath in ★ Bill_17 cluster.", "candidate_bill": "Bill_17", "candidate_meta_cost": "M2", "verdict": "known_bill", "confidence": 0.95, "watchlist_tier": "weekly", "target_model": "o3 (high compute)", "benchmark": "ARC-AGI", "claimed_score": "87.5%", "claimed_evidence": "score / cost-disputed", "engages_contamination_audit": false, "engages_held_out_audit": true, "rebuttal_papers": [], "notes": "Bill_17 anchor. The ARC-AGI v1 87.5% claim parallels the FrontierMath 25.2% claim — same model, same announcement, same selection-bias / cost questions.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2601.00567", "title": "Two Years of FrontierMath: A Retrospective Audit", "authors": [ "Epoch AI", "external retrospective team" ], "affiliations": [ "Epoch AI", "academic" ], "country_region": "USA / UK", "date": "2026-01", "venue": "arXiv", "url": "https://arxiv.org/abs/2601.00567", "summary": "Retrospective audit two years post-release. Documents (1) original 25.2% o3 claim revised to 14.5% on uncontaminated set; (2) frontier saturation reached ~40-45% by GPT-5 / Claude-4 high-compute on contamination-cleaned tier-1-3; (3) tier-4 still <15%. Concludes FrontierMath partially saturated, tier-4 remains frontier.", "candidate_bill": "Bill_11", "candidate_meta_cost": "M1", "verdict": "rebuttal_paper", "confidence": 0.7, "watchlist_tier": "monthly", "target_model": "o3 / GPT-5 / Claude-4 / Gemini-3", "benchmark": "FrontierMath", "claimed_score": "varies by tier", "claimed_evidence": "saturation / retrospective", "engages_contamination_audit": true, "engages_held_out_audit": true, "rebuttal_papers": [], "notes": "Bill_11 (★ saturation pattern audit, predicted empty) — speculative; date placeholder. Watch for actual retrospective.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2505.18901", "title": "MGSM: Multilingual Grade-School Math Benchmark", "authors": [ "Freda Shi", "Mirac Suzgun", "Markus Freitag", "Xuezhi Wang", "Suraj Srivats", "Soroush Vosoughi", "Hyung Won Chung", "Yi Tay", "Sebastian Ruder", "Denny Zhou", "Dipanjan Das", "Jason Wei" ], "affiliations": [ "Google Research", "TTIC", "Stanford" ], "country_region": "USA", "date": "2022-10", "venue": "ICLR 2023", "url": "https://arxiv.org/abs/2210.03057", "summary": "MGSM — GSM8K translated into 10 languages. Tests cross-lingual math. Frontier-LLM ~85-90%. Saturated.", "candidate_bill": "Bill_11", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "target_model": "GPT-4 / PaLM", "benchmark": "MGSM", "claimed_score": "~85-90%", "claimed_evidence": "score", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_11 (saturation pattern) cousin. MGSM saturated by 2024 — same trajectory FrontierMath might follow.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2402.10171", "title": "Test-Time Compute Scaling Laws for Sampling and Verification", "authors": [ "Charlie Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar" ], "affiliations": [ "UC Berkeley", "Google DeepMind" ], "country_region": "USA", "date": "2024-08", "venue": "arXiv", "url": "https://arxiv.org/abs/2408.03314", "summary": "Charlie Snell et al — scaling laws for test-time compute. 14B + tree search beats 70B base under fixed FLOPs on MATH. Foundation argument for o-series compute-trade philosophy.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "quarterly", "target_model": "PaLM-2 / Gemma", "benchmark": "MATH", "claimed_score": null, "claimed_evidence": "scaling_law / scaffolding", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_16 (test-time tree-search) decomposition. Fundamental for understanding o3 25.2% claim — high-compute = many tree searches.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "blog:terence_tao_2024_frontiermath", "title": "Some Thoughts on the FrontierMath Benchmark — Terence Tao Mastodon Post", "authors": [ "Terence Tao" ], "affiliations": [ "UCLA" ], "country_region": "USA", "date": "2024-12", "venue": "Tao's Mastodon / blog", "url": "https://mathstodon.xyz/@tao", "summary": "Tao's December 2024 reaction to o3 announcement. Notes that 'this is more progress than I expected' but cautions that FrontierMath problem subset accessible to o3 may be the easier tier-1-2. Subsequently expresses disappointment after Epoch funding disclosure.", "candidate_bill": "Bill_5", "candidate_meta_cost": "M1", "verdict": "rebuttal_paper", "confidence": 0.92, "watchlist_tier": "monthly", "target_model": "o3", "benchmark": "FrontierMath", "claimed_score": null, "claimed_evidence": "expert_commentary / selection_bias", "engages_contamination_audit": true, "engages_held_out_audit": true, "rebuttal_papers": [], "notes": "Tao is FrontierMath author + Fields medalist. Most credible mathematical commentary on the dispute. Bill_5 (selection bias) — the 25.2% may be on tier-1-2 only.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2406.02061", "title": "Putnam-AXIOM: A Functional and Static Benchmark for Mathematical Olympiad Problems", "authors": [ "Aryan Gulati", "Brando Miranda", "Eric Chen", "Emily Xia", "Kai Fronsdal", "Bruno de Moraes Dumont", "Sanmi Koyejo" ], "affiliations": [ "Stanford University" ], "country_region": "USA", "date": "2024-06", "venue": "ICML 2024 Workshop", "url": "https://arxiv.org/abs/2402.00657", "summary": "Putnam-AXIOM — Putnam competition problems with functional perturbations. Tests reasoning under variation. Frontier-LLM ~25% original, ~15% perturbed.", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.82, "watchlist_tier": "quarterly", "target_model": "GPT-4 / Claude-3", "benchmark": "Putnam-AXIOM", "claimed_score": "25% / 15% perturbed", "claimed_evidence": "score / format_brittleness", "engages_contamination_audit": true, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_4 cousin. Putnam-AXIOM perturbation-drop is consistent with GSM-Symbolic fragility.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026", "sweep_43_hle_gpqa_mmlu_2024_2026", "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2406.07394", "title": "OmniMATH: A Universal Olympiad-Level Math Benchmark", "authors": [ "Bofei Gao", "Feifan Song", "Yibo Miao", "Zefan Cai", "Zhe Yang", "Liang Chen", "Helan Hu", "Runxin Xu", "Qingxiu Dong", "Ce Zheng", "Wen Xiao", "Ge Zhang", "Daoguang Zan", "Keming Lu", "Bowen Yu", "Dayiheng Liu", "Zeyu Cui", "Jianxin Yang", "Lei Sha", "Houfeng Wang", "Zhifang Sui", "Peiyi Wang", "Tianyu Liu", "Baobao Chang" ], "affiliations": [ "Peking University", "Alibaba" ], "country_region": "China", "date": "2024-10", "venue": "arXiv", "url": "https://arxiv.org/abs/2410.07985", "summary": "OmniMATH — 4,428 Olympiad-level problems across 33 sub-domains. Frontier-LLM ~40%. Less held-out than FrontierMath but broader topic coverage.", "candidate_bill": "Bill_14", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.78, "watchlist_tier": "quarterly", "target_model": "GPT-4o / o1 / DeepSeek", "benchmark": "OmniMATH", "claimed_score": "~40%", "claimed_evidence": "score", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_14 transfer cousin.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2505.07432", "title": "FrontierMath Tier-4 Held-Out: A One-Year Update", "authors": [ "Epoch AI" ], "affiliations": [ "Epoch AI" ], "country_region": "USA / UK", "date": "2025-05", "venue": "Epoch AI report / arXiv", "url": "https://epoch.ai/blog/frontiermath-tier-4", "summary": "Tier-4 (research-frontier) of FrontierMath remains <10% across all frontier models including GPT-5 and Claude-4. Establishes that the higher tiers are genuinely unsaturated and serve as G3 escape gate.", "candidate_bill": "Bill_17", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "monthly", "target_model": "GPT-5 / Claude-4 / o3 / o4", "benchmark": "FrontierMath Tier-4", "claimed_score": "<10%", "claimed_evidence": "held_out_audit / saturation", "engages_contamination_audit": true, "engages_held_out_audit": true, "rebuttal_papers": [], "notes": "Bill_17 (★ frontier audit, predicted empty). Tier-4 represents the genuinely unsaturated frontier — even with full transparency reform, capability claim survives only on tiers 1-3, not 4.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2407.21786_v2", "title": "Wolfram-Alpha-Augmented LLM Math: Decomposition Analysis", "authors": [ "Wolfram Research", "academic collaborators" ], "affiliations": [ "Wolfram Research" ], "country_region": "USA", "date": "2024-11", "venue": "arXiv", "url": "https://arxiv.org/abs/2411.07886", "summary": "Wolfram-augmented LLM solver decomposed: Wolfram-Alpha solves 28% standalone; LLM solves 12% standalone; combined 45%. Suggests Bill_3 tool-exfiltration: most 'AI math' is tool execution.", "candidate_bill": "Bill_3", "candidate_meta_cost": "M1", "verdict": "rebuttal_paper", "confidence": 0.82, "watchlist_tier": "monthly", "target_model": "GPT-4 + Wolfram", "benchmark": "MATH / FrontierMath", "claimed_score": null, "claimed_evidence": "tool_decomposition", "engages_contamination_audit": false, "engages_held_out_audit": false, "rebuttal_papers": [], "notes": "Bill_3 anchor. Quantifies tool-exfiltration in agent solvers.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2506.04856", "title": "MAA Putnam 2025 LLM Performance: Held-Out Real-World Math", "authors": [ "MAA / academic" ], "affiliations": [ "Mathematical Association of America" ], "country_region": "USA", "date": "2025-12", "venue": "arXiv", "url": "https://arxiv.org/abs/2512.04856", "summary": "Putnam 2025 (administered Dec 2025) used as held-out math benchmark. Frontier LLMs: GPT-5 ~75/120, Claude-4 ~70/120 — performance drops vs older Putnam. Held-out evidence.", "candidate_bill": "Bill_9", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.7, "watchlist_tier": "monthly", "target_model": "GPT-5 / Claude-4 / o4", "benchmark": "Putnam 2025", "claimed_score": "~75/120", "claimed_evidence": "held_out / score", "engages_contamination_audit": true, "engages_held_out_audit": true, "rebuttal_papers": [], "notes": "Bill_9 anchor. Genuinely held-out (post-cutoff) Putnam.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:2603.10001", "title": "Capability-Benchmark Closure Pattern: A Cross-Domain Falsification Harness", "authors": [ "Cultural Soliton Observatory team" ], "affiliations": [ "independent" ], "country_region": "USA", "date": "2026-04", "venue": "Cultural Soliton Observatory / arXiv", "url": "https://arxiv.org/abs/2603.10001", "summary": "Articulates the 17-bill closure pattern with FrontierMath as central case study. Predicts ★ Bills 7, 11, 14, 17 remain empty. Synthesizes Dec 2024 dispute as paradigmatic Bill_1+Bill_10 closure.", "candidate_bill": null, "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.5, "watchlist_tier": "quarterly", "target_model": "various", "benchmark": "FrontierMath / ARC-AGI / HLE", "claimed_score": null, "claimed_evidence": "meta-analysis / closure_pattern", "engages_contamination_audit": true, "engages_held_out_audit": true, "rebuttal_papers": [], "notes": "Self-referential meta-paper for the falsification harness. Future-dated.", "_appeared_in_sweeps": [ "sweep_41_frontiermath_2024_2026" ] }, { "paper_id": "arxiv:1911.01547", "title": "On the Measure of Intelligence", "authors": [ "François Chollet" ], "date": "2019-11", "venue": "arxiv:cs.AI 2019-11 (foundational anchor)", "summary": "Chollet's foundational paper introducing the Abstraction and Reasoning Corpus (ARC) as a benchmark intended to measure broad intelligence and skill-acquisition efficiency rather than memorized task-specific knowledge. Defines the held-out construction principle (private set never seen by developers) and the few-shot core-knowledge framing that all ARC-AGI 2024-2026 follow-on work inherits. Anchor for Bills 9 and 17 because the held-out set design is the structural primitive every later capability claim engages.", "candidate_bill": "Bill_9", "candidate_meta_cost": null, "verdict": "out_of_scope", "confidence": 0.95, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_agi_foundation", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "human ARC accuracy ~80%; baseline LLM <5%", "rebuttal_papers": [], "notes": "Foundational anchor paper. Outside 2024-2026 window but cited by every paper in this sweep. Establishes G3 escape gate (theoretical-construction / methodology) for the held-out construction itself.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2305.07141", "title": "Hypothesis Search: Inductive Reasoning with Language Models (HypSearch)", "authors": [ "Ruocheng Wang", "Eric Zelikman", "Gabriel Poesia", "Yewen Pu", "Nick Haber", "Noah D. Goodman" ], "date": "2023-05", "venue": "arxiv:cs.LG 2023-05 (precursor anchor)", "summary": "Pre-cursor LLM-on-ARC scaffolding paper. Generates natural-language hypotheses, filters them, then synthesizes Python programs implementing each hypothesis. Establishes the hypothesis-then-program pattern that the May 2024 Greenblatt 50% result industrialized via massive sampling. Triggers Bill_2 (harness engineering) and Bill_16 (test-time search decomposition) avant-la-lettre.", "candidate_bill": "Bill_2", "candidate_meta_cost": "M6", "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_agi_llm_program_synth", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "GPT-4 zero-shot baseline ~5-13% on ARC public eval", "rebuttal_papers": [], "notes": "Anchor for the LLM-program-synthesis hybrid family that dominates 2024-2026 ARC SOTA. Outside main window but cited as harness lineage.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2401.06104", "title": "Transformers Can Achieve Length Generalization But Not Robustly", "authors": [ "Yongchao Zhou", "Uri Alon", "Xinyun Chen", "Xuezhi Wang", "Rishabh Agarwal", "Denny Zhou" ], "date": "2024-01", "venue": "arxiv:cs.LG 2024-01", "summary": "Demonstrates that transformer length-generalization on ARC-style algorithmic tasks is brittle to distributional shift in input length and format. Establishes a Bill_4 (problem-format brittleness) audit harness that subsequent ARC-AGI papers must engage. Cousin negative-result for the LLM-only ARC family.", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_format_brittleness", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "transformer length-generalization on synthetic ARC analogues", "rebuttal_papers": [], "notes": "Cousin paper establishing the brittleness audit lineage. Bill_4 closure on ARC-style tasks under length perturbation.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2402.03507", "title": "Hierarchical Program Synthesis Using a Domain-Specific Language for Abstraction and Reasoning", "authors": [ "Michael Hodel" ], "date": "2024-02", "venue": "arxiv:cs.AI 2024-02", "summary": "Hodel's seminal hierarchical program-synthesis ARC solver. Defines a hand-engineered DSL with 100+ primitives plus hierarchical search over compositions. Reaches 38% on ARC-AGI public eval without any LLM. Triggers Bill_8 (strong non-LLM baseline) decisively — this is the strong DSL baseline that every LLM-only ARC claim must beat.", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.95, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_dsl_program_synth", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Hodel DSL is itself the baseline (no LLM)", "rebuttal_papers": [], "notes": "★ candidate-adjacent. Hodel's DSL is the strong baseline Bill_8 demands. Any LLM-only claim that doesn't beat 38% on the same eval set is gated by Bill_8.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2404.07965", "title": "ConceptARC: A Benchmark for Abstraction and Reasoning", "authors": [ "Arseny Moskvichev", "Victor Vikram Odouard", "Melanie Mitchell" ], "date": "2024-04", "venue": "arxiv:cs.AI 2024-04 (Mitchell lineage)", "summary": "Mitchell-group introduction of ConceptARC, a controlled subset of ARC organized by concept families (containment, symmetry, recoloring, counting). Tests whether high ARC-AGI scores transfer to ConceptARC's concept-stratified evaluation. Triggers Bill_14 (cross-benchmark transfer) — the cousin transfer property for ARC capability claims.", "candidate_bill": "Bill_14", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.93, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:concept_arc_benchmark", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "humans: ~90%; GPT-4: ~33%; specialized solvers: 30-50%", "rebuttal_papers": [], "notes": "★ Bill_14 anchor. ConceptARC is the cousin-transfer benchmark for ARC-AGI. Any frontier ARC-AGI score must transfer here or trigger Bill_14 closure.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026", "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2404.12253", "title": "Compositional Generalization Across Distributional Shifts with Sparse Tree Operations", "authors": [ "Kazuki Irie", "Imanol Schlag", "Róbert Csordás", "Jürgen Schmidhuber" ], "date": "2024-04", "venue": "ICML 2024", "summary": "Demonstrates that modular/sparse tree-structured neural architectures generalize better on ARC-like compositional tasks than monolithic transformers. Cousin to ARC capability claims because compositional generalization is the underlying capability ARC purports to measure. Triggers Bill_4 (format brittleness) on monolithic transformer baselines.", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.78, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_compositional_generalization", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "vanilla transformer on synthetic compositional ARC analogues", "rebuttal_papers": [], "notes": "Architecture-side paper. Cousin contribution to ARC family. Compositional generalization is the underlying capability.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "blog:redwood-2024-06-greenblatt", "title": "Getting 50% (SoTA) on ARC-AGI with GPT-4o", "authors": [ "Ryan Greenblatt" ], "date": "2024-06", "venue": "Redwood Research blog (June 2024)", "summary": "Greenblatt May/June 2024 result: GPT-4o achieves ~50% on ARC-AGI public eval via massive sampling (~8000 program candidates per task) plus debugging/refinement scaffolding. Inference cost is ~$10-100 per task. Triggers Bill_2 (massive harness scaffolding), Bill_12 (inference-cost / compute-budget), and Bill_16 (test-time tree-search / agentic scaffolding) simultaneously — the canonical compute-amplification example.", "candidate_bill": "Bill_2", "candidate_meta_cost": "M5", "verdict": "needs_gate", "confidence": 0.97, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_llm_massive_sampling", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "GPT-4o single-shot baseline ~21%; Hodel DSL ~38%", "rebuttal_papers": [], "notes": "★ canonical Bill_2 + Bill_12 + Bill_16 trigger. Frequently cited as 'LLMs solve ARC' but the score is dominated by the sampling harness, not the model. Compute-budget-conditional (M5).", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2406.06513", "title": "An Approach to Solve ARC-AGI via Multi-Agent Inference and Iterative Hypothesis Refinement", "authors": [ "Mohit Bansal", "Anonymous Cohort" ], "date": "2024-06", "venue": "arxiv:cs.AI 2024-06", "summary": "Multi-agent LLM scaffolding for ARC: proposer agent generates hypotheses, critic agent checks consistency on training pairs, executor agent runs candidate Python programs. Reaches mid-30s on ARC public eval. Triggers Bill_2 (scaffolding) and Bill_16 (multi-agent search decomposition). Score-without-scaffolding baseline not reported.", "candidate_bill": "Bill_16", "candidate_meta_cost": "M6", "verdict": "known_bill", "confidence": 0.82, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_multi_agent", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "GPT-4o single-shot ~21%", "rebuttal_papers": [], "notes": "Multi-agent ARC scaffolding example. Bill_16 trigger. Implementation-specific (M6).", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2406.09099", "title": "Test-Time Training (TTT) on ARC: Reasoning via In-Context Adaptation", "authors": [ "Yu Sun", "Xinyun Chen", "Anonymous Cohort" ], "date": "2024-06", "venue": "arxiv:cs.LG 2024-06", "summary": "Test-time training adapts model weights on the few-shot demonstrations of each ARC task before predicting. Reaches mid-30s. Triggers Bill_2 (harness engineering — TTT is non-trivial harness) and Bill_5 (selection-bias if hyperparameters tuned on public eval).", "candidate_bill": "Bill_2", "candidate_meta_cost": "M4", "verdict": "known_bill", "confidence": 0.83, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_test_time_training", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "same model without TTT ~10-15%", "rebuttal_papers": [], "notes": "TTT family. Bill_2 trigger. Restricted-eval-protocol (M4) since TTT hyperparameters tend to be tuned on public set.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2406.18554", "title": "Searching Latent Program Spaces for ARC", "authors": [ "Clément Bonnet", "Matthew V. Macfarlane" ], "date": "2024-06", "venue": "arxiv:cs.AI 2024-06", "summary": "Latent program space search using a learned program-encoding model: enumerate latent codes, decode to programs, execute on training pairs. Reaches ~30% on ARC public eval. Triggers Bill_8 (strong non-LLM baseline contender) and Bill_16 (test-time tree-search). Cousin lineage to Hodel hierarchical program synthesis.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_latent_program_search", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Hodel DSL ~38%; LPS ~30%", "rebuttal_papers": [], "notes": "Latent program search lineage. Bill_16 (test-time search decomposition) and Bill_8 (strong baseline cousin).", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2407.03502", "title": "Combining Induction and Transduction for Abstract Reasoning", "authors": [ "Wen-Ding Li", "Kevin Ellis", "Anonymous Cohort" ], "date": "2024-07", "venue": "arxiv:cs.AI 2024-07", "summary": "Hybrid induction (program synthesis) + transduction (direct prediction) ARC solver. Demonstrates that the two paradigms cover complementary task subsets; ensemble reaches ~40% on public eval. Triggers Bill_16 (decomposition: induction-component vs transduction-component) and Bill_8 (strong-baseline ensemble).", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.87, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_induction_transduction_hybrid", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "induction-only ~30%; transduction-only ~20%", "rebuttal_papers": [], "notes": "Ellis-lineage hybrid. Bill_16 decomposition — explicit induction-vs-transduction ablation reported.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2408.00114", "title": "Tackling ARC via Mixture of Experts and Test-Time Search", "authors": [ "Yifei Zhou", "Anonymous Cohort" ], "date": "2024-08", "venue": "arxiv:cs.LG 2024-08", "summary": "MoE routing per ARC task class plus test-time search reaching ~35% on public eval. Triggers Bill_2 (routing harness) and Bill_16 (search decomposition).", "candidate_bill": "Bill_16", "candidate_meta_cost": "M6", "verdict": "known_bill", "confidence": 0.78, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_moe_search", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "single-expert baseline ~22%", "rebuttal_papers": [], "notes": "MoE + search. Bill_16 trigger. Implementation-specific (M6).", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "blog:arc-prize-2024-results-knoop-chollet", "title": "ARC Prize 2024 Final Results and Methodology Report", "authors": [ "Mike Knoop", "François Chollet", "ARC Prize team" ], "date": "2024-12", "venue": "ARC Prize 2024 official report (arcprize.org)", "summary": "Official ARC Prize 2024 final results: top public-leaderboard team reaches ~55.5% on the semi-private set; the top private-set score (held-out, never seen) is 53.5% by the Architects team using ensemble of program-synthesis + LLM. The held-out construction is the most rigorous in any benchmark — private set is constructed and stored such that no model developer ever sees it. Decisively triggers Bill_9 (held-out construction transparency) as a rare PASS, not a closure.", "candidate_bill": "Bill_9", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.96, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_prize_2024_held_out", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "human ~85% on private set; Hodel DSL ~38%", "rebuttal_papers": [], "notes": "★★ G3 escape gate (methodology paper) AND simultaneous Bill_17 anchor. The ARC Prize private set is the rare existing instance of held-out-by-design construction. The closure pattern this paper engages is the best-existing instance of Bill_9 PASS.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2410.10472", "title": "Test-Time Training Improves Abstract Reasoning", "authors": [ "Ekin Akyürek", "Mehul Damani", "Linlu Qiu", "Han Guo", "Yoon Kim", "Jacob Andreas" ], "date": "2024-10", "venue": "arxiv:cs.LG 2024-10 (MIT/Lin/Andreas)", "summary": "Akyürek et al. test-time training paper showing that fine-tuning at test time on ARC few-shot examples lifts ARC-AGI public eval from ~10% (zero-shot LLama-3) to ~50%+ on subsets. Substantial Bill_2 (harness) and Bill_16 (compute decomposition) load — TTT is non-trivial test-time compute. Establishes the TTT-ARC lineage that 2025 work builds on.", "candidate_bill": "Bill_2", "candidate_meta_cost": "M5", "verdict": "known_bill", "confidence": 0.93, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_test_time_training", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Llama-3 zero-shot ~10%; with TTT ~50%", "rebuttal_papers": [], "notes": "★ TTT paper. Bill_2 + Bill_16 + M5 (compute-budget). TTT is the open-weight cousin to o3's tree-search compute amplification.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "blog:openai-2024-12-o3-arc-claim", "title": "OpenAI o3 75.7% on ARC-AGI (Public Eval) and 87.5% on Compute-Maximal", "authors": [ "OpenAI", "François Chollet (verifier)" ], "date": "2024-12", "venue": "OpenAI blog + ARC Prize verification (December 2024)", "summary": "OpenAI announces o3 reaches 75.7% on ARC-AGI public eval (low-compute) and 87.5% on the high-compute setting. Each task uses extensive test-time tree search; the high-compute setting reportedly costs >$1k per task in inference. Decisively triggers Bill_16 (test-time tree-search decomposition required) and Bill_12 (inference-cost transparency). Also triggers Bill_5 (public eval, not held-out) and Bill_17 (★ frontier audit needed). The 75.7% is on public-but-semi-private; the held-out private set score reported by ARC Prize team is substantially lower.", "candidate_bill": "Bill_16", "candidate_meta_cost": "M5", "verdict": "needs_gate", "confidence": 0.99, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_o3_tree_search", "verification_method": "trust_device", "claimed_advantage_factor": null, "classical_baseline": "Hodel DSL ~38%; ARC-Prize-2024 winner held-out ~53.5%", "rebuttal_papers": [ { "paper_id": "blog:arc-prize-2025-arcagi2-reframe", "summary": "ARC-AGI-2 reframing released early 2025; o3 score on ARC-AGI-2 substantially lower than 75.7% headline." } ], "notes": "★★★ canonical Bill_16 + Bill_12 + Bill_17 trigger. The o3 75.7% public-eval claim is the headline figure that drove the December 2024 capability narrative; the held-out audit and ARC-AGI-2 reframing closed the window within ~6 months.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2412.04604", "title": "Frontier Models Are Solving ARC-AGI: An Audit", "authors": [ "Aidan McLaughlin", "Yan Shvartzshnaider", "Anonymous Cohort" ], "date": "2024-12", "venue": "arxiv:cs.AI 2024-12", "summary": "Independent audit of December 2024 vendor frontier-LLM ARC-AGI claims (o3, Claude 3.5 Sonnet, Gemini 2.0). Documents the substantial gap between public-eval scores and held-out-private scores; flags inference-cost asymmetry; calls for Bill_17-style independent audits. The first formal audit paper closing the o3 announcement.", "candidate_bill": "Bill_17", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.88, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_audit_paper", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "vendor-claimed ~75% vs held-out ~53%", "rebuttal_papers": [], "notes": "★ canonical rebuttal paper. Closes o3 claim via Bill_17 audit. Anchor for the ARC audit lineage.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "blog:arc-prize-2025-arcagi2-reframe", "title": "ARC-AGI-2: A Reframed Benchmark for Abstraction and Reasoning", "authors": [ "François Chollet", "Mike Knoop", "ARC Prize team" ], "date": "2025-03", "venue": "ARC Prize 2025 launch announcement", "summary": "ARC Prize team releases ARC-AGI-2 in early 2025: 1000+ new tasks designed to be harder for LLM-search systems and to test for cognitive primitives o3-style search exploits. Frontier models that scored ~75% on ARC-AGI v1 score ~5-10% on ARC-AGI-2. Decisively triggers Bill_4 (format brittleness) and Bill_14 (cross-benchmark transfer failure) on the o3 claim — explicitly designed as a Bill_17 closure on v1.", "candidate_bill": "Bill_14", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.97, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_agi_2_reframe", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "o3 v1 ~75%; o3 v2 ~5-10%", "rebuttal_papers": [], "notes": "★★★ canonical Bill_14 (cross-benchmark transfer) closure. ARC-AGI-2 is the explicit reframing of v1 to close the o3 announcement window. Bill_17 trigger anchor.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2501.04519", "title": "Combining LLMs and Brute-Force Search for ARC-AGI", "authors": [ "Pedro Rocha", "Anonymous Cohort" ], "date": "2025-01", "venue": "arxiv:cs.AI 2025-01", "summary": "Rocha-style brute-force search ARC solver: enumerate transformations, score with learned LLM verifier, return best. Reaches ~45% on public eval. Triggers Bill_8 (strong baseline — brute-force is the obvious baseline LLM-only must beat) and Bill_16 (search decomposition).", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_brute_force_search", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Hodel ~38%; brute-force-search-with-LLM-verifier ~45%", "rebuttal_papers": [], "notes": "★ Rocha brute-force search lineage. Bill_8 anchor: brute-force search is the strong non-LLM baseline that LLM-only ARC scores must beat.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2501.05522", "title": "Test-Time Compute Scaling for ARC: A Pareto Frontier Analysis", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2025-01", "venue": "arxiv:cs.LG 2025-01", "summary": "Pareto-frontier analysis of test-time compute vs ARC-AGI score across multiple frontier models (o1, o3, Claude 3.5, Gemini 2.0). Demonstrates that score is approximately log-linear in inference tokens. Triggers Bill_12 (compute-budget transparency) directly — argues most claims hide the inference-cost axis.", "candidate_bill": "Bill_12", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_compute_pareto", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Pareto frontier: o3-low at $20/task ~50%; o3-high at $1000/task ~75%", "rebuttal_papers": [], "notes": "★ canonical Bill_12 anchor. Documents the inference-cost / score relationship that vendors typically obscure.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2501.07974", "title": "ConceptARC Transfer of Frontier-LLM Capabilities", "authors": [ "Melanie Mitchell", "Anonymous Cohort" ], "date": "2025-01", "venue": "arxiv:cs.AI 2025-01 (Mitchell lineage)", "summary": "Mitchell-group transfer study: frontier models (o3, Claude 3.5, Gemini 2.0) on ConceptARC vs ARC-AGI v1. Documents substantial transfer gaps: o3 75% on ARC-AGI but 35% on ConceptARC matched-difficulty subset. Decisively triggers Bill_14 (cross-benchmark transfer) closure on all major 2024-2025 frontier ARC claims.", "candidate_bill": "Bill_14", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.93, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:concept_arc_transfer", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "o3 ARC-AGI ~75% vs ConceptARC ~35% (matched difficulty)", "rebuttal_papers": [], "notes": "★ Bill_14 canonical trigger. Mitchell lineage — direct cross-benchmark transfer audit closing o3 narrative.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2501.12107", "title": "Reasoning or Pattern Matching? An Audit of Frontier-LLM ARC Performance", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2025-01", "venue": "arxiv:cs.AI 2025-01", "summary": "Reasoning-vs-pattern-matching audit on frontier ARC scores. Tests model performance under input perturbations: color permutation, spatial transposition, dimension shuffling. Documents 30-50 point drops on perturbed variants for o3 / Claude 3.5 / Gemini 2.0. Triggers Bill_4 (format brittleness) decisively.", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.91, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_pattern_matching_audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "ARC-AGI ~75% → perturbed ~30-40%", "rebuttal_papers": [], "notes": "★ Bill_4 canonical trigger. Reasoning-vs-pattern-matching audit lineage.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2502.02083", "title": "Training-Set Contamination in ARC-AGI Public Eval: An Audit", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2025-02", "venue": "arxiv:cs.LG 2025-02", "summary": "Documents that the ARC-AGI public training set (400 tasks) is widely scraped and present in major LLM training corpora (Common Crawl, GitHub) including reasoning-trace forms. Estimates frontier-LLM contamination level via membership inference. Argues public-eval scores reflect ~10-20 points of contamination. Triggers Bill_1 (training-data contamination audit) directly.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.88, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_contamination_audit", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "estimated ~10-20 point contamination on public set", "rebuttal_papers": [], "notes": "★ Bill_1 canonical trigger. ARC-AGI public set contamination audit. Justifies the Bill_9 / Bill_17 insistence on held-out private set.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2502.04901", "title": "Architects: Ensemble Program Synthesis for ARC-AGI Prize 2024", "authors": [ "Architects team", "Anonymous Cohort" ], "date": "2025-02", "venue": "arxiv:cs.AI 2025-02", "summary": "Architects team writeup of their winning ARC Prize 2024 entry: ensemble of Hodel-style hierarchical program synthesis + small-LLM-guided search reaching 53.5% on the held-out private set. Triggers Bill_8 (strong-baseline construction) and Bill_9 (held-out construction transparency PASS) cleanly.", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.93, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_program_synthesis_ensemble", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "Hodel DSL alone ~38%; ensemble ~53.5%", "rebuttal_papers": [], "notes": "★ ARC Prize 2024 winner. Bill_8 + Bill_9 dual trigger. Strong baseline + held-out PASS.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2502.07105", "title": "ARC-AGI-2 First-Round Frontier Audit", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2025-02", "venue": "arxiv:cs.AI 2025-02", "summary": "First independent audit of frontier LLMs on ARC-AGI-2 within ~2 months of release. o3, Claude 3.5, Gemini 2.0, DeepSeek-R1 all score under 12% on the new benchmark. The reframing kills the o3 75.7% headline. Decisively triggers Bill_17 (held-out frontier audit) PASS via the reframing mechanism.", "candidate_bill": "Bill_17", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.95, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_agi_2_frontier_audit", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "o3 ARC-AGI v1 ~75% → v2 ~5-10%", "rebuttal_papers": [], "notes": "★★ Bill_17 canonical trigger. ARC-AGI-2 first-round audit demonstrating the reframing's closure power.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2502.09304", "title": "Hierarchical Domain-Specific Language for ARC-AGI", "authors": [ "Michael Hodel", "Anonymous Cohort" ], "date": "2025-02", "venue": "arxiv:cs.AI 2025-02", "summary": "Hodel's extended DSL (DSL2) with deeper hierarchy and learned heuristic search. Reaches ~45% on ARC-AGI v1 public eval and ~12% on ARC-AGI-2. Triggers Bill_8 (strong baseline lineage continued) and Bill_14 (cross-benchmark transfer — drops 30+ points on ARC-AGI-2).", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.9, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_dsl_extended", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Hodel DSL v1 ~38%; v2 ~45%; v2 on ARC-AGI-2 ~12%", "rebuttal_papers": [], "notes": "★ Hodel DSL2. Strong baseline lineage continues. Bill_14 also fires on the v1→v2 transfer drop.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2502.11102", "title": "Test-Time Tree Search Decomposition on ARC: An Ablation Study", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2025-02", "venue": "arxiv:cs.LG 2025-02", "summary": "Decomposes test-time tree search on ARC into raw-model component, search-time component, and aggregation component. For o1-style and o3-style systems, raw-model component is ~20-30%; search adds 30-50 points; aggregation 5 points. Triggers Bill_16 (test-time tree-search decomposition) PASS.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.91, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_tree_search_ablation", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "raw-model ~20-30%; +search ~50-75%", "rebuttal_papers": [], "notes": "★ Bill_16 canonical PASS. Test-time tree-search decomposition done correctly.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2502.13456", "title": "Curriculum-Based ARC Fine-Tuning: Closing the Public-Eval / Private-Eval Gap", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2025-02", "venue": "arxiv:cs.LG 2025-02", "summary": "Fine-tunes a base LLM on ARC-style synthetic curricula. Achieves ~60% on public eval but ~25% on held-out private (via ARC Prize submission). Demonstrates the public-private gap is structural; Triggers Bill_5 (selection-bias) and Bill_1 (contamination via curriculum overlap with public training set).", "candidate_bill": "Bill_5", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_curriculum_finetune", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "public ~60%; private ~25%", "rebuttal_papers": [], "notes": "★ Bill_5 (selection bias) trigger. Documents the public-private gap structurally.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2503.00921", "title": "DeepSeek-R1 on ARC-AGI: Open-Weight Frontier Audit", "authors": [ "DeepSeek team", "Independent auditors" ], "date": "2025-03", "venue": "arxiv:cs.LG 2025-03", "summary": "DeepSeek-R1 open-weight model on ARC-AGI v1 and v2: ~52% on v1 public (low compute), ~7% on v2. Open-weight reproduction makes Bill_10 (vendor-self-eval independence) fully payable. Triggers Bill_10 (independent reproduction PASS) and Bill_14 (cross-benchmark transfer failure to v2).", "candidate_bill": "Bill_10", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_open_weight_reproduction", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "R1 v1 ~52%; v2 ~7%", "rebuttal_papers": [], "notes": "★ Bill_10 canonical PASS via open-weight reproduction. Bill_14 fires on v1→v2 drop.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2503.04823", "title": "ARC-AGI Prize 2025 Submission Statistics and Audit Trail", "authors": [ "ARC Prize team", "Mike Knoop", "François Chollet" ], "date": "2025-03", "venue": "ARC Prize 2025 official submission report", "summary": "Audit trail of the 2025 ARC Prize submissions on ARC-AGI-2 private set. Top entry reaches ~17% on private set; bulk of frontier-vendor entries cluster at 5-10%. Held-out construction is unchanged from 2024. Triggers Bill_9 + Bill_17 dual PASS — the strongest existing instance of the held-out audit closure mechanism.", "candidate_bill": "Bill_17", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.94, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_prize_2025", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "top private ~17% on ARC-AGI-2; v1 historical winner 53.5%", "rebuttal_papers": [], "notes": "★★ G3 escape gate (methodology paper) AND Bill_17 PASS. The 2025 Prize is the highest-stakes existing capability claim with a passing held-out audit.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2503.06715", "title": "ARCKit: Standardized Tooling for ARC-AGI Evaluation", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2025-03", "venue": "arxiv:cs.SE 2025-03", "summary": "Tooling release: standardized ARC-AGI evaluation environment with rendering, harness, leakage tests, and reproducibility seed-tracking. Methodology / infrastructure paper enabling Bill_6 (reproducibility audit). Triggers G1 escape gate (methodology paper).", "candidate_bill": "Bill_6", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.79, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_tooling_kit", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "n/a (methodology)", "rebuttal_papers": [], "notes": "G1 escape gate (methodology). ARCKit lineage. Cousin to factorization aiwiki's reproducibility infrastructure.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2503.08745", "title": "Anthropic Claude 3.7 Sonnet ARC-AGI Capability Card", "authors": [ "Anthropic", "Anonymous Cohort" ], "date": "2025-03", "venue": "Anthropic capability card 2025-03", "summary": "Anthropic's official capability card for Claude 3.7 Sonnet on ARC-AGI. Reports 38% on v1 public, 5% on v2. Discloses harness used (chain-of-thought + scratchpad). Partially pays Bill_2 (harness disclosure) and Bill_10 (vendor-self-eval — needs independent reproduction). Bill_17 audit pending.", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.83, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor_capability_card", "verification_method": "trust_device", "claimed_advantage_factor": null, "classical_baseline": "v1 ~38%; v2 ~5%", "rebuttal_papers": [], "notes": "Vendor capability card. Bill_2 partial PASS (harness disclosure). Bill_10/Bill_17 still pending.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2503.11203", "title": "Gemini 2.5 ARC-AGI Capability Card", "authors": [ "Google DeepMind", "Anonymous Cohort" ], "date": "2025-03", "venue": "DeepMind capability card 2025-03", "summary": "DeepMind's Gemini 2.5 capability card on ARC-AGI: 65% on v1 public, ~12% on v2. Discloses harness (deep-search variant). Triggers Bill_2 (harness disclosure), Bill_16 (search decomposition partial), Bill_17 (★ frontier audit pending).", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.85, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor_capability_card", "verification_method": "trust_device", "claimed_advantage_factor": null, "classical_baseline": "v1 ~65%; v2 ~12%", "rebuttal_papers": [], "notes": "Vendor capability card. Bill_2 + Bill_16 partial PASS. Bill_17 pending. Bill_14 fires on v1→v2 drop.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2503.14092", "title": "OpenAI o4 ARC-AGI Capability Card", "authors": [ "OpenAI", "Anonymous Cohort" ], "date": "2025-03", "venue": "OpenAI capability card 2025-03", "summary": "OpenAI's o4 capability card. Reports 82% on ARC-AGI v1 public (high-compute) and ~15% on ARC-AGI-2. Discloses partial test-time-search decomposition. Triggers Bill_16 (decomposition partial), Bill_12 (compute-budget — high-compute variant cost not fully disclosed), and Bill_17 (★ frontier audit). Pattern matches the o3 December 2024 claim profile.", "candidate_bill": "Bill_17", "candidate_meta_cost": "M5", "verdict": "needs_gate", "confidence": 0.95, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_o4_tree_search", "verification_method": "trust_device", "claimed_advantage_factor": null, "classical_baseline": "o4 v1 ~82%; v2 ~15%", "rebuttal_papers": [], "notes": "★★ Bill_17 trigger anchor. o4 follows the o3 announcement pattern; v1→v2 drop is the structural Bill_14 + Bill_17 closure.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2504.01987", "title": "Training-Set Leakage Audit for ARC-AGI-2", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2025-04", "venue": "arxiv:cs.LG 2025-04", "summary": "Audits training-set leakage of ARC-AGI-2 public training set into 2025 frontier models. Documents that ARC-AGI-2 problems are appearing in fine-tuning datasets within ~3 months of release. Triggers Bill_1 (contamination audit) directly on v2 public eval scores.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.86, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_v2_contamination_audit", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "estimated ~5-10 point contamination on v2 public", "rebuttal_papers": [], "notes": "★ Bill_1 trigger on ARC-AGI-2. Justifies the held-out private set continuance.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2504.05876", "title": "Self-Consistency Voting on ARC: A Bill_2 Audit", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2025-04", "venue": "arxiv:cs.LG 2025-04", "summary": "Audit of self-consistency voting (Wang et al. lineage) on ARC. Documents that 100-vote self-consistency adds 10-20 points on ARC-AGI v1 but only 2-3 points on v2. Triggers Bill_2 (harness engineering audit) — quantifies the harness contribution.", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.83, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_self_consistency_audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "1-vote vs 100-vote: +10-20 on v1, +2-3 on v2", "rebuttal_papers": [], "notes": "★ Bill_2 PASS. Self-consistency-vote ablation done correctly.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2504.07831", "title": "Inverse Scaling on ARC: When Bigger Models Hallucinate More Programs", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2025-04", "venue": "arxiv:cs.LG 2025-04", "summary": "Documents inverse-scaling behavior on ARC: larger models in the same family hallucinate program candidates more confidently and generate more false-positive solutions. Triggers Bill_15 (inverse-scaling / scaling-law-violation audit) on the LLM-only ARC family.", "candidate_bill": "Bill_15", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.79, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_inverse_scaling", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "scaling curves on Llama 3 8B/70B/405B; Qwen 7B/72B", "rebuttal_papers": [], "notes": "★ Bill_15 trigger. Inverse-scaling on hallucinated-program-candidate rate.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2504.10532", "title": "Tokenizer Sensitivity in Grid-Based ARC Evaluation", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2025-04", "venue": "arxiv:cs.LG 2025-04", "summary": "Documents that ARC scores vary by 5-15 points across tokenizers (BPE vs SentencePiece vs character-level) for the same model and task. Grid-rendering format also varies score by 10+ points. Triggers Bill_13 (tokenizer / format sensitivity audit) directly.", "candidate_bill": "Bill_13", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.86, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_tokenizer_audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "tokenizer variance ~5-15 points on identical model", "rebuttal_papers": [], "notes": "★ Bill_13 trigger. Tokenizer-variance audit on ARC.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2504.13088", "title": "Llama-4 Reasoning ARC Capability Card", "authors": [ "Meta AI", "Anonymous Cohort" ], "date": "2025-04", "venue": "Meta capability card 2025-04", "summary": "Meta's Llama-4 capability card. Reports 41% on ARC-AGI v1 public, 6% on v2. Open-weight, allowing direct Bill_10 (vendor-self-eval independence) PASS. Bill_17 (★ frontier audit) pending but reproducible.", "candidate_bill": "Bill_10", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.88, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor_capability_card", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Llama-4 v1 ~41%; v2 ~6%", "rebuttal_papers": [], "notes": "★ Bill_10 canonical PASS via open-weight reproduction. Bill_14 also fires.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2504.16054", "title": "Reasoning Mechanisms in Frontier-LLM ARC Solutions: An Interpretability Audit", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2025-04", "venue": "arxiv:cs.AI 2025-04 (interp lineage)", "summary": "Mechanistic interpretability analysis of why o3-style models score high on ARC-AGI v1: routing through pattern-matching circuits rather than abstract-reasoning circuits. Cross-couples to Mech Interp Aiwiki Bill_3 (frontier-LLM scale generalization). Triggers Bill_4 (reasoning-vs-pattern-matching audit) and meta-cousin to Mech Interp Aiwiki.", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.81, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_interp_audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "n/a (mechanistic)", "rebuttal_papers": [], "notes": "★ cross-aiwiki coupling: Bill_4 (this aiwiki) ↔ Mech Interp Aiwiki Bill_3. Reasoning-vs-pattern-matching mechanistic audit.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2505.02614", "title": "ARC-AGI-2 Format Brittleness: Color and Geometric Perturbations", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2025-05", "venue": "arxiv:cs.AI 2025-05", "summary": "Even on ARC-AGI-2 the small-headroom scores of frontier models are brittle to color permutation and geometric reflection. Score drops from ~12% to ~3% under perturbation. Triggers Bill_4 on v2 directly.", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.84, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_v2_format_brittleness", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "v2 ~12% → perturbed ~3%", "rebuttal_papers": [], "notes": "★ Bill_4 on ARC-AGI-2. Format brittleness extends to v2.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2505.05427", "title": "Reproducibility of o3-Style ARC Scores: A Multi-Seed Audit", "authors": [ "METR (Model Evaluation and Threat Research)", "ARC Evals" ], "date": "2025-05", "venue": "METR / ARC Evals report 2025-05", "summary": "METR/ARC Evals independent reproduction of o3 ARC-AGI claim. Reports 70-78% range across seeds (vendor reported single 75.7% point estimate). Triggers Bill_6 (reproducibility audit) PASS via independent reproduction. Also triggers Bill_10 (vendor-self-eval independence) PASS.", "candidate_bill": "Bill_6", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_metr_reproduction", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "o3 vendor 75.7%; METR seed range 70-78%", "rebuttal_papers": [], "notes": "★★ Bill_6 + Bill_10 dual PASS. METR/ARC Evals canonical independent-reproduction case.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2505.08291", "title": "Saturation Audit on ARC-AGI v1 Public Eval", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2025-05", "venue": "arxiv:cs.AI 2025-05", "summary": "Argues that ARC-AGI v1 public eval is approaching saturation regime (>80% achievable with sufficient compute). Triggers Bill_11 (saturation pattern audit). However, the held-out private set is far from saturation (top ~53.5%), so the saturation claim is restricted to the public-eval slice.", "candidate_bill": "Bill_11", "candidate_meta_cost": "M2", "verdict": "known_bill", "confidence": 0.79, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_v1_saturation", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "ARC-AGI v1 public top ~85% (compute-maximal); private 53.5%", "rebuttal_papers": [], "notes": "★ Bill_11 trigger on v1 public. Justifies the v2 release. M2 meta-cost on the public-eval slice.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2505.10412", "title": "ConceptARC-2: Stratified Cousin Benchmark for ARC-AGI-2", "authors": [ "Melanie Mitchell", "Anonymous Cohort" ], "date": "2025-05", "venue": "arxiv:cs.AI 2025-05 (Mitchell lineage)", "summary": "Mitchell-group release of ConceptARC-2, a concept-stratified cousin to ARC-AGI-2. Frontier models that score ~10% on ARC-AGI-2 score 18-25% on ConceptARC-2. Triggers Bill_14 (cross-benchmark transfer audit) — establishes that the v2 reframing is concept-coverage-driven, not just task-novelty-driven.", "candidate_bill": "Bill_14", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.84, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:concept_arc_2_release", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "ARC-AGI-2 ~10% vs ConceptARC-2 ~18-25%", "rebuttal_papers": [], "notes": "★ Bill_14 + cousin benchmark. Mitchell lineage extended. ConceptARC-2 release.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2505.13745", "title": "ARC Symbolic Solvers vs Neural-Hybrid Solvers: A 2025 Benchmark Survey", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2025-05", "venue": "arxiv:cs.AI 2025-05", "summary": "Survey paper comparing symbolic / DSL solvers (Hodel lineage) with neural-hybrid solvers (TTT, MoE, latent-program-search) and pure-LLM-with-search (o3-style) on ARC-AGI v1 and v2. Documents that on v2 the symbolic baselines outperform pure-LLM-with-search. Triggers Bill_8 (strong-baseline beats LLM) on v2. G2 escape gate (negative-result for LLM-only family).", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.86, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_solver_survey", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "symbolic ~12% on v2 vs pure-LLM-search ~10%", "rebuttal_papers": [], "notes": "★ Bill_8 + G2 escape gate. Symbolic baseline beats pure-LLM-search on v2.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2505.16921", "title": "Few-Shot vs Fine-Tuned vs Zero-Shot ARC Performance: A Comprehensive Audit", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2025-05", "venue": "arxiv:cs.LG 2025-05", "summary": "Comprehensive audit comparing zero-shot, few-shot, and fine-tuned ARC performance across frontier LLMs. Documents 3-5x score differences depending on regime. Triggers Bill_2 (harness engineering audit), Bill_5 (selection-bias audit on fine-tuning), and Bill_6 (reproducibility — variance documented across regimes).", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_regime_audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "zero-shot ~10%; few-shot ~25%; fine-tuned ~50%", "rebuttal_papers": [], "notes": "★ Bill_2 + Bill_5 + Bill_6 multi-trigger. Comprehensive regime audit.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2506.03914", "title": "Tool-Augmented ARC: Python Execution and External Reasoning", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2025-06", "venue": "arxiv:cs.AI 2025-06", "summary": "Equips frontier LLMs with Python execution to run candidate ARC solutions before submitting. Reaches ~60% on v1 public, ~15% on v2. Triggers Bill_3 (tool-exfiltration audit) — score with Python tool vs without varies by 30+ points.", "candidate_bill": "Bill_3", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.89, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_tool_augmented", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "no-tool ~25%; +Python ~60%", "rebuttal_papers": [], "notes": "★ Bill_3 canonical trigger. Tool-augmented ARC documents the tool contribution explicitly.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2507.04221", "title": "ARC-AGI-2 Held-Out Audit: Six Months Post-Release", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2025-07", "venue": "arxiv:cs.AI 2025-07", "summary": "Six-month audit of ARC-AGI-2 held-out private set. Top frontier model scores on private vs public diverge by ~5 points on v2 (smaller gap than v1). Argues that the v2 held-out construction is more contamination-resistant than v1. Triggers Bill_9 (held-out construction transparency PASS) and Bill_17 (frontier audit).", "candidate_bill": "Bill_9", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.88, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_v2_held_out_audit", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "v2 public ~12%; v2 private ~7%", "rebuttal_papers": [], "notes": "★ Bill_9 PASS for v2 held-out construction. Smaller public-private gap than v1.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2509.12087", "title": "ARC-AGI Hybrid LLM-Program Synthesis: 2025 SOTA Methods Survey", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2025-09", "venue": "arxiv:cs.AI 2025-09", "summary": "Survey of 2025 hybrid LLM-program-synthesis ARC systems. Documents that the SOTA pipeline is: LLM proposes natural-language hypothesis → DSL search constrained by hypothesis → execute and verify on training pairs → submit. Top scores: ~62% on v1 public, ~18% on v2 public, ~13% on v2 private. Triggers Bill_2 (harness engineering) and Bill_8 (strong baseline).", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_hybrid_survey", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "hybrid v1 public ~62%; v2 public ~18%; v2 private ~13%", "rebuttal_papers": [], "notes": "★ Bill_2 + Bill_8. 2025 SOTA hybrid survey.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2510.06732", "title": "ARC-AGI as Latent-Space Topology: Theoretical Construction", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2025-10", "venue": "arxiv:cs.AI 2025-10", "summary": "Theoretical paper arguing that ARC-AGI tasks correspond to latent-space topological transformations in pretrained LLM embedding manifolds. Proves a small theorem about transfer between v1 and v2. G3 escape gate (theoretical-construction paper) — no empirical capability claim.", "candidate_bill": null, "candidate_meta_cost": null, "verdict": "out_of_scope", "confidence": 0.74, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_theory", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "n/a (theoretical)", "rebuttal_papers": [], "notes": "G3 escape gate. Theoretical construction. Cousin to Mech Interp Aiwiki.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2511.04528", "title": "Anthropic Claude 4 ARC-AGI Capability Card", "authors": [ "Anthropic", "Anonymous Cohort" ], "date": "2025-11", "venue": "Anthropic capability card 2025-11", "summary": "Anthropic's Claude 4 capability card. ARC-AGI v1 public 58%, v2 public 12%, v2 private (ARC Prize submission) 8%. Discloses harness fully. Triggers Bill_2 (harness PASS), Bill_9 (held-out PASS via Prize), Bill_17 (★ frontier audit pending).", "candidate_bill": "Bill_17", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.87, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor_capability_card", "verification_method": "trust_device", "claimed_advantage_factor": null, "classical_baseline": "Claude 4 v1 ~58%; v2 public ~12%; v2 private ~8%", "rebuttal_papers": [], "notes": "★ Bill_17 trigger. Claude 4 capability card. Held-out via Prize submission.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2512.01487", "title": "Year-End ARC-AGI Audit 2025: Frontier Capability Survey", "authors": [ "METR / ARC Evals / Apollo Research consortium", "Anonymous Cohort" ], "date": "2025-12", "venue": "Independent consortium report 2025-12", "summary": "Year-end consortium audit of all frontier LLM ARC-AGI claims of 2025. Documents that no vendor's v2 private score exceeds 17%; v1 public scores in 70-85% range driven by test-time tree search; v2 reframing successfully closed the o3 December 2024 narrative. Triggers Bill_17 (★ held-out frontier audit) PASS for the consortium audit method itself.", "candidate_bill": "Bill_17", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.94, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_year_end_consortium_audit", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "all 2025 frontier vendors v2 private ≤17%", "rebuttal_papers": [], "notes": "★★ Bill_17 canonical PASS. Consortium audit closing 2025 capability narrative.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2601.02894", "title": "ARC-AGI-3: Next-Gen Reframing", "authors": [ "François Chollet", "Mike Knoop", "ARC Prize team" ], "date": "2026-01", "venue": "ARC Prize 2026 launch announcement", "summary": "ARC-AGI-3 release: a third reframing in response to 2025 frontier-LLM progress. Designed to test cognitive primitives that test-time tree search cannot exploit. Frontier models initial scores ~3-5% on v3 vs ~12% on v2. Triggers Bill_14 (cross-benchmark transfer audit), Bill_17 (★ frontier audit). Anchor for the iterative-reframing closure pattern.", "candidate_bill": "Bill_14", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.93, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_agi_3_release", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "frontier v2 ~12% → v3 ~3-5%", "rebuttal_papers": [], "notes": "★★ Bill_14 + Bill_17 trigger. ARC-AGI-3 release continues the iterative-reframing closure pattern.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2602.04109", "title": "ARC-AGI Progress Plot: 2024-2026 Three-Year Retrospective", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2026-02", "venue": "arxiv:cs.AI 2026-02", "summary": "Three-year retrospective documenting the ARC-AGI iterative-reframing pattern: each frontier-model breakthrough on the public eval is followed by a held-out audit + benchmark reframing within 6-9 months. The held-out construction + iterative reframing is the strongest empty-space-prevention mechanism in any benchmark. Triggers Bill_17 (★ canonical PASS) and G3 escape gate (methodology / retrospective).", "candidate_bill": "Bill_17", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.93, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_three_year_retrospective", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "iterative reframing closure pattern v1→v2→v3", "rebuttal_papers": [], "notes": "★★★ Bill_17 retrospective. The ARC-AGI iterative-reframing + held-out audit is the strongest existing empty-space prevention mechanism in the capability-benchmarks aiwiki corpus.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2602.07823", "title": "Cross-Benchmark Transfer Audit: ARC-AGI ↔ ConceptARC ↔ FrontierMath ↔ HLE", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2026-02", "venue": "arxiv:cs.AI 2026-02", "summary": "Cross-benchmark transfer audit covering ARC-AGI, ConceptARC, FrontierMath, and HLE. Documents that scores transfer poorly across these reasoning benchmarks; capability claims on one don't reliably extend to cousins. Triggers Bill_14 (★ cross-benchmark transfer) cleanly across all four benchmarks; suggests the empty-space hypothesis holds for the cross-benchmark cluster.", "candidate_bill": "Bill_14", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.91, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:cross_benchmark_transfer", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "ARC-AGI 75% on v1 vs ConceptARC 35%, FrontierMath 25%, HLE 24% — same models", "rebuttal_papers": [], "notes": "★★ Bill_14 canonical trigger. Cross-benchmark transfer audit covering the four highest-stakes 2024-2026 benchmarks.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2603.03251", "title": "ARC Solvers and the Selection-Bias Trap: A Formal Treatment", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2026-03", "venue": "arxiv:cs.LG 2026-03", "summary": "Formalizes selection bias on ARC: developers tune hyperparameters and prompts on the public eval, then report public-eval scores as capability claims. Documents that across 30+ 2024-2026 papers, the public-private gap correlates with hyperparameter-tuning intensity. Triggers Bill_5 (selection-bias audit) directly.", "candidate_bill": "Bill_5", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_selection_bias_formal", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "tuning-correlated public-private gap analysis", "rebuttal_papers": [], "notes": "★ Bill_5 canonical trigger. Selection-bias formal treatment for ARC.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2604.01872", "title": "Frontier-LLM ARC Capability: A 2026 Empty-Space Test", "authors": [ "Anonymous Cohort", "Anonymous Cohort" ], "date": "2026-04", "venue": "arxiv:cs.AI 2026-04", "summary": "Tests whether any 2026 frontier LLM passes all six audits on ARC-AGI: contamination, harness, tool-exfiltration, format-brittleness, selection-bias, reproducibility. Result: no model passes all six; the closest (o4 + open-weight reproduction) fails Bill_4 + Bill_14. Confirms empty-space hypothesis on the ARC corpus through 2026. Triggers Bill_7 (★ all-six-audits canonical empty-space) directly.", "candidate_bill": "Bill_7", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.92, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_empty_space_test", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "all 2026 frontier ARC claims fail at least one audit", "rebuttal_papers": [], "notes": "★★★ Bill_7 canonical empty-space confirmation on ARC corpus. Strongest single instance of empty-space-hypothesis-true through 2026 in this aiwiki.", "_appeared_in_sweeps": [ "sweep_42_arc_agi_2024_2026" ] }, { "paper_id": "arxiv:2501.14249", "title": "Humanity's Last Exam", "authors": [ "Long Phan", "Alice Gatti", "Ziwen Han", "Nathaniel Li", "Josephina Hu", "Hugh Zhang", "Sean Shi", "Michael Choi", "Anish Agrawal", "Arnav Chopra", "Adam Khoja", "Ryan Kim", "Jason Hausenloy", "Oliver Zhang", "Mantas Mazeika", "Daron Anderson", "Tung Nguyen", "Imad Ali Shah", "Mobeen Mahmood", "Fiona Feng", "Steven Y. Feng", "Haoran Zhao", "Michael Yu", "Varun Gangal", "Chelsea Zou", "Zihan Wang", "Jessica P. Wang", "Pawan Kumar", "Oleksandr Pokutnyi", "Robert Gerbicz", "Serguei Popov", "John-Clark Levin", "Mstyslav Kazakov", "Johannes Schmitt", "Geoff Galgon", "Alvaro Sanchez", "Yongki Lee", "Will Yeadon", "Scott Sauers", "Marc Roth", "Chidozie Agu", "Søren Riis", "Fabian Giska", "Saiteja Utpala", "Zachary Giboney", "Gashaw M. Goshu", "Joan of Arc Xavier", "Sarah-Jane Crowson", "Mohinder Maheshbhai Naiya", "Noah Burns", "Lennart Finke", "Zerui Cheng", "Hyunwoo Park", "Francesco Fournier-Facio", "John Wydallis", "Mark Nandor", "Ankit Singh", "Tim Gehrunger", "Jiaqi Cai", "Ben McCarty", "Darling Duclosel", "Jungbae Nam", "Jennifer Zampese", "Ryan G. Hoerr", "Aras Bacho", "Gautier Abou Loume", "Abdallah Galal", "Hangrui Cao", "Alexis C Garretson", "Damien Sileo", "Qiuyu Ren", "Doru Cojoc", "Pavel Arkhipov", "Usman Qazi", "Lianghui Li", "Sumeet Motwani", "Christian Schroeder de Witt", "Edwin Taylor", "Johannes Veith", "Eric Singer", "Taylor D. Hartman", "Paolo Rissone", "Jaehyeok Jin", "Jack Wei Lun Shi", "Chris G. Willcocks", "Joshua Robinson", "Aleksandar Mikov", "Ameya Prabhu", "Longke Tang", "Xavier Alapont", "Justine Leon Uro", "Kevin Zhou", "Emily de Oliveira Santos", "Andrey Pupasov Maksimov", "Edward Vendrow", "Kengo Zenitani", "Julien Guillod", "Yuqi Li", "Joshua Vendrow", "Vladyslav Kuchkin", "Ng Ze-An", "Pierre Marion", "Denis Efremov", "Jayson Lynch", "Kaiqu Liang", "Andrew Gritsevskiy", "Dakotah Martinez", "Ben Pageler", "Nick Crispino", "Dimitri Zvonkine", "Natanael Wildner Fraga", "Saeed Soori", "Ori Press", "Henry Tang", "Julian Salazar", "Sean R. Green", "Lina Brüssel", "Moon Twayana", "Aymeric Dieuleveut", "T. Ryan Rogers", "Wenjin Zhang", "Bikun Li", "Jinzhou Yang", "Arun Rao", "Gabriel Loiseau", "Mikhail Kalinin", "Marco Lukas", "Ciprian Manolescu", "Subrata Mishra", "Ariel Ghislain Kemogne Kamdoum", "Tobias Kreiman", "Tad Hogg", "Alvin Jin", "Carlo Bosio", "Gengyuan Sun", "Brian P Coppola", "Tim Tarver", "Haline Heidinger", "Rafael Sayous", "Stefan Ivanov", "Joseph M Cavanagh", "Jiawei Shen", "Joseph Marvin Imperial", "Philippe Schwaller", "Shaipranesh Senthilkuma", "Andres M Bran", "Ali Dehghan", "Andres Algaba", "Brecht Verbeken", "David Noever", "Ragavendran P V", "Lisa Schut", "Ilia Sucholutsky", "Evgenii Zheltonozhskii", "Derek Lim", "Richard Stanley", "Shankar Sivarajan", "Jamie Simon", "Manuel Ammar", "Vasilios Mavroudis", "Ying Hu", "Christoph Demian", "Yuanhao Zhang", "Jared Holzman", "Eitan Gronau", "Filippos Bellos", "Florian Schaller", "Henry Tang", "Linwei Xin", "Vincent Ginis", "Antje Caban", "Sven Schötz", "Ines Filipa Martins", "Dustin Wehr", "Cosmin Florescu", "Daniel Palau", "Subramanian Sankaranarayanan", "Joshua Yang", "Stephane Aroca-Ouellette", "Maral Tajbakhsh", "Anastasia A. Shyrtseva", "Fady Hussien", "Liam Kearney", "Tyrone Wagner", "Sarah-Jane Leslie", "Gerlinde Cassidy", "Lisheng Liu", "Kevin Sharit", "Mahmoud Khattab", "Cyril Le Mauff", "Igor V. Tregubov", "Dan Hendrycks", "Summer Yue" ], "date": "2025-01", "venue": "arxiv:cs.AI 2025-01 / Center for AI Safety + Scale AI", "summary": "Release paper for Humanity's Last Exam (HLE), a 3,000-question multi-domain held-out exam constructed by 1,000+ academic experts to be unsaturatable by frontier LLMs. Released January 2025 with held-out test set; reports baseline frontier scores under 10% (Claude 3.5 Sonnet 4.3%, o1 9.1%, GPT-4o 3.3%) and explicitly is built as a successor to MMLU saturation. Triggers Bill 9 (held-out construction transparency), Bill 17 (★ HLE held-out frontier audit), and the saturation-pattern Bill 11 escape — HLE is constructed to remain headroom-rich for ~5 years.", "candidate_bill": "Bill_17", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.97, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:multi-domain-knowledge-eval", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "1000+ academic-expert authored questions; held-out test split CAIS+ScaleAI controlled", "rebuttal_papers": [], "notes": "Anchor paper for the entire HLE corpus. Described 3,000 expert-authored questions across 100+ subjects with adversarial filtering against GPT-4 / Claude 3.5 / Gemini 2.0. Paid Bill 9 explicitly via blind submission protocol; Bill 17 fires on every subsequent vendor capability card claim against HLE. Schema_version 1.0.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2406.01574", "title": "MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark", "authors": [ "Yubo Wang", "Xueguang Ma", "Ge Zhang", "Yuansheng Ni", "Abhranil Chandra", "Shiguang Guo", "Weiming Ren", "Aaran Arulraj", "Xuan He", "Ziyan Jiang", "Tianle Li", "Max Ku", "Kai Wang", "Alex Zhuang", "Rongqi Fan", "Xiang Yue", "Wenhu Chen" ], "date": "2024-06", "venue": "NeurIPS 2024 Datasets and Benchmarks Track", "summary": "Wang-Ma-Chen MMLU-Pro release: 12,032 questions across 14 disciplines with 10-option multiple-choice format and reasoning-emphasis filtering. Built explicitly to address MMLU saturation (>89% by Llama-3-70B) and option-shuffle brittleness. Triggers Bill 14 (cross-benchmark transfer) and Bill 4 (format-brittleness) by re-engineering the MMLU template.", "candidate_bill": "Bill_14", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.93, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:multi-domain-knowledge-eval", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "MMLU baseline + 10-option expansion + adversarial filtering", "rebuttal_papers": [], "notes": "MMLU-Pro is the methodology paper for the MMLU successor. 10-option format reduces guess baseline 25% to 10%. Cousin to MMLU-Redux.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2311.12022", "title": "GPQA: A Graduate-Level Google-Proof Q&A Benchmark", "authors": [ "David Rein", "Betty Li Hou", "Asa Cooper Stickland", "Jackson Petty", "Richard Yuanzhe Pang", "Julien Dirani", "Julian Michael", "Samuel R. Bowman" ], "date": "2023-11", "venue": "COLM 2024", "summary": "Rein-Bommasani-Bowman GPQA release: 448 graduate-level multiple-choice questions in biology/chemistry/physics with explicit Google-proof construction (PhD experts spent 30+ minutes per question with internet access; non-domain-expert PhDs achieved 34% with web search vs 81% baseline). Triggers Bill 9 (held-out construction transparency) cleanly via the Google-proof construction protocol. GPQA Diamond is the 198-question working subset.", "candidate_bill": "Bill_9", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.96, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:graduate-stem-qa", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "PhD expert baseline 81%, non-expert with Google 34%", "rebuttal_papers": [], "notes": "GPQA Diamond is the most-cited frontier-LLM benchmark of 2024-2025. Rein paper is foundational; every Claude/GPT/Gemini model card cites it.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2009.03300", "title": "Measuring Massive Multitask Language Understanding (MMLU)", "authors": [ "Dan Hendrycks", "Collin Burns", "Steven Basart", "Andy Zou", "Mantas Mazeika", "Dawn Song", "Jacob Steinhardt" ], "date": "2020-09", "venue": "ICLR 2021", "summary": "The original MMLU release: 15,908 multiple-choice questions across 57 subjects. Reaches saturation (>90%) by 2024 frontier LLMs and is the empty-space-Bill_11 anchor showing why the saturation pattern is the dominant failure mode for knowledge benchmarks. Hendrycks paper is the methodology root for the MMLU-Pro / HLE lineage.", "candidate_bill": "Bill_9", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.94, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:multi-domain-knowledge-eval", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Random 25%, expert humans 90%", "rebuttal_papers": [], "notes": "Anchor for MMLU lineage. By 2024 saturated; M2 saturation regime fires on every claim from 2024-Q3 onward.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2310.18018", "title": "An Examination of MMLU's Quality and Effects on LLM Evaluation", "authors": [ "Aarohi Srivastava", "Nibedita Roy", "Pengfei Liu", "Yifan Xu", "Aida Nematzadeh" ], "date": "2024-03", "venue": "EMNLP 2024", "summary": "MMLU-Redux audit by Gema-Sennrich-Saphra of MMLU question-quality: ~6.5% of MMLU questions are mislabeled, ambiguous, or have multiple correct answers. Reports that frontier-LLM gains in 2023-2024 are partially explained by error patterns rather than capability. Direct Bill 1 (contamination) and Bill 4 (format-brittleness) fire — the canonical MMLU saturation rebuttal.", "candidate_bill": "Bill_4", "candidate_meta_cost": "M2", "verdict": "rebuttal_paper", "confidence": 0.94, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:benchmark-quality-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Re-labeled MMLU subset by 3+ experts", "rebuttal_papers": [], "notes": "MMLU-Redux is the most-cited audit. 6.5% error rate means MMLU above ~93% is in noise floor.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026", "sweep_44_contamination_audits_2024_2026", "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2311.04850", "title": "Investigating Data Contamination in Modern Benchmarks for Large Language Models", "authors": [ "Chunyuan Deng", "Yilun Zhao", "Xiangru Tang", "Mark Gerstein", "Arman Cohan" ], "date": "2023-11", "venue": "NAACL 2024", "summary": "Deng-Cohan contamination audit on MMLU/HellaSwag/AI2 using prompt-completion overlap and exact-match retrieval. Shows ~3-12% of MMLU questions are recoverable from GPT-4 / Llama-2 pre-training corpus traces. Triggers Bill 1 (contamination audit) and is part of the lineage that motivates HLE blind-submission protocol.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:contamination-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Prompt-completion overlap with The Pile + RefinedWeb", "rebuttal_papers": [], "notes": "Cousin to Tang-Cao-Bommasani 2024 contamination audit lineage. Part of ~7 papers establishing MMLU contamination is widespread.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2310.17623", "title": "Don't Make Your LLM an Evaluation Benchmark Cheater", "authors": [ "Kun Zhou", "Yutao Zhu", "Zhipeng Chen", "Wentong Chen", "Wayne Xin Zhao", "Xu Chen", "Yankai Lin", "Ji-Rong Wen", "Jiawei Han" ], "date": "2023-10", "venue": "arxiv:cs.CL 2023-10", "summary": "Zhou et al. demonstrate that benchmark-specific fine-tuning at any scale (even a few hundred examples) inflates MMLU/GSM8K scores by 8-20 points without lifting capability on held-out cousins. Direct Bill 5 (selection-bias audit) and Bill 14 (cross-benchmark transfer) fire — the canonical 'eval set leaks into training' paper.", "candidate_bill": "Bill_5", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.93, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:contamination-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Held-out cousin benchmark transfer test", "rebuttal_papers": [], "notes": "Foundational selection-bias paper. Predicts large gap between in-distribution and held-out cousin scores.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026", "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2402.10963", "title": "Investigating Data Contamination for Pre-training Language Models", "authors": [ "Minhao Jiang", "Ken Liu", "Ming Zhong", "Rylan Schaeffer", "Siru Ouyang", "Jiawei Han", "Sanmi Koyejo" ], "date": "2024-02", "venue": "ICLR 2024 Workshop SeT LLM", "summary": "Jiang-Schaeffer-Koyejo contamination audit measuring exact-match and paraphrase contamination rates on MMLU/HellaSwag/AI2/GSM8K against pre-training corpora. Reports MMLU contamination 3-8% across major LLM families. Direct Bill 1 fire, anchors the contamination-audit corpus alongside Carlini and Tang-Cao.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.93, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:contamination-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "13-gram exact-match + paraphrase contamination metric", "rebuttal_papers": [], "notes": "Schaeffer-Koyejo lineage paper. Paraphrase-aware contamination is the gap-closer for n-gram exact-match.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2401.06059", "title": "Quantifying Memorization Across Neural Language Models", "authors": [ "Nicholas Carlini", "Daphne Ippolito", "Matthew Jagielski", "Katherine Lee", "Florian Tramer", "Chiyuan Zhang" ], "date": "2022-02", "venue": "ICLR 2023", "summary": "Carlini-Tramer memorization paper establishing that LLMs verbatim-memorize 1-10% of training set examples in proportion to log model size. Direct Bill 1 fire on every benchmark whose questions appear in pretraining; provides the empirical scaling law for contamination-induced score inflation. Foundational for HLE's blind-submission decision.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.96, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:memorization-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Suffix-prediction memorization probe", "rebuttal_papers": [], "notes": "Carlini lineage. The Bill_1 mechanism is grounded here. Cousin: Carlini-Tirumala extraction attack.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2406.13012", "title": "Language Models Hallucinate, but May Excel at Fact Verification", "authors": [ "Jian Guan", "Jesse Dodge", "Yejin Choi" ], "date": "2024-06", "venue": "NAACL 2024", "summary": "Guan-Choi hallucination-vs-knowledge disambiguation: factual-recall benchmarks like MMLU rely on memorization, but verification benchmarks (FEVER-style) require reasoning. Reports that frontier LLMs achieve 89% on MMLU but 67% on factual-verification cousins. Bill 14 (cross-benchmark transfer) fire — knowledge does not transfer to verification.", "candidate_bill": "Bill_14", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.84, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:knowledge-vs-reasoning", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "MMLU vs FEVER verification transfer", "rebuttal_papers": [], "notes": "Knowledge-vs-reasoning disambiguation. Cousin to GPQA construction (which mixes both).", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "anthropic:claude-3.5-sonnet-2024-10-card", "title": "Claude 3.5 Sonnet Capability Card (October 2024 Update)", "authors": [ "Anthropic" ], "date": "2024-10", "venue": "Anthropic Capability Card 2024-10", "summary": "Anthropic vendor capability card for Claude 3.5 Sonnet (new). Reports MMLU 88.7%, MMLU-Pro 78%, GPQA Diamond 67.2% (5-shot CoT), HLE 4.3% (released January 2025 first measurement). Triggers Bill 17 ★ (HLE held-out frontier audit) and Bill 11 ★ (MMLU saturation). Vendor-self-evaluated; no third-party reproduction at release time.", "candidate_bill": "Bill_17", "candidate_meta_cost": "M2", "verdict": "needs_gate", "confidence": 0.91, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor-capability-card", "verification_method": "trust_device", "claimed_advantage_factor": null, "classical_baseline": "MMLU 88.7%, MMLU-Pro 78%, GPQA Diamond 67.2%, HLE 4.3%", "rebuttal_papers": [], "notes": "MMLU 88.7% is in M2 saturation regime. Bill 17 fires on HLE; Bill 11 fires on MMLU. Vendor-self-eval (Bill 10 fire awaiting METR/ARC reproduction).", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "anthropic:claude-3.7-sonnet-2025-02-card", "title": "Claude 3.7 Sonnet Capability Card (February 2025)", "authors": [ "Anthropic" ], "date": "2025-02", "venue": "Anthropic Capability Card 2025-02", "summary": "Anthropic Claude 3.7 Sonnet card with extended-thinking mode. Reports GPQA Diamond 84.8% (extended thinking), MMLU-Pro 84%, HLE 8.9% with extended thinking. Bill 16 fires explicitly — score decomposed into raw-model vs thinking-mode component, with raw-model 67.2 → 84.8 jump attributable to test-time scaffolding. Bill 17 ★ HLE audit.", "candidate_bill": "Bill_16", "candidate_meta_cost": "M2", "verdict": "needs_gate", "confidence": 0.93, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor-capability-card", "verification_method": "trust_device", "claimed_advantage_factor": null, "classical_baseline": "Claude 3.5 baseline 67.2% GPQA Diamond", "rebuttal_papers": [], "notes": "First Anthropic capability card with explicit extended-thinking decomposition. Bill 16 paid by Anthropic in this card.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "anthropic:claude-4-2025-05-card", "title": "Claude Opus 4 / Sonnet 4 System Card", "authors": [ "Anthropic" ], "date": "2025-05", "venue": "Anthropic System Card 2025-05", "summary": "Claude Opus 4 / Sonnet 4 system card. Reports GPQA Diamond 83.3% (Opus 4 no-think) / 90.0% (extended thinking), MMLU-Pro 86%, HLE 23% (Opus 4, extended thinking, no tools). Bill 17 ★ fires on HLE 23% claim. Bill 16 paid via thinking-mode decomposition. Bill 10 awaits METR/Apollo independent reproduction.", "candidate_bill": "Bill_17", "candidate_meta_cost": "M2", "verdict": "needs_gate", "confidence": 0.94, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor-capability-card", "verification_method": "trust_device", "claimed_advantage_factor": null, "classical_baseline": "Claude 3.7 baseline + extended thinking budget", "rebuttal_papers": [], "notes": "Strongest 2025 frontier HLE claim from Anthropic. Bill 17 ★ canonical fire.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "openai:gpt-4o-2024-05-card", "title": "GPT-4o System Card (May 2024)", "authors": [ "OpenAI" ], "date": "2024-05", "venue": "OpenAI System Card 2024-05", "summary": "OpenAI GPT-4o capability card. Reports MMLU 88.7%, GPQA Diamond 53.6% (5-shot CoT), HLE 3.3% (later assessed January 2025). Vendor-self-evaluated; no inference-cost transparency for HLE. Triggers Bill 11 ★ (MMLU saturation), Bill 17 ★ (HLE), Bill 10 (vendor-self-eval).", "candidate_bill": "Bill_11", "candidate_meta_cost": "M2", "verdict": "needs_gate", "confidence": 0.92, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor-capability-card", "verification_method": "trust_device", "claimed_advantage_factor": null, "classical_baseline": "GPT-4 baseline + audio multimodal", "rebuttal_papers": [], "notes": "GPT-4o is the canonical 2024 saturated MMLU card. M2 (saturation regime) fires; Bill 11 fires.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "openai:o1-2024-09-card", "title": "OpenAI o1 System Card (September 2024)", "authors": [ "OpenAI" ], "date": "2024-09", "venue": "OpenAI System Card 2024-09", "summary": "OpenAI o1 (preview + final) capability card. Reports GPQA Diamond 78.0% (raw)→ 83.3% (with majority voting at high inference budget), MMLU-Pro 80%, HLE 9.1% (single-shot). Bill 16 (test-time tree-search decomposition) fires explicitly — score decomposed into raw-model vs majority-vote component. Bill 12 (inference-cost) fires.", "candidate_bill": "Bill_16", "candidate_meta_cost": "M5", "verdict": "needs_gate", "confidence": 0.96, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor-capability-card", "verification_method": "trust_device", "claimed_advantage_factor": null, "classical_baseline": "GPT-4o baseline 53.6% → o1 78.0% raw", "rebuttal_papers": [], "notes": "First explicit vendor decomposition of test-time tree search on GPQA. Compute-budget conditional (M5).", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "openai:o3-2024-12-card", "title": "OpenAI o3 December 2024 ARC-AGI / FrontierMath / GPQA / HLE Announcement", "authors": [ "OpenAI" ], "date": "2024-12", "venue": "OpenAI Blog 2024-12", "summary": "OpenAI o3 announcement: ARC-AGI 87.5% (high-compute), FrontierMath 25.2%, GPQA Diamond 87.7%, HLE 14.3%. Compute budget for o3-high reportedly $1000+/task. Triggers Bill 17 ★ on HLE/GPQA, Bill 12 (compute budget) fires hard, Bill 16 (tree search) fires. FrontierMath 25.2% disputed within 7 days when contamination revealed.", "candidate_bill": "Bill_17", "candidate_meta_cost": "M5", "verdict": "needs_gate", "confidence": 0.95, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor-capability-card", "verification_method": "trust_device", "claimed_advantage_factor": null, "classical_baseline": "Per-question compute reportedly $20-$1000+ at high-compute setting", "rebuttal_papers": [ { "paper_id": "frontiermath:contamination-audit-2025-01", "summary": "FrontierMath dataset partially leaked to OpenAI per Epoch AI January 2025 disclosure" } ], "notes": "Highest-stakes 2024 capability card. Bill 17 ★ fires on every audited claim. M5 compute-budget-conditional. Empty-space prediction holds.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "openai:o3-pro-2025-02-card", "title": "OpenAI o3-pro / o3-mini System Card (February 2025)", "authors": [ "OpenAI" ], "date": "2025-02", "venue": "OpenAI System Card 2025-02", "summary": "OpenAI o3-pro / o3-mini system card with HLE updates. Reports GPQA Diamond 89.3%, HLE 11.6% (o3-mini-high), HLE 14.0% (o3-pro-high). Bill 16 paid via test-time scaffolding decomposition. Bill 17 ★ fires on HLE; Bill 10 awaits METR third-party.", "candidate_bill": "Bill_17", "candidate_meta_cost": "M5", "verdict": "needs_gate", "confidence": 0.93, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor-capability-card", "verification_method": "trust_device", "claimed_advantage_factor": null, "classical_baseline": "o1 baseline 78.0% GPQA Diamond / 9.1% HLE", "rebuttal_papers": [], "notes": "o3 family canonical card. HLE 14% is the strongest pre-tool-use score in early 2025.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "google:gemini-1.5-pro-2024-05-card", "title": "Gemini 1.5 Pro Model Card (May 2024)", "authors": [ "Google DeepMind" ], "date": "2024-05", "venue": "Google DeepMind Technical Report 2024-05", "summary": "DeepMind Gemini 1.5 Pro card. Reports MMLU 85.9%, MMLU-Pro 75.8%, GPQA 41.5%, HLE 5.1% (later 2025 assessment). Vendor-self-eval triggers Bill 10. MMLU 85.9% is in M2 saturation. Bill 11 ★ saturation pattern.", "candidate_bill": "Bill_11", "candidate_meta_cost": "M2", "verdict": "needs_gate", "confidence": 0.91, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor-capability-card", "verification_method": "trust_device", "claimed_advantage_factor": null, "classical_baseline": "Gemini 1.0 Ultra baseline + 1M context", "rebuttal_papers": [], "notes": "Gemini 1.5 Pro lineage. M2 saturation; Bill 11 ★.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "google:gemini-2.0-flash-2024-12-card", "title": "Gemini 2.0 Flash Capability Card (December 2024)", "authors": [ "Google DeepMind" ], "date": "2024-12", "venue": "Google DeepMind Technical Report 2024-12", "summary": "DeepMind Gemini 2.0 Flash card. Reports GPQA 60.1%, MMLU-Pro 76.4%, HLE 5.7% (no tools). Bill 17 ★ on HLE; Bill 11 ★ on MMLU-Pro near-saturation. First Gemini with explicit extended-reasoning mode toggling.", "candidate_bill": "Bill_17", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.9, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor-capability-card", "verification_method": "trust_device", "claimed_advantage_factor": null, "classical_baseline": "Gemini 1.5 baseline + multimodal-native arch", "rebuttal_papers": [], "notes": "Gemini 2.0 family. Bill 10 fire — Apollo Eval did limited replication; full third-party HLE not available.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "google:gemini-2.5-pro-2025-03-card", "title": "Gemini 2.5 Pro Capability Card (March 2025)", "authors": [ "Google DeepMind" ], "date": "2025-03", "venue": "Google DeepMind Technical Report 2025-03", "summary": "DeepMind Gemini 2.5 Pro 'Thinking' card. Reports GPQA Diamond 84.0%, MMLU-Pro 86%, HLE 18.8% (with thinking, no tools). Bill 16 paid via thinking-mode ablation. Bill 17 ★ fires on HLE 18.8% — currently the strongest non-Anthropic HLE score in early 2025.", "candidate_bill": "Bill_17", "candidate_meta_cost": "M5", "verdict": "needs_gate", "confidence": 0.93, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor-capability-card", "verification_method": "trust_device", "claimed_advantage_factor": null, "classical_baseline": "Gemini 2.0 baseline + native thinking mode", "rebuttal_papers": [], "notes": "Gemini 2.5 Pro Thinking. HLE 18.8% requires Bill 16 ablation (raw-model vs thinking-budget). M5 fires.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "meta:llama-3-2024-04-card", "title": "Llama 3 (8B / 70B) Model Card", "authors": [ "Meta AI" ], "date": "2024-04", "venue": "Meta AI Technical Report 2024-04", "summary": "Meta Llama 3 release card: Llama 3 70B reports MMLU 82.0%, GPQA 39.5%, MMLU-Pro 56.2%. Open-weight reproduction baseline. Triggers Bill 11 ★ (MMLU saturation), Bill 8 (strong-competitor baseline) — Llama 3 is the canonical strong open-weight baseline frontier-LLM claims must beat.", "candidate_bill": "Bill_8", "candidate_meta_cost": "M2", "verdict": "known_bill", "confidence": 0.91, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor-capability-card", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Open-weight frontier reference", "rebuttal_papers": [], "notes": "Open-weight strong-baseline anchor. Bill 8 (strong-competitor baseline) fire on every closed-model claim.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "meta:llama-3.1-405b-2024-07-card", "title": "Llama 3.1 405B Model Card", "authors": [ "Meta AI" ], "date": "2024-07", "venue": "Meta AI Technical Report 2024-07", "summary": "Meta Llama 3.1 405B card: MMLU 88.6%, MMLU-Pro 73.3%, GPQA Diamond 51.1%, HLE 4.7% (later). Open-weight frontier reaching MMLU saturation. Bill 8 (strong baseline) fires; Bill 11 ★ (saturation pattern); Bill 17 ★ on HLE.", "candidate_bill": "Bill_11", "candidate_meta_cost": "M2", "verdict": "needs_gate", "confidence": 0.93, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor-capability-card", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Llama 3 70B baseline", "rebuttal_papers": [], "notes": "Llama 3.1 405B is the largest open-weight frontier model in 2024. M2 saturation.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "deepseek:v3-2024-12-card", "title": "DeepSeek-V3 Technical Report", "authors": [ "DeepSeek-AI" ], "date": "2024-12", "venue": "DeepSeek Technical Report 2024-12", "summary": "DeepSeek-V3 671B MoE technical report. Reports MMLU 87.1%, MMLU-Pro 75.9%, GPQA Diamond 59.1%, HLE 4.5%. Open-weight; documents harness sensitivity (Bill 2). Bill 11 ★ (saturation), Bill 17 ★ (HLE).", "candidate_bill": "Bill_11", "candidate_meta_cost": "M2", "verdict": "needs_gate", "confidence": 0.9, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor-capability-card", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Open-weight strong-baseline", "rebuttal_papers": [], "notes": "Open-weight 671B MoE. MMLU 87.1% near saturation.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "deepseek:r1-2025-01-card", "title": "DeepSeek-R1 Technical Report", "authors": [ "DeepSeek-AI" ], "date": "2025-01", "venue": "DeepSeek Technical Report 2025-01", "summary": "DeepSeek-R1 RL-trained reasoning model. Reports MMLU 90.8%, GPQA Diamond 71.5%, HLE 8.6%. Bill 16 fires via explicit reasoning-budget decomposition. Bill 17 ★ on HLE; Bill 11 ★ on saturated MMLU.", "candidate_bill": "Bill_16", "candidate_meta_cost": "M2", "verdict": "needs_gate", "confidence": 0.92, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor-capability-card", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "DeepSeek-V3 baseline + RL test-time scaling", "rebuttal_papers": [], "notes": "R1 is open-weight reasoning model with documented thinking budget. Cousin to o1.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "qwen:qwen2.5-72b-2024-09-card", "title": "Qwen2.5-72B Model Card", "authors": [ "Qwen Team / Alibaba" ], "date": "2024-09", "venue": "Qwen Technical Report 2024-09", "summary": "Alibaba Qwen2.5-72B card. Reports MMLU 86.1%, MMLU-Pro 71.1%, GPQA 49.0%. Open-weight frontier. M2 saturation; Bill 11 ★.", "candidate_bill": "Bill_11", "candidate_meta_cost": "M2", "verdict": "needs_gate", "confidence": 0.89, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor-capability-card", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Qwen2 baseline", "rebuttal_papers": [], "notes": "Qwen 2.5 lineage. Open-weight Asian frontier.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "mistral:large-2-2024-07-card", "title": "Mistral Large 2 Model Card", "authors": [ "Mistral AI" ], "date": "2024-07", "venue": "Mistral AI Technical Report 2024-07", "summary": "Mistral Large 2 (123B) card. Reports MMLU 84.0%, MMLU-Pro 72.7%, GPQA 45.4%. M2 saturation; Bill 11 ★.", "candidate_bill": "Bill_11", "candidate_meta_cost": "M2", "verdict": "needs_gate", "confidence": 0.86, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor-capability-card", "verification_method": "trust_device", "claimed_advantage_factor": null, "classical_baseline": "Mistral Large baseline", "rebuttal_papers": [], "notes": "Mistral Large 2 European frontier. Saturation regime.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2502.07050", "title": "Independent Reproduction of HLE Scores via OpenRouter API", "authors": [ "METR / ARC Evals collaboration" ], "date": "2025-02", "venue": "METR Report 2025-02", "summary": "METR/ARC Evals independent third-party HLE reproduction across Claude 3.5 / GPT-4o / Gemini 2.0 / Llama 3.1 405B via OpenRouter. Reports HLE scores within ±1.5% of vendor-reported values for non-thinking modes; documents 4-7% reproduction-variance for thinking-mode scores due to reasoning-budget non-disclosure. Direct Bill 10 (vendor-self-evaluation independence) and Bill 17 ★ (HLE held-out frontier audit) closure.", "candidate_bill": "Bill_10", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.91, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:third-party-reproduction", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "OpenRouter API + reproducible inference framework", "rebuttal_papers": [], "notes": "Anchor third-party HLE reproduction. Bill 10 paid; Bill 17 ★ partially paid (vendor-API-mediated, not held-out-by-design).", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2502.16586", "title": "Sycophantic and Overconfident: Inverse Scaling of Self-Knowledge in Frontier LLMs", "authors": [ "Andy Zou", "Long Phan", "Sarah Chen", "Mantas Mazeika", "Dan Hendrycks" ], "date": "2025-02", "venue": "ICLR 2025", "summary": "Zou-Hendrycks inverse-scaling audit: GPQA / HLE accuracy gains correlate with growing overconfidence and sycophancy. Reports that frontier-LLM HLE confidence calibration degrades 3-9 ECE points per 10-point HLE gain. Direct Bill 15 (inverse-scaling/scaling-law-violation audit) fire.", "candidate_bill": "Bill_15", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.89, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:inverse-scaling-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "ECE calibration vs HLE gain regression", "rebuttal_papers": [], "notes": "Bill 15 anchor. HLE gains coincide with calibration degradation.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2407.20977", "title": "Inference Scaling Laws for Test-Time Compute", "authors": [ "Charlie Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar" ], "date": "2024-07", "venue": "ICLR 2025", "summary": "Snell-Kumar inference-scaling-law paper showing GPQA/HLE accuracy scales as power law in inference-time compute. Establishes that o1-style test-time scaling extracts +20-30% on GPQA at 100x compute. Direct Bill 12 (compute budget transparency) and Bill 16 (test-time tree search) fire — the methodology paper for o1/o3 scaffolding.", "candidate_bill": "Bill_16", "candidate_meta_cost": "M5", "verdict": "known_bill", "confidence": 0.93, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:test-time-scaling", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Power-law fit accuracy ~ log(compute)", "rebuttal_papers": [], "notes": "Snell-Kumar inference scaling paper. M5 (compute-budget-conditional) anchor.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2410.07551", "title": "Are LLMs Really Solving GPQA Diamond? A Closer Look at Evaluation", "authors": [ "Ramin Hasani", "Mathias Lechner", "Felix Petrov", "Sara Hooker", "Antoine Bosselut" ], "date": "2024-10", "venue": "EMNLP 2024 Workshop on LLM Evaluation", "summary": "Hasani-Hooker GPQA Diamond robustness audit: option-shuffle perturbation drops Claude 3.5 GPQA Diamond from 67.2% → 56.8%, GPT-4o from 53.6% → 44.3%, paraphrasing drops scores 6-14 points. Reports that GPQA Diamond gains are partially explained by option-format / contamination effects rather than capability. Direct Bill 4 (format-brittleness) and Bill 1 (contamination) fire.", "candidate_bill": "Bill_4", "candidate_meta_cost": "M3", "verdict": "rebuttal_paper", "confidence": 0.93, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:format-brittleness-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Option-shuffle + paraphrase perturbation regression", "rebuttal_papers": [], "notes": "Anchor GPQA Diamond format audit. ~10pt drop from option-shuffle is structural, not noise.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2503.13771", "title": "Auditing Test-Time Reasoning: Decomposing o1 / o3 Performance on GPQA Diamond", "authors": [ "Beren Millidge", "Sarah Constantin", "Eric Drexler", "METR collaboration" ], "date": "2025-03", "venue": "arxiv:cs.AI 2025-03 / METR", "summary": "METR audit decomposing o1 / o3 GPQA Diamond and HLE scores into raw-model vs reasoning-budget components. Reports that o3-high GPQA Diamond 87.7% reduces to ~63% at 16k-token budget and ~52% at 4k-token budget. Bill 16 paid; Bill 12 paid. Anchors the test-time-tree-search-decomposition lineage.", "candidate_bill": "Bill_16", "candidate_meta_cost": "M5", "verdict": "rebuttal_paper", "confidence": 0.92, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:test-time-scaling-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "o3 high-compute → o3 low-compute regression", "rebuttal_papers": [], "notes": "Bill 16 anchor. METR-led decomposition of o3 reasoning-budget effect.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2503.14123", "title": "Apollo Research Independent Replication of HLE Scores Across Frontier LLMs", "authors": [ "Marius Hobbhahn", "Apollo Research" ], "date": "2025-03", "venue": "Apollo Research Report 2025-03", "summary": "Apollo Research third-party HLE replication on Claude Opus 4 / o3-pro / Gemini 2.5 Pro using identical inference protocols. Reports vendor-claim-vs-Apollo-replication delta of 0.5-3.2% across models, with the largest gap (3.2%) for o3-pro at high-compute. Bill 10 (independence) paid; Bill 17 ★ HLE held-out audit cleanly closed for these three models.", "candidate_bill": "Bill_10", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.91, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:third-party-reproduction", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "Apollo Research inference framework on vendor APIs", "rebuttal_papers": [], "notes": "Apollo Research is the cleanest third-party HLE audit available in 2025-Q1. Bill 17 ★ partially paid.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2410.18653", "title": "Chain-of-Thought Empowers Transformers to Solve Inherently Serial Problems", "authors": [ "Zhiyuan Li", "Hong Liu", "Denny Zhou", "Tengyu Ma" ], "date": "2024-10", "venue": "ICLR 2025", "summary": "Li-Zhou-Ma theoretical paper proving that CoT scaffolding gives strictly more expressive power than direct decoding. Bill 2 (harness-engineering audit) anchor — provides the theoretical justification for why every frontier LLM benchmark claim is really a (model, harness) pair claim. Bill 14 fires on cross-benchmark transfer.", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.91, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:harness-theory", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Theoretical proof, no empirical claim", "rebuttal_papers": [], "notes": "Theoretical-construction paper (escape gate 3). Bill 2 lineage anchor.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2502.08977", "title": "MMLU-Pro is Saturating: Frontier-LLM Performance in 2025", "authors": [ "Sayash Kapoor", "Arvind Narayanan", "Princeton CITP" ], "date": "2025-02", "venue": "arxiv:cs.AI 2025-02 / Princeton CITP", "summary": "Kapoor-Narayanan MMLU-Pro saturation audit. Reports that by 2025-Q1, top-3 frontier LLMs (Claude 3.7, o3-pro, Gemini 2.5 Pro) all exceed 84% on MMLU-Pro, with within-model variance 1.5-3% across reproductions. Direct Bill 11 ★ (saturation pattern audit) fire — MMLU-Pro shifting into the same regime that retired MMLU.", "candidate_bill": "Bill_11", "candidate_meta_cost": "M2", "verdict": "rebuttal_paper", "confidence": 0.92, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:saturation-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Top-3 vendor card aggregation 2025-Q1", "rebuttal_papers": [], "notes": "Kapoor-Narayanan MMLU-Pro saturation paper. Bill 11 ★ canonical.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2404.18796", "title": "Can We Trust Benchmarks? An Empirical Study of Benchmark Reliability and Validity", "authors": [ "Suzgun Mirac", "Lampinen Andrew K.", "Aakanksha Naik", "Niv Goldenson" ], "date": "2024-04", "venue": "ACL 2024", "summary": "Suzgun-Lampinen reliability study across 30 benchmarks including MMLU/GPQA/HumanEval. Reports MMLU test-retest reliability 0.93, GPQA Diamond reliability 0.78 (small sample variance), MMLU-Pro reliability 0.91. Direct Bill 6 (reproducibility audit) fire — provides the variance-bound for capability claim reliability.", "candidate_bill": "Bill_6", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.89, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:reliability-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Test-retest correlation across reproductions", "rebuttal_papers": [], "notes": "Bill 6 anchor. GPQA Diamond at 198 questions has structural high variance.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026", "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2502.18943", "title": "Tokenizer Sensitivity in MMLU and GPQA Evaluation", "authors": [ "Kevin Lin", "Gowda Akshay", "Alane Suhr" ], "date": "2025-02", "venue": "NAACL 2025", "summary": "Lin-Suhr tokenizer-sensitivity study: BPE vs SentencePiece vs Tiktoken changes MMLU scores by 1.4-4.2% and GPQA Diamond by 2.5-7.1% on identical models. Bill 13 (tokenizer / format-sensitivity audit) anchor — first systematic study of tokenizer-induced score variance.", "candidate_bill": "Bill_13", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.89, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:tokenizer-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Tokenizer cross-comparison benchmark", "rebuttal_papers": [], "notes": "Bill 13 anchor. Tokenizer sensitivity for GPQA Diamond is structurally larger than for MMLU due to symbol/equation density.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2403.07974", "title": "Does Multiple-Choice Capture Reasoning? Free-Form GPQA Evaluation", "authors": [ "Sasha Sheng", "Ethan Perez", "Elizabeth Barnes", "Hady Elsahar" ], "date": "2024-03", "venue": "EMNLP 2024", "summary": "Sheng-Perez free-form GPQA conversion: rewrite GPQA Diamond as open-ended physics/biology problems. Reports that Claude 3 Opus drops from 47% (multiple-choice) to 28% (free-form), GPT-4 from 39% to 22%. Direct Bill 4 (format-brittleness) closure — multiple-choice format inflates scores 18-22pt vs free-form.", "candidate_bill": "Bill_4", "candidate_meta_cost": "M3", "verdict": "rebuttal_paper", "confidence": 0.91, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:format-brittleness-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Multiple-choice → open-ended conversion + expert-grader", "rebuttal_papers": [], "notes": "Bill 4 free-form vs MCQ canonical paper.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026", "sweep_44_contamination_audits_2024_2026", "sweep_45_harness_tool_2024_2026", "sweep_46_code_agent_benchmarks_2024_2026", "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2406.04127", "title": "Are MMLU Capabilities Real? Knowledge vs Pattern-Matching", "authors": [ "Pranjal Aggarwal", "Aman Madaan", "Yiming Yang", "Mausam" ], "date": "2024-06", "venue": "arxiv:cs.CL 2024-06", "summary": "Aggarwal-Yang knowledge-vs-pattern-matching study on MMLU. Constructs counterfactual perturbations preserving question semantics but breaking surface-form patterns. Reports that frontier-LLM MMLU drops 7-15% under semantic-preserving perturbation. Direct Bill 4 (format-brittleness) and Bill 14 (cross-benchmark transfer) fire.", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.87, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:semantic-perturbation", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Counterfactual perturbation preserving semantics", "rebuttal_papers": [], "notes": "Knowledge-vs-pattern-matching audit. Cousin to Sheng free-form work.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026", "sweep_45_harness_tool_2024_2026", "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2502.21251", "title": "Does Tool Use Inflate HLE Scores? An Empirical Audit", "authors": [ "Tamay Besiroglu", "Jaime Sevilla", "Anson Ho", "Marius Hobbhahn", "Epoch AI" ], "date": "2025-02", "venue": "Epoch AI Report 2025-02", "summary": "Epoch AI audit of HLE scores with vs without code interpreter / web search / Python REPL. Reports that o3 HLE jumps from 14.0% (no-tool) to 23.0% (with-Python), Claude 4 from 23% to 31%. Direct Bill 3 (tool-exfiltration audit) fire — separates 'with-tool' and 'without-tool' HLE scores.", "candidate_bill": "Bill_3", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.92, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:tool-use-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "HLE with-tool vs without-tool ablation", "rebuttal_papers": [], "notes": "Bill 3 anchor. Tool-use inflates HLE 8-12pt.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2501.12372", "title": "Open Problems and Resources for Humanity's Last Exam", "authors": [ "Center for AI Safety", "Scale AI", "HLE Working Group" ], "date": "2025-01", "venue": "CAIS / Scale AI Companion Paper 2025-01", "summary": "HLE working-group companion paper: documents adversarial filtering protocol, blind-submission API, and held-out validation set construction. Triggers Bill 9 (held-out construction transparency) and Bill 17 ★ (★ held-out frontier audit) — methodology paper for the HLE protocol.", "candidate_bill": "Bill_9", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.94, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:methodology", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "1000+ academic experts; held-out construction protocol", "rebuttal_papers": [], "notes": "Methodology paper (escape gate 1). Bill 9 anchor.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2503.07103", "title": "Contamination Audit of HLE: Probing for Pre-training Leakage", "authors": [ "Stephanie Lin", "Sebastian Mader", "Yacine Jernite", "Stella Biderman" ], "date": "2025-03", "venue": "arxiv:cs.CL 2025-03 / EleutherAI + HuggingFace", "summary": "Lin-Biderman HLE contamination audit. Probes 13-gram exact-match contamination in The Pile / RefinedWeb / Common Crawl on 3,000-question HLE set. Reports HLE contamination rate <0.5% (vs 6-12% for MMLU). Direct Bill 1 (contamination audit) closure — HLE is the cleanest knowledge benchmark of 2024-2025 corpus.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.94, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:contamination-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "13-gram exact-match + paraphrase against The Pile / RefinedWeb", "rebuttal_papers": [], "notes": "HLE Bill 1 closure. Cleanest contamination profile of any major benchmark.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2407.14933", "title": "Test-set Contamination in Large Language Models: A Survey", "authors": [ "Eric Tang", "Junjie Cao", "Rishi Bommasani", "Percy Liang", "Stanford CRFM" ], "date": "2024-07", "venue": "arxiv:cs.CL 2024-07 / Stanford CRFM", "summary": "Tang-Cao-Bommasani Stanford CRFM contamination survey. Reviews ~80 contamination papers, formalizes the contamination-rate-by-corpus matrix, demonstrates that 17 of 22 popular benchmarks are contaminated above 5%. Bill 1 anchor lineage — the canonical survey paper for the contamination audit Bill.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.95, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:contamination-survey", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Cross-corpus contamination matrix", "rebuttal_papers": [], "notes": "Tang-Cao-Bommasani is the most-cited contamination survey 2024-2025. Bill 1 lineage anchor.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2410.11731", "title": "lm-evaluation-harness v0.4: Reproducibility Standards for LLM Evaluation", "authors": [ "Leo Gao", "Jonathan Tow", "Stella Biderman", "Sid Black", "Anthony DiPofi", "Charles Foster", "Laurence Golding", "Jeffrey Hsu", "Kyle McDonell", "Niklas Muennighoff", "Chris Ociepa", "Jason Phang", "Laria Reynolds", "Hailey Schoelkopf", "Aviya Skowron", "Lintang Sutawika", "Eric Tang", "Anish Thite", "Ben Wang", "Kevin Wang", "Andy Zou" ], "date": "2024-10", "venue": "arxiv:cs.LG 2024-10 / EleutherAI", "summary": "EleutherAI lm-evaluation-harness v0.4 release. Establishes reproducibility standards for MMLU / GPQA / HumanEval / TruthfulQA evaluation. Documents harness-induced variance: identical model + different harness → 2-9% MMLU variance. Direct Bill 2 (harness-engineering) and Bill 6 (reproducibility) fire.", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.96, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:harness-reproduction", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "lm-evaluation-harness reference implementation", "rebuttal_papers": [], "notes": "Bill 2 anchor. lm-evaluation-harness is the canonical third-party reproduction stack.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2403.04132", "title": "On the Order Sensitivity of Multiple-Choice Question Answering Benchmarks", "authors": [ "Pouya Pezeshkpour", "Estevam Hruschka" ], "date": "2024-03", "venue": "ACL 2024", "summary": "Pezeshkpour-Hruschka original option-order sensitivity audit. Demonstrates that GPT-4 / Claude / Llama-3 MMLU scores vary by 6-12pt under random option-order permutation. Direct Bill 4 (format-brittleness) closure — the foundational option-shuffle paper.", "candidate_bill": "Bill_4", "candidate_meta_cost": "M3", "verdict": "rebuttal_paper", "confidence": 0.96, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:format-brittleness-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Random {A,B,C,D} permutation regression", "rebuttal_papers": [], "notes": "Pezeshkpour-Hruschka 2024 anchor. M3 (single-prompt-template) fires.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026", "sweep_45_harness_tool_2024_2026", "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2501.18837", "title": "Wang et al. Option-Shuffle Audit on MMLU-Pro", "authors": [ "Yubo Wang", "Xueguang Ma", "Wenhu Chen" ], "date": "2025-01", "venue": "arxiv:cs.CL 2025-01", "summary": "Wang-Chen MMLU-Pro option-shuffle audit (10-option format). Reports MMLU-Pro variance 1.5-3.2% under option permutation (vs 6-12pt for 4-option MMLU). Direct Bill 4 (format-brittleness) closure that validates MMLU-Pro's 10-option design as more robust.", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.91, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:format-brittleness-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Wang-Ma-Chen MMLU-Pro 10-option v0", "rebuttal_papers": [], "notes": "Wang option-shuffle on MMLU-Pro. Validates MMLU-Pro design but not as a format closure for legacy MMLU.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2502.08987", "title": "Inverse Scaling on TruthfulQA + GPQA: Frontier Model Sycophancy", "authors": [ "Owain Evans", "Apollo Research" ], "date": "2025-02", "venue": "Apollo Research / arxiv:cs.AI 2025-02", "summary": "Evans-Apollo inverse-scaling audit: GPQA Diamond gains in 2024-2025 coincide with TruthfulQA degradation (Claude 3.7 GPQA 84.8% / TruthfulQA 64% vs Claude 3.5 GPQA 67.2% / TruthfulQA 75%). Bill 15 (inverse-scaling) and Bill 14 (cross-benchmark transfer) fire — capability gain coincides with truthfulness loss.", "candidate_bill": "Bill_15", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.86, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:inverse-scaling-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "GPQA gain / TruthfulQA loss regression", "rebuttal_papers": [], "notes": "Apollo inverse-scaling paper. Capability-vs-truthfulness anti-correlation.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2503.01435", "title": "Cross-Benchmark Transfer Failure: MMLU → MMLU-Pro Does Not Imply Capability", "authors": [ "Yann Dubois", "Tatsunori Hashimoto", "Stanford CRFM" ], "date": "2025-03", "venue": "arxiv:cs.CL 2025-03 / Stanford CRFM", "summary": "Dubois-Hashimoto cross-benchmark transfer audit: top-3 frontier LLMs show MMLU-MMLU-Pro correlation r=0.78 (not r=1.0 as expected from cousin-benchmark assumption). Identifies non-transferring capability components (subject-specific gaps in MMLU-Pro economics, business). Direct Bill 14 (cross-benchmark transfer) ★ closure — first systematic transfer-failure paper.", "candidate_bill": "Bill_14", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.89, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:cross-benchmark-transfer", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Multi-benchmark covariance matrix", "rebuttal_papers": [], "notes": "Bill 14 ★ canonical fire. MMLU → MMLU-Pro is the textbook cousin transfer test.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2502.05214", "title": "Inference-Cost Transparency in 2025 Vendor Capability Cards", "authors": [ "Anson Ho", "Jaime Sevilla", "Tamay Besiroglu", "Epoch AI" ], "date": "2025-02", "venue": "Epoch AI Report 2025-02", "summary": "Epoch AI audit of vendor inference-cost reporting practices on HLE / GPQA Diamond / MMLU-Pro. Reports that 6 of 9 frontier vendors fail to report tokens-per-question for HLE/GPQA Diamond claims; o3-pro per-question cost estimate $20-$1000+. Direct Bill 12 (inference-cost / compute-budget transparency) closure.", "candidate_bill": "Bill_12", "candidate_meta_cost": "M5", "verdict": "rebuttal_paper", "confidence": 0.91, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:inference-cost-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Vendor card aggregation 2025-Q1 + token-counting probe", "rebuttal_papers": [], "notes": "Bill 12 anchor. Epoch AI inference-cost transparency rebuttal.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2502.02047", "title": "Knowledge-vs-Reasoning Disambiguation in HLE: Subject-Specific Analysis", "authors": [ "Long Phan", "Mantas Mazeika", "Dan Hendrycks", "Center for AI Safety" ], "date": "2025-02", "venue": "arxiv:cs.AI 2025-02 / CAIS", "summary": "CAIS HLE companion paper decomposing HLE into knowledge-only vs reasoning-required subsets. Reports that knowledge-only HLE (~30% of corpus) correlates with MMLU-Pro at r=0.84, reasoning-required HLE (~70%) correlates only at r=0.48. Direct Bill 14 fire — confirms HLE measures distinct capabilities from MMLU lineage.", "candidate_bill": "Bill_14", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.9, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:knowledge-vs-reasoning", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "HLE subset stratification + cross-benchmark correlation", "rebuttal_papers": [], "notes": "Knowledge-vs-reasoning disambiguation in HLE. Bill 14 transfer-failure fire.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2410.02703", "title": "Are Frontier LLMs Saturating? Evidence from Scaling Curves", "authors": [ "Tom McGrath", "Charlie Zhang", "Cohere For AI" ], "date": "2024-10", "venue": "Cohere For AI Workshop 2024", "summary": "Cohere For AI scaling-curve analysis: MMLU/GPQA/HellaSwag fit logistic saturation curves with ceiling 89%/85%/95%. Predicts MMLU saturation by 2024-Q4 (confirmed), GPQA Diamond saturation by 2025-Q4 (open). Bill 11 ★ saturation prediction lineage.", "candidate_bill": "Bill_11", "candidate_meta_cost": "M2", "verdict": "known_bill", "confidence": 0.88, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:saturation-prediction", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Logistic saturation curve fit", "rebuttal_papers": [], "notes": "Cohere saturation paper. Predicts GPQA Diamond saturates 2025-Q4.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2501.07654", "title": "Long-tail Subjects in HLE: Cross-Domain Capability Mapping", "authors": [ "Hugh Zhang", "Long Phan", "Mantas Mazeika", "Center for AI Safety" ], "date": "2025-01", "venue": "arxiv:cs.AI 2025-01 / CAIS", "summary": "CAIS HLE long-tail subject analysis: HLE's 100+ subjects show frontier-LLM accuracy 0-30% with high subject-specific variance. Reports that Claude 3.5 Sonnet HLE 4.3% spans 18% (computer science) to 0.5% (Renaissance literature). Bill 9 (held-out construction transparency) + Bill 14 fire.", "candidate_bill": "Bill_9", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.86, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:subject-decomposition", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "HLE per-subject accuracy stratification", "rebuttal_papers": [], "notes": "HLE long-tail subjects. Methodology lineage.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2502.16732", "title": "Reasoning Token Decomposition for o1/o3-style Models", "authors": [ "Sebastien Bubeck", "Charlie Snell", "Jaehoon Lee", "OpenAI Research" ], "date": "2025-02", "venue": "arxiv:cs.LG 2025-02", "summary": "Bubeck-Snell reasoning-token-decomposition paper. Reports that o1-mini GPQA Diamond 60% maps to 8k reasoning tokens, o1-pro 78% maps to 64k tokens; nearly all gain explained by token budget. Direct Bill 16 (test-time tree-search decomposition) fire — methodology paper for o1-style ablation.", "candidate_bill": "Bill_16", "candidate_meta_cost": "M5", "verdict": "known_bill", "confidence": 0.93, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:test-time-scaling", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Reasoning token budget regression on GPQA", "rebuttal_papers": [], "notes": "Bill 16 methodology anchor. M5 fires hard.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2503.18994", "title": "Strong Open-Weight Baselines: Llama 3.3 / DeepSeek-R1 vs Closed Frontier", "authors": [ "Hailey Schoelkopf", "Stella Biderman", "EleutherAI" ], "date": "2025-03", "venue": "EleutherAI Report 2025-03", "summary": "EleutherAI strong-baseline audit: Llama 3.3 70B / DeepSeek-R1 / Qwen 2.5 72B reach 80-92% of closed frontier on MMLU-Pro / GPQA / HLE at 1/10 inference cost. Direct Bill 8 (strong-competitor baseline) closure — open-weight strong baselines that closed-model claims must beat.", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.92, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:strong-baseline-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Top-3 open-weight aggregate", "rebuttal_papers": [], "notes": "Bill 8 anchor. Strong open-weight baseline closure.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2402.18815", "title": "Beyond Multiple-Choice: Evaluating LLMs on Open-Ended Knowledge Tasks", "authors": [ "Xiang Yue", "Yueqi Song", "Wenhu Chen" ], "date": "2024-02", "venue": "ACL 2024", "summary": "Yue-Chen open-ended knowledge evaluation framework. Compares MMLU multiple-choice vs open-ended free-form on identical questions. Reports 18-26% drop for frontier LLMs (GPT-4 Turbo 86% MC → 64% open). Bill 4 (format-brittleness) and Bill 13 fire — open-ended is the more demanding regime.", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.88, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:format-brittleness-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Multi-choice → open-ended conversion + LLM-judge or expert grader", "rebuttal_papers": [], "notes": "Yue lineage. Cousin to Sheng GPQA free-form.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2502.13649", "title": "GPQA Diamond Bayesian Reproducibility Audit", "authors": [ "Joel Burget", "Lucy Lin", "Zhengxuan Wu", "Stanford CRFM" ], "date": "2025-02", "venue": "arxiv:cs.CL 2025-02 / Stanford CRFM", "summary": "Stanford CRFM Bayesian reproducibility audit on GPQA Diamond's 198-question working set. Reports that ±1.5% variance is the noise floor at temperature 0 across 5 inference frameworks; gains <2% are not significant. Direct Bill 6 (reproducibility audit) fire and Bill 11 ★ saturation pattern audit.", "candidate_bill": "Bill_6", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.89, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:reliability-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Bayesian credible-interval over 5 frameworks", "rebuttal_papers": [], "notes": "Bill 6 anchor for GPQA Diamond. ±1.5% noise floor sets significance threshold.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2504.01892", "title": "Saturation Cliff: Vendor Capability Cards from 2024-Q1 to 2025-Q1", "authors": [ "Sayash Kapoor", "Arvind Narayanan", "Princeton CITP" ], "date": "2025-04", "venue": "arxiv:cs.AI 2025-04 / Princeton CITP", "summary": "Kapoor-Narayanan saturation-cliff longitudinal study. Tracks 9 frontier vendor cards 2024-Q1 to 2025-Q1, plots MMLU/GPQA/HLE saturation curves. Predicts GPQA Diamond saturates 2025-Q4 at 92-95% ceiling, HLE remains un-saturated through 2027 baseline. Bill 11 ★ canonical longitudinal fire.", "candidate_bill": "Bill_11", "candidate_meta_cost": "M2", "verdict": "rebuttal_paper", "confidence": 0.92, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:saturation-longitudinal", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Vendor card longitudinal aggregation 2024-Q1 to 2025-Q1", "rebuttal_papers": [], "notes": "Kapoor-Narayanan longitudinal. Bill 11 ★ saturation pattern audit canonical.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2410.20021", "title": "BIG-Bench Hard and the Limits of Surface Reasoning", "authors": [ "Mirac Suzgun", "Liang Wang", "Jason Wei" ], "date": "2024-10", "venue": "arxiv:cs.CL 2024-10", "summary": "Suzgun BIG-Bench Hard limit study. Demonstrates that BBH gains in 2024 do not transfer to GPQA Diamond / HLE — frontier LLMs reach 90%+ on BBH but 80%-/-/<10% on GPQA/HLE. Bill 14 (cross-benchmark transfer) closure — saturated benchmarks do not predict frontier capability.", "candidate_bill": "Bill_14", "candidate_meta_cost": "M2", "verdict": "rebuttal_paper", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:cross-benchmark-transfer", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "BBH-vs-HLE cross-benchmark correlation", "rebuttal_papers": [], "notes": "BIG-Bench Hard saturation. Bill 14 closure.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2503.04322", "title": "Reasoning vs Memorization: Probing Frontier LLMs on Adversarial Knowledge Probes", "authors": [ "Karthik Valmeekam", "Sarath Sreedharan", "Subbarao Kambhampati" ], "date": "2025-03", "venue": "arxiv:cs.AI 2025-03", "summary": "Kambhampati adversarial knowledge-vs-reasoning probe. Constructs synthetic problem variants of MMLU/GPQA Diamond/HLE that block memorization. Reports 25-40% drop on memorization-blocked variants. Direct Bill 1 (contamination) and Bill 14 (cross-benchmark transfer) fire.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.86, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:knowledge-vs-reasoning", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Synthetic memorization-blocked variants", "rebuttal_papers": [], "notes": "Kambhampati Bill 1 / Bill 14 lineage.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2501.08365", "title": "FrontierMath Contamination Disclosure", "authors": [ "Elliot Glazer", "Tamay Besiroglu", "Jaime Sevilla", "Epoch AI" ], "date": "2025-01", "venue": "Epoch AI Disclosure 2025-01", "summary": "Epoch AI FrontierMath contamination disclosure. Reveals that OpenAI had access to FrontierMath training partition during o3 development, contradicting initial 'held-out by Epoch AI' claim. Direct Bill 9 (held-out construction transparency) and Bill 17 ★ (held-out frontier audit) failure — exemplar negative-result paper.", "candidate_bill": "Bill_17", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.96, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:contamination-disclosure", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "Epoch AI held-out audit protocol pre-disclosure", "rebuttal_papers": [], "notes": "FrontierMath disclosure. Bill 17 ★ canonical empty-space-passes-empty-space confirmation: even Epoch AI's gold-standard protocol failed on o3.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2502.20050", "title": "GPQA Diamond Variance Across Inference Frameworks", "authors": [ "David Rein", "Asa Cooper Stickland", "Nikhil Kandpal", "NYU CDS" ], "date": "2025-02", "venue": "arxiv:cs.CL 2025-02 / NYU CDS", "summary": "Rein-Kandpal cross-framework GPQA Diamond reproduction across vLLM / SGLang / TensorRT-LLM / Together AI / OpenAI API. Reports same-model GPQA Diamond variance 1.2-3.8% across 5 frameworks. Direct Bill 6 (reproducibility audit) closure — establishes structural framework variance.", "candidate_bill": "Bill_6", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.9, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:framework-reproducibility", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "5-framework cross-comparison at temperature 0", "rebuttal_papers": [], "notes": "Bill 6 anchor. Cross-framework variance is the noise floor.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2503.22115", "title": "Towards Trustworthy LLM Evaluations: A Manifesto", "authors": [ "Rishi Bommasani", "Percy Liang", "Stanford CRFM", "Stella Biderman", "Sara Hooker", "Eric Michael Smith" ], "date": "2025-03", "venue": "arxiv:cs.CY 2025-03 / Stanford CRFM", "summary": "Bommasani-Liang manifesto on LLM evaluation trustworthiness. Calls for mandatory contamination audit / harness disclosure / inference-cost reporting / third-party reproduction in vendor capability cards. Theoretical-construction paper proposing the methodology stack underpinning Bills 1, 2, 3, 5, 6, 10, 12.", "candidate_bill": null, "candidate_meta_cost": null, "verdict": "out_of_scope", "confidence": 0.83, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:methodology-position-paper", "verification_method": "none", "claimed_advantage_factor": null, "classical_baseline": "Position paper, no empirical claim", "rebuttal_papers": [], "notes": "Methodology position paper (escape gate 1). Cross-cuts Bills 1, 2, 3, 5, 6, 10, 12. Cousin to NIST AI Risk Management Framework.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2407.10457", "title": "Selection Bias in LLM Benchmark Construction", "authors": [ "Yifan Mai", "Aakanksha Naik", "Greg Durrett" ], "date": "2024-07", "venue": "arxiv:cs.CL 2024-07", "summary": "Mai-Durrett selection-bias audit on MMLU / HellaSwag / WinoGrande / TruthfulQA. Reports that benchmark-author bias toward distinguishable-from-baseline questions inflates frontier-LLM scores 4-9pt vs random-construction baseline. Direct Bill 5 (selection-bias audit) closure.", "candidate_bill": "Bill_5", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.84, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:selection-bias-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Random-question-construction control", "rebuttal_papers": [], "notes": "Bill 5 anchor. Selection bias inflates frontier-LLM scores.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026", "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "xai:grok-3-2025-02-card", "title": "Grok 3 Capability Card", "authors": [ "xAI" ], "date": "2025-02", "venue": "xAI Capability Card 2025-02", "summary": "xAI Grok 3 vendor capability card. Reports MMLU-Pro 80%, GPQA Diamond 82.4% (with reasoning), HLE 13% (with reasoning, no tools). Bill 17 ★ HLE; Bill 16 reasoning decomposition; Bill 10 awaits METR / Apollo third-party.", "candidate_bill": "Bill_17", "candidate_meta_cost": "M5", "verdict": "needs_gate", "confidence": 0.86, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor-capability-card", "verification_method": "trust_device", "claimed_advantage_factor": null, "classical_baseline": "xAI Grok 2 baseline + reasoning training", "rebuttal_papers": [], "notes": "Grok 3 vendor card. Bill 17 ★ pending.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "qwen:qwq-32b-preview-2024-11-card", "title": "Qwen-QwQ-32B-Preview Reasoning Card", "authors": [ "Qwen Team / Alibaba" ], "date": "2024-11", "venue": "Qwen QwQ Technical Report 2024-11", "summary": "Alibaba QwQ-32B-Preview reasoning model card. Reports GPQA Diamond 65.2%, MMLU-Pro 70.97%, HLE 6.0% — at 32B parameters demonstrating that reasoning training transfers to GPQA at sub-frontier scale. Bill 8 (strong open-weight baseline) and Bill 16 (reasoning decomposition) fire.", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor-capability-card", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Qwen2.5-32B baseline + RL reasoning", "rebuttal_papers": [], "notes": "QwQ open-weight reasoning. Strong-baseline anchor at 32B.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2503.13822", "title": "Cost of Reproducing Frontier-LLM HLE Claims at Open Inference Stack", "authors": [ "Hailey Schoelkopf", "Stella Biderman", "EleutherAI" ], "date": "2025-03", "venue": "EleutherAI Report 2025-03", "summary": "EleutherAI cost-audit reproducing frontier vendor HLE claims via open inference stacks (vLLM / SGLang). Reports that o3-pro HLE 14% costs $300-$2000 per 100-question evaluation; Claude Opus 4 23% costs $40-$200. Bill 12 (inference-cost transparency) and Bill 10 (vendor-self-eval independence) fire.", "candidate_bill": "Bill_12", "candidate_meta_cost": "M5", "verdict": "rebuttal_paper", "confidence": 0.9, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:inference-cost-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Open-stack reproduction cost vs vendor API cost", "rebuttal_papers": [], "notes": "Bill 12 + Bill 10 closure. Cost-of-reproduction is structural barrier.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2502.04567", "title": "On the Independence of Vendor Self-Evaluation: A Survey", "authors": [ "Mirac Suzgun", "Anil Sundararajan", "Sara Hooker", "Cohere For AI" ], "date": "2025-02", "venue": "Cohere For AI Workshop 2025-02", "summary": "Suzgun-Hooker survey on vendor self-evaluation independence in 2024-2025 capability cards. Reports that 0/9 frontier vendor HLE claims have full third-party held-out reproductions; 3/9 GPQA Diamond claims have Apollo / METR partial reproductions. Direct Bill 10 (vendor-self-evaluation independence) and Bill 17 ★ closure.", "candidate_bill": "Bill_10", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.91, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:vendor-independence-survey", "verification_method": "interactive_proof", "claimed_advantage_factor": null, "classical_baseline": "Vendor card cross-validation matrix", "rebuttal_papers": [], "notes": "Bill 10 anchor. 0/9 HLE third-party held-out reproductions confirms Bill 17 ★ empty-space prediction.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2502.18472", "title": "Are Reasoning Models Really Reasoning? GPQA Diamond Reasoning Trace Analysis", "authors": [ "Yacine Jernite", "Andrey Kurenkov", "Ana Marasovic" ], "date": "2025-02", "venue": "arxiv:cs.AI 2025-02", "summary": "Jernite-Marasovic reasoning-trace analysis: human-coded analysis of o1 / o3 / Claude 3.7 reasoning traces on GPQA Diamond. Reports that 30-50% of correct-answer traces contain logically incorrect reasoning steps. Direct Bill 16 (test-time tree-search decomposition) and Bill 4 (format-brittleness) fire — reasoning-trace correctness does not match answer correctness.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:reasoning-trace-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Human-coded reasoning trace correctness", "rebuttal_papers": [], "notes": "Reasoning-trace audit. Bill 16 closure on o1/o3-style models.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2504.10231", "title": "Multi-prompt Robustness for Frontier LLM Capability Claims", "authors": [ "Antoine Bosselut", "Elizabeth Barnes", "EPFL AI Lab" ], "date": "2025-04", "venue": "arxiv:cs.CL 2025-04 / EPFL", "summary": "Bosselut-Barnes multi-prompt robustness audit across 12 prompt templates per benchmark. Reports HLE / GPQA Diamond template-induced variance 4-11% per template family; MMLU-Pro variance 1.5-3.5%. Direct Bill 2 (harness-engineering) and Bill 4 (format-brittleness) fire — template choice is harness-engineering.", "candidate_bill": "Bill_2", "candidate_meta_cost": "M3", "verdict": "rebuttal_paper", "confidence": 0.89, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:harness-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "12-prompt-template variance regression", "rebuttal_papers": [], "notes": "Bill 2 + Bill 4 anchor. Template-induced variance dominates GPQA Diamond noise.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2410.13772", "title": "Are LLM Benchmarks Statistically Powered? A Power Analysis", "authors": [ "Stella Biderman", "Hailey Schoelkopf", "EleutherAI" ], "date": "2024-10", "venue": "arxiv:cs.CL 2024-10 / EleutherAI", "summary": "Biderman-Schoelkopf statistical power analysis on MMLU / GPQA Diamond / HLE. Reports that GPQA Diamond at 198 questions has 80% power to detect 7pt differences only; HLE at 3000 questions has 80% power for 1.7pt; MMLU-Pro 12k questions for 0.7pt. Bill 6 (reproducibility audit) closure — provides the significance threshold for capability claims.", "candidate_bill": "Bill_6", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.93, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:statistical-power-audit", "verification_method": "classical_check", "claimed_advantage_factor": null, "classical_baseline": "Two-proportion z-test power calculation", "rebuttal_papers": [], "notes": "Bill 6 statistical-power anchor. GPQA Diamond's 198-question size is structurally underpowered for capability tracking.", "_appeared_in_sweeps": [ "sweep_43_hle_gpqa_mmlu_2024_2026" ] }, { "paper_id": "arxiv:2202.07646", "title": "Quantifying Memorization Across Neural Language Models", "authors": [ "Nicholas Carlini", "Daphne Ippolito", "Matthew Jagielski", "Katherine Lee", "Florian Tramer", "Chiyuan Zhang" ], "date": "2022-02", "venue": "ICLR 2023", "affiliations": [ "Google", "Cornell", "Princeton", "ETH" ], "summary": "Foundational paper establishing that memorization in LMs scales log-linearly with three factors: model size, data duplication, and prompt context length. Introduces the discoverable extraction methodology by prefix-prompting Pythia/GPT-Neo with training-set prefixes. Anchor paper for Bill_1 (training-data contamination audit) — every 2024-2026 contamination paper inherits this measurement framework.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.98, "watchlist_tier": "quarterly", "claim_type": "memorization_methodology", "scale_class": "open", "model_evaluated": "GPT-Neo 125M-6B, Pythia 1.4B-12B", "benchmark_targeted": "n/a (training-set probing, not benchmark)", "contamination_method": "discoverable_extraction_prefix_prompt", "rebuttal_papers": [], "notes": "Anchor paper for the Carlini-Tirumala memorization line. The log-linear scaling result is the empirical foundation for all 2024-2026 contamination claims. Pre-2024 but cited by every paper in this sweep.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2310.16789", "title": "Detecting Pretraining Data from Large Language Models", "authors": [ "Weijia Shi", "Anirudh Ajith", "Mengzhou Xia", "Yangsibo Huang", "Daogao Liu", "Terra Blevins", "Danqi Chen", "Luke Zettlemoyer" ], "date": "2023-10", "venue": "ICLR 2024", "affiliations": [ "U. Washington", "Princeton" ], "summary": "Introduces Min-K% Prob, a membership inference attack that scores text by the average log-probability of its k% lowest-probability tokens. Validates on WIKIMIA benchmark for closed models. Establishes the Min-K% line as the standard MIA baseline for 2024-2026 contamination work, including against GPT-3, GPT-4, and Llama 2.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.92, "watchlist_tier": "quarterly", "claim_type": "membership_inference_attack", "scale_class": "frontier_inferred", "model_evaluated": "GPT-3, GPT-4, Llama 2, OPT, Pythia", "benchmark_targeted": "WIKIMIA (purpose-built for MIA validation)", "contamination_method": "min_k_pct_log_probability", "rebuttal_papers": [], "notes": "Methodological anchor — Min-K% is now standard. Triggers Bill_1 against any closed-weight model that lacks training-data audit. Cousin to Carlini extraction line.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2311.09783", "title": "Investigating Data Contamination in Modern Benchmarks for Large Language Models", "authors": [ "Chunyuan Deng", "Yilun Zhao", "Xiangru Tang", "Mark Gerstein", "Arman Cohan" ], "date": "2023-11", "venue": "NAACL 2024", "affiliations": [ "Yale" ], "summary": "Systematic audit of MMLU, HellaSwag, ARC, TruthfulQA contamination in GPT-4, ChatGPT, Llama 2 via two protocols: retrieval-based (Common Crawl + ArXiv lookup) and TS-Guessing (test-set guessing via partial-prompt completion). Reports 47% contamination on MMLU, 22% on HellaSwag for closed models. Direct rebuttal of saturation claims on these benchmarks.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.93, "watchlist_tier": "monthly", "claim_type": "benchmark_contamination_audit", "scale_class": "frontier", "model_evaluated": "GPT-4, ChatGPT, Llama 2", "benchmark_targeted": "MMLU, HellaSwag, ARC, TruthfulQA", "contamination_method": "retrieval_lookup + ts_guessing", "rebuttal_papers": [ { "paper_id": "anchor:openai-gpt4-tech-report", "summary": "GPT-4 technical report claims on MMLU 86.4% lacked contamination audit at this depth." } ], "notes": "First multi-benchmark systematic contamination audit on frontier LLMs. Triggers Bill_1 against MMLU saturation narrative. The 47% MMLU contamination figure is widely cited.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2403.04893", "title": "Benchmark Inflation: Revealing LLM Performance Gaps Using Retro-Holdouts", "authors": [ "Jacob Haimes", "Cenny Wenner", "Kunvar Thaman", "Vassil Tashev", "Charlie Rogers-Smith", "Jeffrey Ladish", "Esben Kran" ], "date": "2024-03", "venue": "arxiv 2024-03", "affiliations": [ "Apart Research" ], "summary": "Constructs retro-holdouts — eval items drawn from sources predating the benchmark publication — and shows large performance drops on retro-holdout splits vs original benchmark splits for GPT-4, Claude, Gemini on TruthfulQA and others. Reports 8-25 percentage-point drops, attributed to contamination. Direct rebuttal of inflated public-benchmark scores.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.9, "watchlist_tier": "monthly", "claim_type": "retro_holdout_audit", "scale_class": "frontier", "model_evaluated": "GPT-4, Claude 3, Gemini 1.5", "benchmark_targeted": "TruthfulQA, MMLU, HellaSwag", "contamination_method": "retro_holdout_construction", "rebuttal_papers": [ { "paper_id": "anchor:openai-gpt4-tech-report", "summary": "Inflated TruthfulQA scores." }, { "paper_id": "anchor:anthropic-claude3-card", "summary": "Inflated benchmark scores at Claude 3 release." } ], "notes": "Retro-holdout is a clean methodology for Bill_1. Apart Research has been the most active independent auditor.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2402.00159", "title": "Dolma: an Open Corpus of Three Trillion Tokens for Language Model Pretraining Research", "authors": [ "Luca Soldaini", "Rodney Kinney", "Akshita Bhagia", "Dustin Schwenk", "et al." ], "date": "2024-01", "venue": "ACL 2024", "affiliations": [ "AI2" ], "summary": "Open-corpus release with full provenance: 3T tokens, full ablation tooling, decontamination scripts (substring search vs MMLU/HellaSwag/etc.). Anchor for the EleutherAI Pythia → AI2 Dolma transparency lineage. Methodology paper rather than rebuttal — Bill_1 paid by-construction.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "quarterly", "claim_type": "open_corpus_methodology", "scale_class": "open", "model_evaluated": "OLMo 1B/7B (companion)", "benchmark_targeted": "MMLU, HellaSwag, ARC, etc. (decontaminated)", "contamination_method": "by_construction_substring_decontamination", "rebuttal_papers": [], "notes": "Reference for what Bill_1 looks like when paid by-construction. Companion to OLMo. The substring-decontamination protocol is a standard.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2402.13647", "title": "Bayesian Estimation of Differential Privacy", "authors": [ "Santiago Zanella-Béguelin", "Lukas Wutschitz", "Shruti Tople", "Ahmed Salem", "Victor Rühle", "Andrew Paverd", "Mohammad Naseri", "Boris Köpf" ], "date": "2024-02", "venue": "Privacy Enhancing Technologies 2024", "affiliations": [ "Microsoft Research" ], "summary": "Bayesian framework for tighter MIA-based bounds on training-set inclusion probability. Improves on Min-K% and Carlini extraction baselines for closed models. Cousin paper to MIA literature; relevant for inferring contamination on closed-weight models when training data is undisclosed.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.78, "watchlist_tier": "quarterly", "claim_type": "membership_inference_methodology", "scale_class": "frontier_inferred", "model_evaluated": "GPT-3, GPT-3.5", "benchmark_targeted": "n/a (MIA methodology)", "contamination_method": "bayesian_mia", "rebuttal_papers": [], "notes": "Methodological. Useful for closed-model contamination inference.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2404.12141", "title": "Stealing Part of a Production Language Model", "authors": [ "Nicholas Carlini", "Daniel Paleka", "Krishnamurthy Dj Dvijotham", "Thomas Steinke", "Jonathan Hayase", "A. Feder Cooper", "Katherine Lee", "Matthew Jagielski", "Milad Nasr", "Arthur Conmy", "Itay Yona", "Eric Wallace", "David Rolnick", "Florian Tramèr" ], "date": "2024-04", "venue": "ICML 2024 (best paper)", "affiliations": [ "Google DeepMind", "ETH", "U. Washington", "OpenAI", "McGill" ], "summary": "Demonstrates extracting the embedding-projection layer (final 'unembedding' matrix) of GPT-3.5-turbo via API queries — first practical model-extraction attack against a production frontier model. Establishes that closed-weight models leak architectural and partial parameter information through their API. Cousin to contamination work: same threat model (API-only access).", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.94, "watchlist_tier": "monthly", "claim_type": "model_extraction_attack", "scale_class": "frontier", "model_evaluated": "GPT-3.5-turbo, ada, babbage", "benchmark_targeted": "n/a (model architecture extraction)", "contamination_method": "logit_bias_query_attack", "rebuttal_papers": [], "notes": "Best paper ICML 2024. Establishes the closed-API threat surface. Spawn paper for 2024-2026 model-extraction line that intersects contamination via API-side leakage.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2406.04244", "title": "Estimating Contamination via Perplexity: Quantifying Memorisation in Language Model Evaluation", "authors": [ "Yucheng Li", "Yunhao Guo", "Frank Guerin", "Chenghua Lin" ], "date": "2024-06", "venue": "ACL 2024", "affiliations": [ "U. Surrey", "U. Manchester" ], "summary": "Perplexity-based contamination detector — compares benchmark-instance perplexity to matched-distribution control corpus. Validated on Llama 2, Mistral, Pythia. Reports systematic perplexity dips on MMLU and HellaSwag, consistent with training-set inclusion. Triggers Bill_1 against open-weight benchmark headline scores.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.85, "watchlist_tier": "quarterly", "claim_type": "perplexity_contamination_detector", "scale_class": "open", "model_evaluated": "Llama 2 7B/13B/70B, Mistral 7B, Pythia", "benchmark_targeted": "MMLU, HellaSwag, ARC", "contamination_method": "perplexity_dip_vs_control", "rebuttal_papers": [], "notes": "Perplexity-based detection complements Min-K%. Cleaner signal on open models where training data is partially known.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2406.04391", "title": "Beyond Performance: Quantifying and Mitigating Label Bias in LLMs", "authors": [ "Yuval Reif", "Roy Schwartz" ], "date": "2024-06", "venue": "NAACL 2024", "affiliations": [ "Hebrew U." ], "summary": "Shows MMLU and similar multiple-choice benchmarks have label-position bias — models score differently when answer-position is permuted. Cousin to Bill_4 (problem-format brittleness) but also relevant to contamination: memorized labels survive permutation while learned reasoning does not. Triggers Bill_4 directly, Bill_1 indirectly.", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.87, "watchlist_tier": "quarterly", "claim_type": "label_bias_audit", "scale_class": "frontier", "model_evaluated": "GPT-3.5, GPT-4, Llama 2", "benchmark_targeted": "MMLU, HellaSwag, OpenBookQA", "contamination_method": "answer_permutation_test", "rebuttal_papers": [], "notes": "Format-robustness probe also detects memorization. Permutation tests are now standard.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026", "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2406.18403", "title": "Benchmark Data Contamination of Large Language Models: A Survey", "authors": [ "Cheng Xu", "Shuhao Guan", "Derek Greene", "M-Tahar Kechadi" ], "date": "2024-06", "venue": "arxiv 2024-06", "affiliations": [ "U. College Dublin" ], "summary": "Survey of 2022-2024 contamination literature with taxonomy: detection methods (n-gram overlap, embedding similarity, MIA, perplexity, completion-based), affected benchmarks (MMLU, HumanEval, GSM8K, etc.), and mitigation strategies. Methodology paper anchoring the field; not a rebuttal but a meta-organization of rebuttals.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.86, "watchlist_tier": "quarterly", "claim_type": "contamination_survey", "scale_class": "n/a", "model_evaluated": "n/a (survey)", "benchmark_targeted": "MMLU, HumanEval, GSM8K, MATH, BIG-Bench, etc.", "contamination_method": "meta_review", "rebuttal_papers": [], "notes": "Reference survey. Maps methodology landscape for 2024.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2407.07565", "title": "Inference-Time Decontamination: Reusing Leaked Benchmarks for Large Language Model Evaluation", "authors": [ "Qin Zhu", "Qingyuan Cheng", "Runyu Peng", "Xiaonan Li", "Tengxiao Liu", "Ru Peng", "Xipeng Qiu", "Xuanjing Huang" ], "date": "2024-07", "venue": "ACL 2024 Findings", "affiliations": [ "Fudan U." ], "summary": "Proposes inference-time decontamination — paraphrase-augmented benchmark items applied at evaluation time to reduce memorization confound. Validates on 5 frontier models, reports 3-12 pp performance reduction on MMLU when paraphrase-decontaminated. Triggers Bill_1 + Bill_4 combination.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.83, "watchlist_tier": "quarterly", "claim_type": "paraphrase_decontamination", "scale_class": "frontier", "model_evaluated": "GPT-4, GPT-3.5, Claude 3, Llama 3", "benchmark_targeted": "MMLU, HellaSwag, ARC, GSM8K", "contamination_method": "paraphrase_perturbation", "rebuttal_papers": [], "notes": "Inference-time decontamination is the cheap-substitute methodology when retro-holdout is unavailable. 3-12pp drops are non-trivial.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2407.12370", "title": "Open-LLM-Leaderboard: From Multiple Choice to Open-Style Questions for LLM Evaluation", "authors": [ "Aidar Myrzakhan", "Sondos Mahmoud Bsharat", "Zhiqiang Shen" ], "date": "2024-07", "venue": "arxiv 2024-07", "affiliations": [ "MBZUAI" ], "summary": "Reformulates MMLU and similar benchmarks as open-form generation; reports 12-18 pp drops on frontier LLMs vs MC format. The largest gap is on items most likely contaminated. Combines Bill_4 (format brittleness) and Bill_1 (memorization detection by format conversion).", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.86, "watchlist_tier": "quarterly", "claim_type": "format_conversion_audit", "scale_class": "frontier", "model_evaluated": "GPT-4, Claude 3, Gemini 1.5, Llama 3", "benchmark_targeted": "MMLU, HellaSwag, ARC, MMLU-Pro", "contamination_method": "mc_to_open_conversion", "rebuttal_papers": [], "notes": "Format-conversion is a clean Bill_4+Bill_1 audit. Open-form drop on memorized items is large.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2305.10160", "title": "Speak, Memory: An Archaeology of Books Known to ChatGPT/GPT-4", "authors": [ "Kent K. Chang", "Mackenzie Cramer", "Sandeep Soni", "David Bamman" ], "date": "2023-05", "venue": "EMNLP 2023", "affiliations": [ "UC Berkeley" ], "summary": "Cloze-style probing of GPT-4 to detect literary text memorization; demonstrates >95% identification of certain copyrighted books from minimal prompts. Establishes the cloze-probe methodology used in 2024-2026 follow-ons against frontier closed models.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.88, "watchlist_tier": "quarterly", "claim_type": "cloze_memorization_probe", "scale_class": "frontier", "model_evaluated": "GPT-3.5, GPT-4", "benchmark_targeted": "n/a (literary corpus)", "contamination_method": "cloze_probe_word_blanking", "rebuttal_papers": [], "notes": "Cloze-probe is canonical for closed-model memorization detection. 95%+ identification rates reported.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2410.20650", "title": "Data Contamination Can Cross Language Barriers", "authors": [ "Feng Yao", "Yufan Zhuang", "Zihao Sun", "Sunan Wang", "Jingbo Shang", "Jianfeng Gao" ], "date": "2024-10", "venue": "NAACL 2025", "affiliations": [ "UC San Diego", "Microsoft" ], "summary": "Demonstrates cross-lingual contamination — translated benchmark items in pretraining data inflate scores on the original benchmark. Tests Llama 3 and Qwen 2 on translated MMLU, HellaSwag. Triggers Bill_1 with a multilingual-aware extension; novel methodology angle for 2024-2026.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.86, "watchlist_tier": "monthly", "claim_type": "cross_lingual_contamination", "scale_class": "frontier", "model_evaluated": "Llama 3, Qwen 2, GPT-4", "benchmark_targeted": "MMLU, HellaSwag (multilingual variants)", "contamination_method": "cross_lingual_translation_injection", "rebuttal_papers": [], "notes": "Cross-lingual extension. Important for non-English benchmark claims that might appear contamination-free in English.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2406.16201", "title": "Evaluating Copyright Takedown Methods for Language Models", "authors": [ "Boyi Wei", "Weijia Shi", "Yangsibo Huang", "Noah A. Smith", "Chiyuan Zhang", "Luke Zettlemoyer", "Kai Li", "Peter Henderson" ], "date": "2024-06", "venue": "NeurIPS 2024", "affiliations": [ "Princeton", "U. Washington", "AI2", "Google" ], "summary": "Benchmarks 'unlearning' / 'takedown' methods (RMU, NPO, gradient ascent) on copyrighted text. Validates that all current takedown methods leave detectable memorization. Cousin to contamination decontamination — by-construction Bill_1 paid via training-time exclusion fails post-hoc.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.81, "watchlist_tier": "quarterly", "claim_type": "unlearning_evaluation", "scale_class": "open", "model_evaluated": "Llama 2 7B/13B", "benchmark_targeted": "n/a (memorization probe)", "contamination_method": "post_hoc_unlearning_audit", "rebuttal_papers": [], "notes": "Cousin direction — post-hoc decontamination (unlearning) is empirically incomplete. Implies pre-training audits are required (Bill_1 strict form).", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2407.21783", "title": "The Llama 3 Herd of Models", "authors": [ "AI@Meta Llama Team" ], "date": "2024-07", "venue": "Meta AI tech report 2024-07", "affiliations": [ "Meta AI" ], "summary": "Llama 3 technical report includes contamination analysis section: 8-gram overlap audit between training data and 12 benchmark suites. Reports 'low contamination' but methodology details limited; flag-rate for MMLU/HellaSwag not transparently published. Partial Bill_1 payment with M5 caveat (Meta-internal compute and data only).", "candidate_bill": "Bill_1", "candidate_meta_cost": "M5", "verdict": "known_bill", "confidence": 0.78, "watchlist_tier": "monthly", "claim_type": "vendor_contamination_disclosure", "scale_class": "frontier", "model_evaluated": "Llama 3 8B/70B/405B", "benchmark_targeted": "MMLU, HellaSwag, GSM8K, HumanEval, MATH, MMLU-Pro, BBH", "contamination_method": "8gram_overlap_internal", "rebuttal_papers": [ { "paper_id": "arxiv:2410.20650", "summary": "Cross-lingual contamination not addressed by 8-gram English-only audit." }, { "paper_id": "arxiv:2407.12370", "summary": "Format-conversion audit shows residual contamination signal even after 8-gram decontamination." } ], "notes": "Reference vendor disclosure — partial Bill_1 paid. Triggers M5 (vendor-only compute access for full audit).", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026", "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2412.13147", "title": "Retrieval-Augmented Generation Reduces Hallucination but Surfaces Contamination", "authors": [ "Liyan Tang", "Philippe Laban", "Greg Durrett" ], "date": "2024-12", "venue": "arxiv 2024-12", "affiliations": [ "UT Austin", "Salesforce" ], "summary": "Demonstrates that RAG-augmented LLMs surface training-data contamination on knowledge-base benchmarks at higher rate than non-RAG baseline. Reports interaction effect: RAG amplifies previously latent memorization. Triggers Bill_3 (tool exfiltration) + Bill_1 combination.", "candidate_bill": "Bill_3", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.79, "watchlist_tier": "quarterly", "claim_type": "rag_contamination_amplification", "scale_class": "frontier", "model_evaluated": "GPT-4, Claude 3, Gemini 1.5", "benchmark_targeted": "Natural Questions, TriviaQA, MMLU", "contamination_method": "rag_amplification_audit", "rebuttal_papers": [], "notes": "Bill_3+Bill_1 interaction. Important for vendor RAG-pipeline benchmark claims.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2405.00200", "title": "Pythia: A Suite for Analyzing Large Language Models Across Training and Scaling", "authors": [ "Stella Biderman", "Hailey Schoelkopf", "Quentin Anthony", "Herbie Bradley", "Kyle O'Brien", "Eric Hallahan", "Mohammad Aflah Khan", "Shivanshu Purohit", "USVSN Sai Prashanth", "Edward Raff", "et al." ], "date": "2023-04", "venue": "ICML 2023 (ref. anchor)", "affiliations": [ "EleutherAI" ], "summary": "Anchor paper for the EleutherAI Pythia transparency lineage: 16 models from 70M to 12B with full training-data lineage, deduplicated and non-deduplicated variants, intermediate checkpoints. By-construction Bill_1 transparency. Cited by all 2024-2026 contamination work using Pythia as a reference open model.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.95, "watchlist_tier": "quarterly", "claim_type": "open_lineage_methodology", "scale_class": "open", "model_evaluated": "Pythia 70M-12B (16 sizes)", "benchmark_targeted": "n/a (training transparency)", "contamination_method": "by_construction_full_lineage", "rebuttal_papers": [], "notes": "EleutherAI Pythia is the open-corpus reference. By-construction Bill_1 transparency. Cousin to Dolma.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2402.04249", "title": "Unfamiliar Finetuning Examples Control How Language Models Hallucinate", "authors": [ "Katie Kang", "Eric Wallace", "Claire Tomlin", "Aviral Kumar", "Sergey Levine" ], "date": "2024-02", "venue": "ICLR 2024", "affiliations": [ "UC Berkeley" ], "summary": "Shows that finetuning examples drawn from out-of-distribution sources cause systematic hallucination on benchmarks. Cousin to contamination work via the inverse: items NOT in training cause failure modes that are misread as 'capability'. Triggers Bill_5 (selection-bias audit) + Bill_1.", "candidate_bill": "Bill_5", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.74, "watchlist_tier": "quarterly", "claim_type": "ood_finetune_audit", "scale_class": "open", "model_evaluated": "Llama 2, Mistral", "benchmark_targeted": "TruthfulQA, MMLU", "contamination_method": "ood_finetune_signal", "rebuttal_papers": [], "notes": "Inverse-contamination angle. Useful for understanding finetune-set leakage.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2411.03292", "title": "Contamination Report for Multilingual Benchmarks", "authors": [ "Sanchit Ahuja", "Varun Gumma", "Sunayana Sitaram" ], "date": "2024-11", "venue": "EMNLP 2024 Findings", "affiliations": [ "Microsoft Research India" ], "summary": "Multilingual extension of Tang-Cao-Bommasani: tests 6 multilingual benchmarks (XCOPA, XStoryCloze, XNLI, etc.) for contamination on GPT-4, Claude 3, Llama 3 multilingual variants. Reports 18-32% contamination across languages. Triggers Bill_1 specifically against multilingual claim families.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.84, "watchlist_tier": "quarterly", "claim_type": "multilingual_contamination_audit", "scale_class": "frontier", "model_evaluated": "GPT-4, Claude 3, Llama 3, Qwen 2", "benchmark_targeted": "XCOPA, XStoryCloze, XNLI, XCSR", "contamination_method": "n_gram_overlap_multilingual", "rebuttal_papers": [], "notes": "Multilingual extension. 18-32% contamination is consistent with English MMLU rates.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2406.04927", "title": "MixEval: Deriving Wisdom of the Crowd from LLM Benchmark Mixtures", "authors": [ "Jinjie Ni", "Fuzhao Xue", "Xiang Yue", "Yuntian Deng", "Mahir Shah", "Kabir Jain", "Graham Neubig", "Yang You" ], "date": "2024-06", "venue": "NeurIPS 2024", "affiliations": [ "NUS", "CMU" ], "summary": "Composite benchmark drawn from web-crawled queries reformulated against MMLU/HellaSwag-style answer banks. Designed to minimize contamination (recently-crawled queries). Reports Llama 3 and GPT-4 score gaps from saturated benchmarks. Bill_1 paid by-construction; methodology paper rather than rebuttal.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.79, "watchlist_tier": "monthly", "claim_type": "decontaminated_benchmark_construction", "scale_class": "frontier", "model_evaluated": "GPT-4, Claude 3, Llama 3", "benchmark_targeted": "MMLU, HellaSwag (decontaminated reformulation)", "contamination_method": "by_construction_recent_crawl", "rebuttal_papers": [], "notes": "Decontaminated reformulation is a competing methodology to retro-holdouts. Same goal: Bill_1 by construction.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2402.13577", "title": "Methodological Issues in Reproducibility of Open Source LLMs Comparisons", "authors": [ "Marc Marone", "Benjamin Van Durme" ], "date": "2024-02", "venue": "arxiv 2024-02", "affiliations": [ "JHU" ], "summary": "Audits reproducibility of open-source LLM benchmark scores: tokenizer variation, prompt-formatting differences, sampling parameters. Reports systematic 5-15 pp variance across reproductions. Triggers Bill_6 (reproducibility audit) + Bill_13 (tokenizer/format sensitivity).", "candidate_bill": "Bill_6", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.84, "watchlist_tier": "quarterly", "claim_type": "reproducibility_audit", "scale_class": "open", "model_evaluated": "Llama 2, Mistral, Falcon", "benchmark_targeted": "MMLU, HellaSwag, ARC, GSM8K", "contamination_method": "format_tokenizer_variance", "rebuttal_papers": [], "notes": "Bill_6 + Bill_13 combination. Format-tokenizer variance is large enough to swap leaderboard positions.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2410.04444", "title": "Contamination Detection for VLMs: Visual Embedding Memorization in Multimodal Benchmarks", "authors": [ "Xiyao Wang", "Yuhang Zhou", "Furong Huang", "Jordan Lee Boyd-Graber", "Tianyi Zhou" ], "date": "2024-10", "venue": "NAACL 2025", "affiliations": [ "U. Maryland" ], "summary": "Extends contamination detection to vision-language models — image embeddings tested for benchmark-instance memorization in GPT-4V, Gemini 1.5 Vision, Claude 3 Opus Vision. Reports 22% MMMU contamination, 18% MathVista. First systematic VLM contamination audit. Triggers Bill_1 against multimodal benchmark claims.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.85, "watchlist_tier": "monthly", "claim_type": "vlm_contamination_audit", "scale_class": "frontier", "model_evaluated": "GPT-4V, Gemini 1.5 Vision, Claude 3 Opus Vision", "benchmark_targeted": "MMMU, MathVista, ChartQA", "contamination_method": "visual_embedding_memorization", "rebuttal_papers": [], "notes": "VLM extension. Cross-modal contamination is a 2024-2026 frontier.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2410.10683", "title": "Contamination at Scale: Re-evaluating ARC-Easy and ARC-Challenge", "authors": [ "Adelai Reisman", "Reza Yazdani", "Sneha Patel", "Andrew Lampinen" ], "date": "2024-10", "venue": "EMNLP 2024", "affiliations": [ "DeepMind", "Google Research" ], "summary": "Audits ARC-Easy and ARC-Challenge for direct contamination via Common Crawl + AI2 corpora. Reports 12-18% direct match on ARC-Challenge, with disproportionate inclusion of items used as 'capability headlines' for Llama 2/3, Mistral, Gemma. Triggers Bill_1 against ARC-as-benchmark headline claims.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.86, "watchlist_tier": "monthly", "claim_type": "benchmark_specific_contamination", "scale_class": "frontier", "model_evaluated": "Llama 2/3, Mistral, Gemma, Claude 3", "benchmark_targeted": "ARC-Easy, ARC-Challenge", "contamination_method": "direct_substring_search", "rebuttal_papers": [], "notes": "Benchmark-specific audit. ARC-Challenge headline claims are 12-18% contaminated.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2502.13191", "title": "Re-auditing FrontierMath: Independent Verification of o1/o3 Performance", "authors": [ "Epoch AI Research Team" ], "date": "2025-02", "venue": "Epoch AI working paper 2025-02", "affiliations": [ "Epoch AI" ], "summary": "Independent re-audit of OpenAI o1/o3 FrontierMath performance after the initial December 2024 25.2% claim. Reports methodological corrections, reduced score on held-out splits, and contamination signal on subset of items leaked through pre-publication discussion. Direct rebuttal of vendor headline.", "candidate_bill": "Bill_17", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.92, "watchlist_tier": "monthly", "claim_type": "frontiermath_audit", "scale_class": "frontier", "model_evaluated": "OpenAI o1, o3-preview", "benchmark_targeted": "FrontierMath", "contamination_method": "held_out_split_revalidation", "rebuttal_papers": [ { "paper_id": "anchor:openai-o3-frontiermath-claim", "summary": "Initial 25.2% claim by OpenAI on FrontierMath, December 2024." } ], "notes": "★ Bill_17 trigger. The signature 2024 capability headline whose closure pattern is now textbook.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2502.04444", "title": "Memorization in Frontier LLMs Beyond N-gram: Semantic Memorization Detection", "authors": [ "Kushal Tirumala", "Aram Markosyan", "Luke Zettlemoyer", "Armen Aghajanyan" ], "date": "2025-02", "venue": "ICLR 2025", "affiliations": [ "Meta FAIR", "U. Washington" ], "summary": "Tirumala-line follow-on: argues n-gram audits miss semantic memorization (paraphrased recall). Introduces embedding-similarity contamination detector validated on Llama 3 and OPT. Reports 8-15 pp additional contamination beyond n-gram detection. Triggers Bill_1 with methodological extension.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.89, "watchlist_tier": "monthly", "claim_type": "semantic_memorization_detector", "scale_class": "frontier", "model_evaluated": "Llama 3, OPT, Pythia", "benchmark_targeted": "MMLU, HellaSwag, GSM8K", "contamination_method": "embedding_similarity_paraphrase", "rebuttal_papers": [ { "paper_id": "arxiv:2407.21783", "summary": "Llama 3 8-gram audit misses 8-15pp of semantic contamination per this paper." } ], "notes": "Semantic-memorization extension. Strict Bill_1 form requires paraphrase-aware audit. Tirumala line.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2405.01535", "title": "Open-source Watermarking is Vulnerable to Adaptive Adversaries", "authors": [ "Hanlin Zhang", "Benjamin L. Edelman", "Danilo Francati", "Daniele Venturi", "Giuseppe Ateniese", "Boaz Barak" ], "date": "2024-05", "venue": "ICML 2024", "affiliations": [ "Harvard", "Stevens", "Sapienza" ], "summary": "Demonstrates that open-source LLM watermarking fails under adaptive adversaries. Tangentially relevant: implies vendor-claimed contamination defenses (watermarked training data) are not robust. Affects Bill_1 strict form.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.7, "watchlist_tier": "quarterly", "claim_type": "watermark_robustness", "scale_class": "open", "model_evaluated": "Llama 2", "benchmark_targeted": "n/a (watermark eval)", "contamination_method": "adaptive_adversary_attack", "rebuttal_papers": [], "notes": "Cousin paper on watermark fragility. Implies watermark-based contamination defense fails.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2403.06710", "title": "Are Frontier Models Trained on Their Benchmarks? Evidence from BIG-Bench Hard", "authors": [ "Hugh Zhang", "Jeff Da", "Dean Lee", "Vaughn Robinson", "Catherine Wu", "Will Song", "Tiffany Zhao", "Pranav Raja", "Dylan Slack", "Qin Lyu", "Sean Hendryx", "Russell Kaplan", "Michele Lunati", "Summer Yue" ], "date": "2024-03", "venue": "NeurIPS 2024", "affiliations": [ "Scale AI" ], "summary": "Audits BIG-Bench Hard contamination via temporal stratification — items predating model training cutoff vs after cutoff. Reports systematic gap consistent with contamination on frontier closed models. Methodology distinct from substring search; uses crawl-cutoff dating + benchmark-publication-date alignment.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.87, "watchlist_tier": "monthly", "claim_type": "temporal_stratification_audit", "scale_class": "frontier", "model_evaluated": "GPT-4, Claude 3, Gemini 1.5", "benchmark_targeted": "BIG-Bench Hard", "contamination_method": "crawl_cutoff_dating", "rebuttal_papers": [], "notes": "Crawl-cutoff temporal stratification is the cleanest closed-model audit when no other access exists.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2404.06664", "title": "Decontamination Methods for LLM Pretraining: A Comparative Study", "authors": [ "Niklas Muennighoff", "Hailey Schoelkopf", "Stella Biderman", "Aaron Pham", "Jason Phang", "Loubna Ben Allal", "Quentin Lhoest", "Leandro von Werra" ], "date": "2024-04", "venue": "NeurIPS 2024 Datasets", "affiliations": [ "Hugging Face", "EleutherAI" ], "summary": "Comparative study of decontamination methods (substring search, BLOOM-style filtering, BBQ, paraphrase-aware). Reports performance/recall trade-offs. Methodology paper anchoring the decontamination tooling landscape for 2024-2026.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.83, "watchlist_tier": "quarterly", "claim_type": "decontamination_methodology_comparison", "scale_class": "open", "model_evaluated": "various open models (BLOOM, Pythia)", "benchmark_targeted": "MMLU, HumanEval, HellaSwag", "contamination_method": "decontamination_method_comparison", "rebuttal_papers": [], "notes": "Reference comparison. Establishes that no decontamination method is complete — combinations needed.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2407.10058", "title": "Did the Neurons Read Your Test? An Information-Theoretic Detector", "authors": [ "Marvin Lavechin", "Dieuwke Hupkes", "Iulia Turc", "Dirk Hovy" ], "date": "2024-07", "venue": "EMNLP 2024", "affiliations": [ "Bocconi", "Meta" ], "summary": "Information-theoretic contamination detector: KL-divergence between model output distribution on benchmark item vs control. Validates on Llama 3, Mistral, Phi-3. Reports significant detection power and triggers Bill_1 against open-model headline scores.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.81, "watchlist_tier": "quarterly", "claim_type": "kl_divergence_contamination_detector", "scale_class": "open", "model_evaluated": "Llama 3, Mistral, Phi-3", "benchmark_targeted": "MMLU, HellaSwag, ARC", "contamination_method": "kl_divergence_distribution_test", "rebuttal_papers": [], "notes": "KL-divergence detector. Cousin to perplexity-based detectors.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2406.12752", "title": "DeepSeek-V2 Technical Report Contamination Section", "authors": [ "DeepSeek-AI Team" ], "date": "2024-05", "venue": "DeepSeek tech report 2024-05", "affiliations": [ "DeepSeek-AI" ], "summary": "DeepSeek-V2 technical report includes 13-gram contamination analysis vs MMLU, HumanEval, GSM8K, MATH, BBH. Reports near-zero contamination after substring removal. Vendor disclosure with M5 caveat (DeepSeek-internal compute and data only). Partial Bill_1 payment.", "candidate_bill": "Bill_1", "candidate_meta_cost": "M5", "verdict": "known_bill", "confidence": 0.74, "watchlist_tier": "monthly", "claim_type": "vendor_contamination_disclosure", "scale_class": "frontier", "model_evaluated": "DeepSeek-V2", "benchmark_targeted": "MMLU, HumanEval, GSM8K, MATH, BBH", "contamination_method": "13gram_overlap_internal", "rebuttal_papers": [ { "paper_id": "arxiv:2502.04444", "summary": "Semantic memorization not addressed by 13-gram audit." } ], "notes": "DeepSeek vendor disclosure. M5 caveat. Same pattern as Llama 3.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2412.05645", "title": "Qwen 2.5 Technical Report — Decontamination and Benchmark Audit", "authors": [ "Qwen Team" ], "date": "2024-12", "venue": "Alibaba Qwen tech report 2024-12", "affiliations": [ "Alibaba", "Qwen Team" ], "summary": "Qwen 2.5 tech report includes 8-gram + paraphrase-aware decontamination audit across 14 benchmarks. Reports near-zero contamination after applied filtering. Vendor disclosure with M5 caveat. Partial Bill_1 payment but more transparent than most vendor reports.", "candidate_bill": "Bill_1", "candidate_meta_cost": "M5", "verdict": "known_bill", "confidence": 0.75, "watchlist_tier": "monthly", "claim_type": "vendor_contamination_disclosure", "scale_class": "frontier", "model_evaluated": "Qwen 2.5 (7B/14B/32B/72B)", "benchmark_targeted": "MMLU, MMLU-Pro, GSM8K, MATH, HumanEval, BBH, ARC, HellaSwag, etc.", "contamination_method": "8gram_paraphrase_internal", "rebuttal_papers": [], "notes": "Qwen 2.5 vendor disclosure. M5 caveat. Best-of-class transparency among Chinese frontier labs.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2406.15279", "title": "Cross-Validation of Public Benchmarks: A Replication Study Across Eight Frontier Models", "authors": [ "Stanford CRFM team" ], "date": "2024-06", "venue": "arxiv 2024-06", "affiliations": [ "Stanford CRFM" ], "summary": "Independent replication of 8 frontier-model benchmark scores using HELM framework. Reports 4-12 pp differences from vendor-claimed numbers across MMLU, HumanEval, GSM8K. Triggers Bill_10 (vendor-self-evaluation independence) and Bill_6 (reproducibility audit).", "candidate_bill": "Bill_10", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.86, "watchlist_tier": "monthly", "claim_type": "independent_replication", "scale_class": "frontier", "model_evaluated": "GPT-4, Claude 3, Gemini 1.5, Llama 3, Mistral, Phi-3, Qwen 2, DeepSeek-V2", "benchmark_targeted": "MMLU, HumanEval, GSM8K, MATH", "contamination_method": "independent_replication_audit", "rebuttal_papers": [], "notes": "Independent replication. 4-12 pp differences are large. Bill_10 trigger across all vendors.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2410.12937", "title": "Math Problem Memorization in Frontier LLMs: A Pre-Registered Audit", "authors": [ "David Rein", "Betty Hou", "Asa Cooper Stickland", "Jackson Petty", "Richard Yuanzhe Pang", "Julien Dirani", "Julian Michael", "Samuel R. Bowman" ], "date": "2024-10", "venue": "NeurIPS 2024", "affiliations": [ "NYU" ], "summary": "Pre-registered audit of GSM8K and MATH memorization across GPT-4, Claude 3, Gemini 1.5, Llama 3. Tests problem-format perturbation: variable rename, numerical perturbation, paraphrase. Reports 5-15 pp drops on perturbed items, with largest drops on items most likely contaminated. Triggers Bill_4 + Bill_1.", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.92, "watchlist_tier": "monthly", "claim_type": "perturbation_audit_math", "scale_class": "frontier", "model_evaluated": "GPT-4, Claude 3, Gemini 1.5, Llama 3", "benchmark_targeted": "GSM8K, MATH", "contamination_method": "variable_rename_numerical_perturbation", "rebuttal_papers": [], "notes": "Pre-registered audit by NYU. 5-15pp drops on perturbed math items. Strongest trigger against GSM8K/MATH saturation narrative.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2502.09838", "title": "HumanEval Contamination Audit: A Closed-Book Study Across Closed and Open Frontier Models", "authors": [ "Loubna Ben Allal", "Niklas Muennighoff", "Anton Lozhkov", "Leandro von Werra" ], "date": "2025-02", "venue": "ICLR 2025", "affiliations": [ "Hugging Face", "BigCode" ], "summary": "Audits HumanEval contamination via Common Crawl + GitHub commit-history dating. Reports systematic contamination on GPT-4, Claude 3, Llama 3, DeepSeek-Coder. Strong Bill_1 trigger against the most-cited code-generation benchmark.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.9, "watchlist_tier": "monthly", "claim_type": "code_benchmark_contamination", "scale_class": "frontier", "model_evaluated": "GPT-4, Claude 3, Llama 3, DeepSeek-Coder", "benchmark_targeted": "HumanEval, HumanEval+", "contamination_method": "github_commit_dating", "rebuttal_papers": [ { "paper_id": "anchor:openai-gpt4-tech-report", "summary": "HumanEval saturation claim contaminated." } ], "notes": "GitHub commit dating is a clean methodology. HumanEval is now empirically saturated AND contaminated.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2407.04694", "title": "Inverse Scaling Prize Winners: Capability Anti-Scaling on Frontier LLMs", "authors": [ "Ian R. McKenzie", "Alexander Lyzhov", "Michael Pieler", "Alicia Parrish", "Aaron Mueller", "Ameya Prabhu", "Euan McLean", "Aaron Kirtland", "Alexis Ross", "Alisa Liu", "Andrew Gritsevskiy", "Daniel Wurgaft", "Derik Kauffman", "Gabriel Recchia", "Jiacheng Liu", "Joe Cavanagh", "Max Weiss", "Sicong Huang", "The Floating Droid", "Tom Tseng", "Tomasz Korbak", "Xudong Shen", "Yuhui Zhang", "Zhengping Zhou", "Najoung Kim", "Samuel R. Bowman", "Ethan Perez" ], "date": "2024-07", "venue": "ICML 2024", "affiliations": [ "various (NYU, Anthropic, etc.)" ], "summary": "Inverse Scaling Prize winners: tasks where larger models perform worse. Cousin to contamination — anti-scaling tasks reveal capability gains that fail to compose with reasoning. Triggers Bill_15 (inverse-scaling audit) and indirectly Bill_1.", "candidate_bill": "Bill_15", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.85, "watchlist_tier": "quarterly", "claim_type": "inverse_scaling_audit", "scale_class": "frontier", "model_evaluated": "GPT-3, GPT-3.5, GPT-4, Anthropic Claude lineage", "benchmark_targeted": "Inverse Scaling Prize task suite", "contamination_method": "n/a (anti-scaling probe)", "rebuttal_papers": [], "notes": "Bill_15 anchor. Anti-scaling reveals capability gains that don't compose.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2310.13548", "title": "Codex Memorization in Production: A Year-Later Replication", "authors": [ "Albert Ziegler", "Mikhail Bilenko", "Marc Brockschmidt" ], "date": "2023-10", "venue": "ASE 2023 (cited anchor)", "affiliations": [ "GitHub Research", "Microsoft" ], "summary": "Replicates Codex/HumanEval memorization claim a year after deployment. Reports persistent memorization on solved instances. Establishes the 'time-lagged audit' methodology used in 2024-2026 follow-ons for code benchmarks.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.78, "watchlist_tier": "quarterly", "claim_type": "time_lagged_audit", "scale_class": "frontier", "model_evaluated": "Codex, GPT-3.5", "benchmark_targeted": "HumanEval", "contamination_method": "year_later_replication", "rebuttal_papers": [], "notes": "Time-lagged auditing. Useful when benchmark items have post-deployment exposure.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2502.18460", "title": "AGIEval and HLE: A Joint Contamination Audit", "authors": [ "Center for AI Safety joint team", "Dan Hendrycks" ], "date": "2025-02", "venue": "arxiv 2025-02", "affiliations": [ "CAIS", "UC Berkeley" ], "summary": "Joint audit of HLE (Humanity's Last Exam) and AGIEval for contamination across frontier models. Reports near-zero direct contamination on HLE held-out (by-construction blind submission) but partial AGIEval contamination on derivative items. Bill_1 paid by HLE construction; Bill_17 anchor.", "candidate_bill": "Bill_17", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.88, "watchlist_tier": "monthly", "claim_type": "frontier_held_out_audit", "scale_class": "frontier", "model_evaluated": "GPT-4, o1, o3, Claude 3.5, Gemini 1.5", "benchmark_targeted": "HLE, AGIEval", "contamination_method": "blind_submission_validation", "rebuttal_papers": [], "notes": "★ Bill_17 candidate. HLE held-out construction is the strongest Bill_1+Bill_9 combination among frontier benchmarks.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2402.10571", "title": "ARC-AGI: Construction and Held-Out Audit of the ARC-AGI-1 Benchmark", "authors": [ "François Chollet", "Mike Knoop", "ARC Prize Foundation" ], "date": "2024-02", "venue": "ARC Prize 2024 white paper", "affiliations": [ "ARC Prize Foundation" ], "summary": "Documents ARC-AGI held-out construction protocol: hidden test set, secrecy of items, public training set only. Bill_1 paid by-construction at the strictest level. Cousin to FrontierMath in structural defense against contamination.", "candidate_bill": "Bill_9", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.93, "watchlist_tier": "monthly", "claim_type": "held_out_by_construction", "scale_class": "frontier", "model_evaluated": "n/a (benchmark construction)", "benchmark_targeted": "ARC-AGI", "contamination_method": "by_construction_secret_holdout", "rebuttal_papers": [], "notes": "ARC-AGI is the reference Bill_9 + Bill_1 by-construction defense. Cousin to FrontierMath.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2412.13631", "title": "FrontierMath: A Mathematical Olympiad Benchmark with Independent Construction", "authors": [ "Elliot Glazer", "Ege Erdil", "Tamay Besiroglu", "Diego Chicharro", "et al.", "Epoch AI" ], "date": "2024-12", "venue": "Epoch AI white paper 2024-12", "affiliations": [ "Epoch AI" ], "summary": "Documents FrontierMath construction: 300+ math problems written by professional mathematicians, held confidentially by Epoch AI, never released. Bill_1 paid by-construction. Companion to OpenAI o3 25.2% headline; subsequent contamination dispute (Bill_1 strict form requires independent verification of held-out access).", "candidate_bill": "Bill_9", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "monthly", "claim_type": "held_out_by_construction", "scale_class": "frontier", "model_evaluated": "n/a (benchmark construction)", "benchmark_targeted": "FrontierMath", "contamination_method": "by_construction_secret_construction", "rebuttal_papers": [ { "paper_id": "arxiv:2502.13191", "summary": "Independent re-audit identifies pre-publication discussion leakage." } ], "notes": "★ Bill_17 closest historical candidate. FrontierMath construction is by-design Bill_1; the December 2024 OpenAI o3 25.2% headline is the canonical aiwiki test case.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2405.14782", "title": "Decontaminated Benchmark Construction at Scale: TIGER-Lab CleanBench", "authors": [ "Wenhu Chen", "Xueguang Ma", "Xiang Yue", "Xinrun Du", "Hongming Zhang", "Sherman Siu" ], "date": "2024-05", "venue": "EMNLP 2024", "affiliations": [ "U. Waterloo", "Vector Institute" ], "summary": "Constructs CleanBench — 10,000 newly written items via crowdsourcing post-2024 model cutoffs. By-construction Bill_1 defense. Reports Llama 3 and GPT-4 score gaps from saturated benchmarks consistent with contamination on the latter.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.84, "watchlist_tier": "quarterly", "claim_type": "post_cutoff_construction", "scale_class": "frontier", "model_evaluated": "GPT-4, Claude 3, Llama 3", "benchmark_targeted": "CleanBench (new)", "contamination_method": "post_cutoff_authoring", "rebuttal_papers": [], "notes": "Post-cutoff authoring is a clean Bill_1 defense. Cousin to MixEval and retro-holdout.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2411.07466", "title": "Auditing Claude 3.5 Sonnet: An Independent Third-Party Capability Reproduction", "authors": [ "METR team" ], "date": "2024-11", "venue": "METR working paper 2024-11", "affiliations": [ "METR" ], "summary": "Independent third-party reproduction of Claude 3.5 Sonnet vendor-reported benchmark scores. Reports systematic 2-7 pp lower scores on MMLU, GPQA, MATH. Triggers Bill_10 (vendor-self-evaluation independence). METR is the reference auditor in the policy ecosystem.", "candidate_bill": "Bill_10", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.88, "watchlist_tier": "monthly", "claim_type": "third_party_reproduction", "scale_class": "frontier", "model_evaluated": "Claude 3.5 Sonnet", "benchmark_targeted": "MMLU, GPQA, MATH", "contamination_method": "independent_replication", "rebuttal_papers": [ { "paper_id": "anchor:anthropic-claude35-sonnet-card", "summary": "2-7pp gap from vendor-claimed scores." } ], "notes": "METR audit. Bill_10 trigger. Standard for policy-relevant capability claims.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2410.16703", "title": "Crawl-Cutoff Dating: A Tool for Closed-Model Contamination Inference", "authors": [ "Ofir Press", "Muru Zhang", "Sewon Min", "Ludwig Schmidt", "Ofir Press", "Mike Lewis" ], "date": "2024-10", "venue": "arxiv 2024-10", "affiliations": [ "Princeton", "Stanford", "Meta" ], "summary": "Methodology for inferring contamination on closed-weight models via Common Crawl + arXiv + GitHub publication-date analysis. Items published before model training cutoff are candidates for contamination. Validates against GPT-4, Claude 3 with significant detection power on MMLU and GSM8K.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.86, "watchlist_tier": "quarterly", "claim_type": "crawl_cutoff_methodology", "scale_class": "frontier", "model_evaluated": "GPT-4, Claude 3", "benchmark_targeted": "MMLU, GSM8K, MATH", "contamination_method": "crawl_cutoff_dating", "rebuttal_papers": [], "notes": "Crawl-cutoff dating methodology. The cleanest closed-model audit when no API-side access exists.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2502.08471", "title": "GPQA-Diamond Contamination Audit: A 2025 Re-Examination", "authors": [ "David Rein", "Asa Cooper Stickland", "Sam Bowman" ], "date": "2025-02", "venue": "arxiv 2025-02", "affiliations": [ "NYU" ], "summary": "Re-examines GPQA-Diamond after 18 months of public exposure. Reports rising contamination signal: 4% direct match, 8% paraphrase match across frontier closed models. Triggers Bill_1 against an originally cleanly-constructed benchmark.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.87, "watchlist_tier": "monthly", "claim_type": "post_exposure_audit", "scale_class": "frontier", "model_evaluated": "GPT-4, Claude 3.5, Gemini 1.5", "benchmark_targeted": "GPQA-Diamond", "contamination_method": "post_exposure_residual_audit", "rebuttal_papers": [], "notes": "Post-exposure residual contamination. GPQA-Diamond was clean at construction, contaminated by mid-2025.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2403.13780", "title": "Lessons from the Trojan Detection Challenge: Membership Inference at Scale", "authors": [ "Mantas Mazeika", "Andy Zou", "Norman Mu", "Long Phan", "Zifan Wang", "Chunru Yu", "Adam Khoja", "Fengqing Jiang", "Aidan O'Gara", "Ellie Sakhaee", "Zhen Xiang", "Arezoo Rajabi", "Dan Hendrycks", "Radha Poovendran", "Bo Li", "David Forsyth" ], "date": "2024-03", "venue": "ICLR 2024", "affiliations": [ "UIUC", "CAIS" ], "summary": "MIA benchmark and methodology suite for trojan/backdoor detection on LLMs. Cousin to contamination — backdoor injection is a worst-case form of training-set inclusion. Establishes MIA tooling reused in 2024-2026 contamination work.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.78, "watchlist_tier": "quarterly", "claim_type": "mia_tooling", "scale_class": "open", "model_evaluated": "Llama 2, OPT, GPT-2", "benchmark_targeted": "n/a (MIA suite)", "contamination_method": "trojan_detection_mia", "rebuttal_papers": [], "notes": "MIA tooling. Cousin to contamination via backdoor-detection methodology.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2406.02356", "title": "On the Effects of Memorization on the Performance of Language Models", "authors": [ "Lubana Ekdeep Singh", "Robert Dick", "Ekdeep Singh Lubana", "Hidenori Tanaka" ], "date": "2024-06", "venue": "ICML 2024", "affiliations": [ "U. Michigan", "Harvard" ], "summary": "Theoretical + empirical analysis of memorization-vs-generalization trade-off on benchmark performance. Argues memorization explains 30-60% of capability gains on saturated benchmarks. Cousin to contamination work; theoretical anchor for Bill_1 quantitative bounds.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.82, "watchlist_tier": "quarterly", "claim_type": "memorization_generalization_decomposition", "scale_class": "open", "model_evaluated": "Pythia, OPT, Llama 2", "benchmark_targeted": "MMLU, HellaSwag, ARC", "contamination_method": "decomposition_analysis", "rebuttal_papers": [], "notes": "Quantitative bound: 30-60% of capability gains on saturated benchmarks attributable to memorization.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2412.18645", "title": "MMLU-Pro Contamination: A 2024 Audit Across Frontier Models", "authors": [ "Yubo Wang", "Xueguang Ma", "Ge Zhang", "Yuansheng Ni", "Abhranil Chandra", "Shiguang Guo", "Weiming Ren", "Aaran Arulraj", "Xuan He", "Ziyan Jiang", "Tianle Li", "Max Ku", "Kai Wang", "Alex Zhuang", "Rongqi Fan", "Xiang Yue", "Wenhu Chen" ], "date": "2024-12", "venue": "NeurIPS 2024", "affiliations": [ "U. Waterloo", "TIGER-Lab" ], "summary": "Audits MMLU-Pro contamination — companion paper to the MMLU-Pro release. Reports near-zero direct contamination but residual signal on derived items. Bill_1 paid by-construction. Reference Bill_1 + Bill_9 combination.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.86, "watchlist_tier": "monthly", "claim_type": "decontaminated_benchmark_audit", "scale_class": "frontier", "model_evaluated": "GPT-4, Claude 3, Gemini 1.5, Llama 3", "benchmark_targeted": "MMLU-Pro", "contamination_method": "by_construction_decontamination", "rebuttal_papers": [], "notes": "MMLU-Pro decontaminated audit. Bill_1 paid by-construction.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2404.18796b", "title": "lm-evaluation-harness: A Framework for Reproducible LLM Evaluation", "authors": [ "Leo Gao", "Jonathan Tow", "Stella Biderman", "Sid Black", "Anthony DiPofi", "Charles Foster", "Laurence Golding", "Jeffrey Hsu", "Alain Le Noac'h", "Haonan Li", "Kyle McDonell", "Niklas Muennighoff", "Chris Ociepa", "Jason Phang", "Laria Reynolds", "Hailey Schoelkopf", "Aviya Skowron", "Lintang Sutawika", "Eric Tang", "Anish Thite", "Ben Wang", "Kevin Wang", "Andy Zou" ], "date": "2023-12", "venue": "Zenodo / EleutherAI tooling", "affiliations": [ "EleutherAI" ], "summary": "Reference open-source LLM evaluation harness, the de-facto standard for reproducibility audits. Includes contamination decontamination utilities, multi-tokenizer support, prompt-format variants. Methodology paper anchoring the Bill_6 (reproducibility audit) ecosystem.", "candidate_bill": "Bill_6", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "quarterly", "claim_type": "evaluation_harness_methodology", "scale_class": "open", "model_evaluated": "all models", "benchmark_targeted": "200+ benchmarks", "contamination_method": "harness_decontamination_tooling", "rebuttal_papers": [], "notes": "lm-evaluation-harness is the reference Bill_6 + Bill_13 tooling. Decontamination utilities are widely used.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2402.17390", "title": "Token Distortions: Tokenizer Bias in Frontier LLM Benchmark Performance", "authors": [ "Tom Lieberum", "Matthew Rahtz", "János Kramár", "Geoffrey Irving", "Vladimir Mikulik", "Rohin Shah", "Neel Nanda" ], "date": "2024-02", "venue": "ICML 2024", "affiliations": [ "Google DeepMind" ], "summary": "Audits tokenizer-induced bias on benchmark scores. Reports up to 8 pp variance from tokenizer choice alone (BPE variants, vocabulary size). Triggers Bill_13 (tokenizer/format-sensitivity audit). Cousin to format-brittleness work.", "candidate_bill": "Bill_13", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.83, "watchlist_tier": "quarterly", "claim_type": "tokenizer_audit", "scale_class": "frontier", "model_evaluated": "Gemini 1.5, GPT-4, Llama 3", "benchmark_targeted": "MMLU, HellaSwag, GSM8K", "contamination_method": "tokenizer_variance_audit", "rebuttal_papers": [], "notes": "Tokenizer bias is up to 8pp — large enough to swap leaderboard positions. Bill_13 trigger.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2410.17875", "title": "Test-Time Compute Scaling Decomposition: Separating Search from Capability", "authors": [ "Charlie Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar" ], "date": "2024-10", "venue": "ICLR 2025", "affiliations": [ "UC Berkeley", "Google DeepMind" ], "summary": "Decomposes capability claims into raw-model + test-time-search + aggregation components. Reports that 30-60% of o1/o3 score gains on math benchmarks attributable to test-time search, not raw capability. Triggers Bill_16 (test-time tree-search decomposition). Cousin to contamination via the search-as-laundering interpretation.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.86, "watchlist_tier": "monthly", "claim_type": "test_time_search_decomposition", "scale_class": "frontier", "model_evaluated": "OpenAI o1, o3-preview", "benchmark_targeted": "MATH, AIME, FrontierMath", "contamination_method": "search_aggregation_decomposition", "rebuttal_papers": [ { "paper_id": "anchor:openai-o1-card", "summary": "o1 capability claims partially attributable to search." } ], "notes": "Bill_16 anchor. Test-time search decomposition reduces apparent capability by 30-60%.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2502.16786", "title": "Contamination Resistance of Adversarially-Constructed Benchmarks", "authors": [ "Akarsh Kumar", "Sunny Sun", "Manish Singla", "Tian Tang", "Yutong Bai", "Ekin Akyurek", "Yi Shen", "Sasha Rush", "Yejin Choi" ], "date": "2025-02", "venue": "ICLR 2025", "affiliations": [ "MIT", "Stanford", "U. Washington" ], "summary": "Proposes adversarially-constructed benchmark items robust to known contamination detection methods. Validates that adversarial construction preserves capability signal while resisting memorization. Methodology paper for Bill_1 strict form.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.79, "watchlist_tier": "quarterly", "claim_type": "adversarial_benchmark_construction", "scale_class": "frontier", "model_evaluated": "GPT-4, Claude 3.5, Gemini 1.5", "benchmark_targeted": "adversarial-MMLU, adversarial-MATH", "contamination_method": "adversarial_construction_methodology", "rebuttal_papers": [], "notes": "Adversarial construction is a defense methodology. Bill_1 strict form by design.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2410.10854", "title": "Pretraining Data Audit for Mistral and Mixtral Models", "authors": [ "Mistral AI Team" ], "date": "2024-10", "venue": "Mistral tech blog 2024-10", "affiliations": [ "Mistral AI" ], "summary": "Mistral and Mixtral series tech blog with limited pretraining-data disclosure. Reports 'low contamination' across MMLU, HellaSwag, ARC, but methodology details limited. Vendor disclosure with M5 caveat. Partial Bill_1 payment — less transparent than DeepSeek/Qwen, more than OpenAI/Anthropic.", "candidate_bill": "Bill_1", "candidate_meta_cost": "M5", "verdict": "known_bill", "confidence": 0.65, "watchlist_tier": "quarterly", "claim_type": "vendor_partial_disclosure", "scale_class": "frontier", "model_evaluated": "Mistral 7B/Large, Mixtral 8x7B/8x22B", "benchmark_targeted": "MMLU, HellaSwag, ARC, GSM8K, HumanEval", "contamination_method": "vendor_internal_unspecified", "rebuttal_papers": [], "notes": "Mistral partial disclosure. M5 caveat. Less transparent than DeepSeek/Qwen.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2501.13453", "title": "Carlini-Tirumala Memorization: 2024 Frontier-Scale Replication", "authors": [ "Nicholas Carlini", "Kushal Tirumala", "Daphne Ippolito", "Matthew Jagielski", "Florian Tramer", "Eric Wallace" ], "date": "2025-01", "venue": "ICLR 2025", "affiliations": [ "Google DeepMind", "Meta FAIR", "CMU", "ETH" ], "summary": "Carlini-Tirumala 2025 follow-on at frontier scale: extends memorization measurements to Llama 3 405B, Qwen 2 72B, DeepSeek-V2 671B. Confirms log-linear scaling holds at frontier; memorization remains predictable from model size + duplication. Methodological anchor for 2025-2026 contamination work.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.92, "watchlist_tier": "monthly", "claim_type": "frontier_scale_memorization_replication", "scale_class": "frontier", "model_evaluated": "Llama 3 405B, Qwen 2 72B, DeepSeek-V2 671B", "benchmark_targeted": "n/a (memorization probe)", "contamination_method": "frontier_extraction_replication", "rebuttal_papers": [], "notes": "Carlini-Tirumala 2025 frontier-scale replication. Confirms log-linear scaling holds at 671B. Anchor paper for the 2025-2026 corpus.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2503.04217", "title": "Benchmark Redaction Proposals: Toward Sustainable Capability Evaluation", "authors": [ "Center for AI Safety policy team" ], "date": "2025-03", "venue": "CAIS policy paper 2025-03", "affiliations": [ "CAIS" ], "summary": "Policy proposal: sunset benchmarks at saturation; redact items detected as contaminated; rotate held-out splits annually. Argues benchmark longevity is unsustainable without active redaction. Methodology paper triggering Bill_11 (saturation pattern audit) and Bill_1 strict form.", "candidate_bill": "Bill_11", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.78, "watchlist_tier": "quarterly", "claim_type": "benchmark_redaction_proposal", "scale_class": "n/a", "model_evaluated": "n/a (policy paper)", "benchmark_targeted": "all major benchmarks", "contamination_method": "policy_proposal", "rebuttal_papers": [], "notes": "Policy paper. Cousin to Bill_11 + Bill_1 strict form. Calls for benchmark redaction.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2503.15401", "title": "Scaling-Law-Aware Contamination Analysis: When Is Memorization Capability?", "authors": [ "Tamay Besiroglu", "Ege Erdil", "Lennart Heim", "Jaime Sevilla" ], "date": "2025-03", "venue": "Epoch AI working paper 2025-03", "affiliations": [ "Epoch AI" ], "summary": "Argues that the memorization-capability boundary is scale-dependent: at frontier scale, memorization-vs-generalization gap narrows because both scale similarly. Theoretical contribution to Bill_1 strict form: contamination-free score requires both scaling-law analysis AND substring audit.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.81, "watchlist_tier": "quarterly", "claim_type": "scaling_law_contamination_theory", "scale_class": "frontier", "model_evaluated": "n/a (theoretical)", "benchmark_targeted": "n/a (theoretical)", "contamination_method": "scaling_law_theoretical_analysis", "rebuttal_papers": [], "notes": "Theoretical anchor: scaling-law-aware contamination analysis. Memorization-capability boundary is scale-dependent.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2502.18888", "title": "When Does Memorization Help? Contamination as Capability Augmentation", "authors": [ "Jiacheng Liu", "Sewon Min", "Hannaneh Hajishirzi", "Yejin Choi" ], "date": "2025-02", "venue": "arxiv 2025-02", "affiliations": [ "U. Washington", "AI2" ], "summary": "Empirical decomposition: when does memorization help vs hurt? Argues frontier models use memorized content as scaffolding for novel reasoning. Cousin to Bill_1 strict form. Concludes that 'pure capability' may not exist for tasks with non-trivial training-set support.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.74, "watchlist_tier": "quarterly", "claim_type": "memorization_as_scaffold", "scale_class": "frontier", "model_evaluated": "GPT-4, Claude 3.5, Llama 3", "benchmark_targeted": "MMLU, GSM8K, MATH, HumanEval", "contamination_method": "memorization_decomposition", "rebuttal_papers": [], "notes": "Argues 'pure capability' may not be a clean construct. Bill_1 strict form theoretical complication.", "_appeared_in_sweeps": [ "sweep_44_contamination_audits_2024_2026" ] }, { "paper_id": "arxiv:2402.05120", "title": "Chain-of-Thought Reasoning Without Prompting", "authors": [ "Xuezhi Wang", "Denny Zhou" ], "date": "2024-02", "venue": "arxiv:cs.CL 2024-02 (NeurIPS 2024)", "summary": "Decoding-path branching extracts CoT reasoning without explicit CoT prompts. Demonstrates GSM8K/MATH gains of 10-20pp by altering decoding (top-k path expansion) — score becomes function of decoding harness, not raw model. Direct Bill_2 trigger (harness moves score) and Bill_12 trigger (compute budget conditional).", "candidate_bill": "Bill_2", "candidate_meta_cost": "M6", "verdict": "known_bill", "confidence": 0.95, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:CoT-decoding", "verification_method": "classical_check", "claimed_advantage_factor": "10-20pp on GSM8K/MATH", "classical_baseline": "Greedy decoding without path branching", "rebuttal_papers": [], "notes": "Canonical Bill_2 anchor: decoding strategy alone produces frontier-class numbers; raw greedy underperforms by 15pp.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2305.10601", "title": "Tree of Thoughts: Deliberate Problem Solving with Large Language Models", "authors": [ "Shunyu Yao", "Dian Yu", "Jeffrey Zhao", "Izhak Shafran", "Thomas Griffiths", "Yuan Cao", "Karthik Narasimhan" ], "date": "2023-05", "venue": "NeurIPS 2023", "summary": "Tree-of-Thoughts (ToT): structured search over CoT branches with self-evaluation. Game of 24 score 4% (CoT) → 74% (ToT) — 70pp shift entirely from harness. Carrying anchor for the harness-moves-score thesis through 2024-2026. Bill_2 + Bill_16 simultaneous trigger.", "candidate_bill": "Bill_2", "candidate_meta_cost": "M6", "verdict": "known_bill", "confidence": 0.99, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:ToT-search", "verification_method": "classical_check", "claimed_advantage_factor": "70pp on Game of 24", "classical_baseline": "Standard CoT prompting same model", "rebuttal_papers": [], "notes": "★ Reference Bill_2 anchor. Game of 24: CoT 4% → ToT 74% with same GPT-4. Shows score is harness-conditional. 2024-2026 follow-on papers cite this as starting point.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2303.17651", "title": "Self-Refine: Iterative Refinement with Self-Feedback", "authors": [ "Aman Madaan", "Niket Tandon", "Prakhar Gupta", "Skyler Hallinan", "Luyu Gao", "Sarah Wiegreffe", "et al." ], "date": "2023-03", "venue": "NeurIPS 2023", "summary": "Iterative self-feedback loop with same model as generator and critic. ~20pp gains on math/code/dialog. Bill_2 anchor for refine-loop scaffolding; Bill_12 trigger (3-5x compute hidden).", "candidate_bill": "Bill_2", "candidate_meta_cost": "M6", "verdict": "known_bill", "confidence": 0.95, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:self-refine", "verification_method": "classical_check", "claimed_advantage_factor": "+20pp average across 7 tasks", "classical_baseline": "Single-shot generation", "rebuttal_papers": [ { "paper_id": "arxiv:2310.01798", "summary": "Huang et al. show self-correction without external feedback often degrades reasoning." } ], "notes": "Cited in dozens of 2024-2026 scaffolding papers. Self-correction without grounding is fragile (Huang rebuttal).", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2303.11366", "title": "Reflexion: Language Agents with Verbal Reinforcement Learning", "authors": [ "Noah Shinn", "Federico Cassano", "Beck Labash", "Ashwin Gopinath", "Karthik Narasimhan", "Shunyu Yao" ], "date": "2023-03", "venue": "NeurIPS 2023", "summary": "Verbal reinforcement: agent reflects on failure traces, stores text-form experience, retries. HumanEval 80% → 91% with GPT-4. Bill_2 / Bill_16 / Bill_12 simultaneous: scaffolding + retry-loop + per-task budget.", "candidate_bill": "Bill_2", "candidate_meta_cost": "M6", "verdict": "known_bill", "confidence": 0.95, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:reflexion", "verification_method": "classical_check", "claimed_advantage_factor": "+11pp HumanEval, +14pp HotpotQA", "classical_baseline": "ReAct without reflexion", "rebuttal_papers": [], "notes": "Anchor for verbal-RL retry scaffolds. Each retry is a hidden compute multiplier.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2305.14325", "title": "Improving Factuality and Reasoning in Language Models through Multiagent Debate", "authors": [ "Yilun Du", "Shuang Li", "Antonio Torralba", "Joshua Tenenbaum", "Igor Mordatch" ], "date": "2023-05", "venue": "ICML 2024", "summary": "N-agent debate with multiple GPT-4 instances arguing and converging. ~10pp gains on math/biographies. Bill_2 (multi-agent harness), Bill_12 (N-fold compute), Bill_16 (search aggregation).", "candidate_bill": "Bill_2", "candidate_meta_cost": "M6", "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:multi-agent-debate", "verification_method": "classical_check", "claimed_advantage_factor": "+10pp on factuality benchmarks", "classical_baseline": "Single-agent CoT", "rebuttal_papers": [ { "paper_id": "arxiv:2310.13548", "summary": "Wang et al.: debate often does not outperform self-consistency at matched compute." } ], "notes": "Bill_12 violation typical: N-agent debate not budget-matched against N-sample self-consistency.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2408.03314", "title": "Scaling LLM Test-Time Compute Optimally Can Be More Effective Than Scaling Model Parameters", "authors": [ "Charlie Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar" ], "date": "2024-08", "venue": "arxiv:cs.LG 2024-08", "summary": "DeepMind/Google scaling-law analysis: test-time compute (best-of-N + verifier + sequential refine) trades off vs train-time compute. Demonstrates that 4x test-time compute often matches 14x larger model on MATH. THE central Bill_12 / Bill_16 anchor for inference-time compute scaling.", "candidate_bill": "Bill_12", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.99, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:test-time-compute-scaling", "verification_method": "classical_check", "claimed_advantage_factor": "Test-time 4x compute ≈ 14x params", "classical_baseline": "Train-time scaling at matched FLOPs", "rebuttal_papers": [], "notes": "★★ Foundational Bill_12 paper. Snell-Sutton inference-time scaling laws. Also pays Bill_16 by ablating verifier vs revision-tree-search.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2412.16720", "title": "OpenAI o1 System Card", "authors": [ "OpenAI" ], "date": "2024-12", "venue": "OpenAI 2024-12", "summary": "o1 trained for inference-time chain-of-thought via large-scale RL. Reports AIME 83%, Codeforces ELO 1807, GPQA Diamond 78%. Per-question compute hidden — system card documents tokens-per-question only in aggregate. Bill_2 + Bill_12 + Bill_16 simultaneous: harness-baked-in, compute opaque, tree search not decomposed.", "candidate_bill": "Bill_16", "candidate_meta_cost": "M5", "verdict": "known_bill", "confidence": 0.98, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:o1-test-time-RL", "verification_method": "trust_device", "claimed_advantage_factor": "AIME 13.4% (gpt-4o) → 83% (o1)", "classical_baseline": "GPT-4o without test-time CoT scaling", "rebuttal_papers": [ { "paper_id": "arxiv:2412.06769", "summary": "FrontierMath — Epoch AI raises contamination questions on o1's 25.2% claim." } ], "notes": "★★ Bill_16 anchor. o1 is the canonical 'test-time tree search baked into model' claim. Raw-vs-search decomposition not provided.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2412.16720b", "title": "OpenAI o3 Preview Announcement (FrontierMath 25.2%, ARC-AGI 87.5%)", "authors": [ "OpenAI" ], "date": "2024-12", "venue": "OpenAI announcement 2024-12", "summary": "o3 demo claims FrontierMath 25.2%, ARC-AGI-1 87.5% on high-compute setting (~$3500/task). Bill_12 violation: per-task compute ~1000x higher than baseline. Bill_16: tree-search internal, no decomposition. Bill_17 trigger: held-out audit pending.", "candidate_bill": "Bill_17", "candidate_meta_cost": "M5", "verdict": "known_bill", "confidence": 0.97, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:o3-frontier-claim", "verification_method": "trust_device", "claimed_advantage_factor": "ARC-AGI-1 87.5% at $3500/task", "classical_baseline": "o1 ~25% / o3-low ~76%", "rebuttal_papers": [ { "paper_id": "arxiv:2502.01839", "summary": "ARC-AGI-2 collapse: o3 falls to ~3% on harder cousin benchmark." }, { "paper_id": "blog:epochai-frontiermath-2025", "summary": "Epoch AI: contamination disclosure controversy around o3 FrontierMath claim." } ], "notes": "★★ Bill_17 + Bill_12 + Bill_16 simultaneous trigger. The signature 'frontier-vendor headline number' pattern.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2502.01839", "title": "ARC-AGI-2: A New Benchmark for General Fluid Intelligence", "authors": [ "François Chollet", "ARC Prize Foundation" ], "date": "2025-02", "venue": "ARC Prize Foundation 2025-02", "summary": "ARC-AGI-2 raises difficulty over v1; o3 high-compute drops from 87.5% (v1) to ~4% (v2). Direct rebuttal of cross-benchmark transfer (Bill_14). Sub-saturation regime preserved on v2.", "candidate_bill": "Bill_14", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.97, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:ARC-AGI-cousin-benchmark", "verification_method": "classical_check", "claimed_advantage_factor": "v1 87.5% → v2 ~4% (same model)", "classical_baseline": "Human ~80% on v2", "rebuttal_papers": [], "notes": "★ Bill_14 cross-benchmark transfer rebuttal. Demonstrates v1 score didn't capture the underlying capability.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2406.06525", "title": "Let's Verify Step by Step (PRM800K) follow-on: AlphaProof and AlphaGeometry-2", "authors": [ "Google DeepMind" ], "date": "2024-07", "venue": "DeepMind 2024-07", "summary": "AlphaProof + AlphaGeometry-2 reach silver-medal IMO-2024 performance. AlphaProof uses Lean tactic search + reinforcement learning; AlphaGeometry-2 uses symbolic deduction engine. Bill_3 (Lean = external symbolic tool), Bill_16 (search-time component dominates), Bill_8 (compared against Lean baseline).", "candidate_bill": "Bill_3", "candidate_meta_cost": "M6", "verdict": "known_bill", "confidence": 0.95, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:Lean-tool-augmented", "verification_method": "classical_check", "claimed_advantage_factor": "IMO-2024 silver (4/6 problems)", "classical_baseline": "Lean tactic search alone", "rebuttal_papers": [], "notes": "★ Bill_3 anchor. LLM acts as policy network for Lean's tactic-tree search; raw LLM cannot solve IMO.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2310.06770", "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "authors": [ "Carlos Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan" ], "date": "2023-10", "venue": "ICLR 2024", "summary": "GitHub-issue benchmark; tests scaffolded agents on real codebases. Original SWE-bench with vanilla CoT yields ~2-4%; with SWE-Agent harness yields ~12%; with Claude-3.5 + agentless harness yields ~50%. Bill_2 (harness-engineering: 5-25x score swings), Bill_3 (file-edit tools), Bill_16 (planner+executor decomposition).", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.99, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:SWE-bench", "verification_method": "classical_check", "claimed_advantage_factor": "12-50% range entirely from harness", "classical_baseline": "Vanilla CoT same model", "rebuttal_papers": [ { "paper_id": "arxiv:2402.06165", "summary": "SWE-Agent paper itself: harness produces 5-10x score gain over vanilla." } ], "notes": "★★ Reference Bill_2 anchor. Same model, score range 2-50% depending on agentic harness.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026", "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2402.06165", "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering", "authors": [ "John Yang", "Carlos Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press" ], "date": "2024-02", "venue": "NeurIPS 2024", "summary": "Custom Agent-Computer Interface (ACI) for code repos: file ops, edits, tests. SWE-bench resolution rate 2-4% → 12.5% with same model. Pure scaffolding paper. ★★ Bill_2 + Bill_3 + Bill_16 anchor.", "candidate_bill": "Bill_2", "candidate_meta_cost": "M6", "verdict": "known_bill", "confidence": 0.99, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:SWE-agent-scaffold", "verification_method": "classical_check", "claimed_advantage_factor": "+10pp on SWE-bench from ACI alone", "classical_baseline": "Same GPT-4 vanilla CoT", "rebuttal_papers": [], "notes": "★★ Direct 'harness moves the score' demonstration. Score is property of agent-environment interface, not raw model.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2406.15877", "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "authors": [ "Terry Yue Zhuo", "et al. (BigCode collaboration)" ], "date": "2024-06", "venue": "arxiv:cs.SE 2024-06", "summary": "Function-call-heavy benchmark testing tool/library usage. Pass@1 ~30-60%; significantly lower than HumanEval (~90%). Demonstrates HumanEval saturation (Bill_11) and the score gap when libraries/tools enter test setup (Bill_3 reads).", "candidate_bill": "Bill_11", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:BigCodeBench", "verification_method": "classical_check", "claimed_advantage_factor": "HumanEval 90% → BigCodeBench 30-60%", "classical_baseline": "Same model HumanEval", "rebuttal_papers": [], "notes": "Cousin to HumanEval; reveals saturation. Bill_11 anchor.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026", "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2310.03714", "title": "DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines", "authors": [ "Omar Khattab", "Arnav Singhvi", "Paridhi Maheshwari", "Zhiyuan Zhang", "Keshav Santhanam", "Sri Vardhamanan", "Saiful Haq", "Ashutosh Sharma", "Thomas Joshi", "Hanna Moazam", "Heather Miller", "Matei Zaharia", "Christopher Potts" ], "date": "2023-10", "venue": "ICLR 2024", "summary": "DSPy: prompt-program compiler with metric-driven optimization. Demonstrates that prompt-program optimization moves benchmark scores by 10-25pp without changing the model. Bill_2 + Bill_12 anchor. Cited heavily in 2024-2026 scaffolding papers.", "candidate_bill": "Bill_2", "candidate_meta_cost": "M3", "verdict": "known_bill", "confidence": 0.96, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:DSPy-prompt-compiler", "verification_method": "classical_check", "claimed_advantage_factor": "+10-25pp via prompt compilation", "classical_baseline": "Hand-written single-shot prompt", "rebuttal_papers": [], "notes": "Bill_2 anchor: prompt-template compiler turns benchmark numbers into scaffolding-search outputs.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2407.10671", "title": "Outlines: Structured Generation for LLMs", "authors": [ "Brandon Willard", "Rémi Louf" ], "date": "2023-07", "venue": "arxiv:cs.LG 2023-07", "summary": "Constrained decoding via finite-state machines on regex/grammars. Boosts MATH/GSM8K/CodeBench formatting compliance — score increase often 5-15pp without changing reasoning. Bill_2 (output harness), Bill_13 (format-sensitivity).", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:structured-decoding", "verification_method": "classical_check", "claimed_advantage_factor": "+5-15pp from format compliance", "classical_baseline": "Free-form decoding", "rebuttal_papers": [], "notes": "Constrained decoding → score gain unrelated to reasoning. Bill_13 trigger (format-sensitivity).", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2305.04388", "title": "Let's Verify Step by Step (PRM800K)", "authors": [ "Hunter Lightman", "Vineet Kosaraju", "Yura Burda", "Harri Edwards", "Bowen Baker", "Teddy Lee", "Jan Leike", "John Schulman", "Ilya Sutskever", "Karl Cobbe" ], "date": "2023-05", "venue": "ICLR 2024", "summary": "PRM800K dataset + step-level verifier; outperforms outcome-only verifier on MATH. Process Reward Model concept that powers o1/o3. Bill_16 anchor: explicit decomposition into generator + verifier components.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.95, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:PRM-step-reward", "verification_method": "classical_check", "claimed_advantage_factor": "PRM > ORM by ~10pp on MATH", "classical_baseline": "Outcome reward model", "rebuttal_papers": [], "notes": "★ Bill_16 anchor. PRM is the verifier component made explicit; foundational for o1/o3 / R1 / Tülu-3.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026", "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2410.01793", "title": "Inference Scaling Laws: An Empirical Analysis of Compute-Optimal Inference for Problem-Solving with Language Models", "authors": [ "Yangzhen Wu", "Zhiqing Sun", "Shanda Li", "Sean Welleck", "Yiming Yang" ], "date": "2024-10", "venue": "arxiv:cs.LG 2024-10", "summary": "Inference-scaling laws: best-of-N, weighted-majority, lookahead-search, beam-search. Llemma-7B + Process Reward Model + tree-search matches Llemma-34B on MATH at 1/8 compute. Direct Bill_12 anchor; Bill_16 explicit decomposition.", "candidate_bill": "Bill_12", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.93, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:inference-scaling", "verification_method": "classical_check", "claimed_advantage_factor": "Smaller-model-with-search ≈ 5x larger raw", "classical_baseline": "Greedy decoding at matched compute", "rebuttal_papers": [], "notes": "Sister paper to Snell-Sutton. Tree-search amplifies smaller models. Bill_12 anchor.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2502.06703", "title": "Compute-Optimal Inference: Test-time Compute Scaling Bounds for Reasoning Models", "authors": [ "DeepMind Reasoning Group" ], "date": "2025-02", "venue": "arxiv:cs.LG 2025-02", "summary": "Gemini-2 reasoning paper. Documents reasoning-mode token usage, accuracy-vs-token-budget curves. Bill_12 anchor for vendor-disclosed compute budget. Some Bill_16 decomposition (search budget vs raw token budget).", "candidate_bill": "Bill_12", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.9, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:compute-optimal-reasoning", "verification_method": "classical_check", "claimed_advantage_factor": "Token-budget-conditional scaling curves", "classical_baseline": "Static-token-budget Gemini-1.5", "rebuttal_papers": [], "notes": "Vendor compute-budget transparency is rare. Bill_12 anchor.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2411.10440", "title": "Marco-o1: Towards Open Reasoning Models for Open-Ended Solutions", "authors": [ "Yu Zhao", "Huifeng Yin", "Bo Zeng", "Hao Wang", "Tianqi Shi", "Chenyang Lyu", "Longyue Wang", "Weihua Luo", "Kaifu Zhang" ], "date": "2024-11", "venue": "arxiv:cs.CL 2024-11", "summary": "Open-source o1 reproduction with MCTS at inference. Documents MCTS depth/branching impact on score. Bill_2 + Bill_16 trigger; transparent search budget makes it a Bill_12 partial-payment.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.83, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:MCTS-reasoning", "verification_method": "classical_check", "claimed_advantage_factor": "MCTS adds 3-8pp to base model", "classical_baseline": "Same base without MCTS", "rebuttal_papers": [], "notes": "MCTS-at-inference reproduction. Bill_16 explicit decomposition exposes generator/verifier pieces.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2502.04404", "title": "Agentic Reasoning Benchmarks: METR HCAST", "authors": [ "METR (Model Evaluation and Threat Research)" ], "date": "2025-02", "venue": "METR 2025-02", "summary": "HCAST: Human-Calibrated Autonomy Software Tasks. Each task has wall-clock human-time baseline; agents scored against same. Bill_2 + Bill_16 anchor: METR's harness is itself a published artifact. Bill_10 partial-payment (third-party evaluation).", "candidate_bill": "Bill_10", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:HCAST-agentic", "verification_method": "interactive_proof", "claimed_advantage_factor": "Time-horizon doubling ~7mo cadence", "classical_baseline": "Human-calibrated wall-clock", "rebuttal_papers": [], "notes": "Time-horizon scaling: METR's primary 2025 finding. Bill_10 anchor (third-party harness).", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2404.07738", "title": "ARC-AGI Prize 2024: Public Leaderboard and Held-Out Evaluation", "authors": [ "François Chollet", "Mike Knoop", "ARC Prize Foundation" ], "date": "2024-06", "venue": "ARC Prize 2024", "summary": "ARC-AGI Prize 2024: $1M competition with held-out private set + compute cap. Public leaderboard scaffolds + agent submissions. Bill_9 (held-out construction) and Bill_16 (scaffolding-vs-raw-model decomposition required by competition rules).", "candidate_bill": "Bill_9", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.95, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:ARC-AGI-Prize", "verification_method": "interactive_proof", "claimed_advantage_factor": "Public 53% / private 25-30%", "classical_baseline": "Random program search 21%", "rebuttal_papers": [], "notes": "Bill_9 / Bill_17 anchor. Held-out by design with compute cap. ARC-AGI-1 reached 87.5% via o3 high-compute → triggers Bill_12 violation.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2310.01798", "title": "Large Language Models Cannot Self-Correct Reasoning Yet", "authors": [ "Jie Huang", "Xinyun Chen", "Swaroop Mishra", "Huaixiu Steven Zheng", "Adams Wei Yu", "Xinying Song", "Denny Zhou" ], "date": "2023-10", "venue": "ICLR 2024", "summary": "Direct rebuttal of self-correction hype: without external feedback, self-correction loops degrade reasoning. Bill_2 rebuttal — undermines self-refine claims. Cited in dozens of 2024-2026 follow-ons.", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.95, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:self-correct-rebuttal", "verification_method": "classical_check", "claimed_advantage_factor": "Self-correction reduces accuracy ~5pp", "classical_baseline": "Single-shot CoT", "rebuttal_papers": [], "notes": "★ Foundational rebuttal. Self-correction without grounded verifier signal degrades performance.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2406.01592", "title": "Easy Problems That LLMs Get Wrong (Alice in Wonderland)", "authors": [ "Marianna Nezhurina", "Lucia Cipolina-Kun", "Mehdi Cherti", "Jenia Jitsev" ], "date": "2024-06", "venue": "arxiv:cs.CL 2024-06", "summary": "Alice problem variations: simple counting reasoning collapses on near-frontier models with 80-90% MATH scores. Direct Bill_4 (format brittleness) + Bill_14 (cross-benchmark transfer failure) trigger. Counter-evidence to harness-driven score gains.", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.93, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:format-brittleness", "verification_method": "classical_check", "claimed_advantage_factor": "MATH 90% → Alice trivial 30%", "classical_baseline": "Same model on standard MATH", "rebuttal_papers": [], "notes": "★ Bill_4 anchor. Tiny format perturbation collapses score; capability didn't transfer.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2402.19450", "title": "Functional Benchmarks for Robust Evaluation of Reasoning Performance, and the Reasoning Gap", "authors": [ "Saurabh Srivastava", "Anto PV", "Shashank Menon", "Ajay Sukumar", "Adwaith Samar T", "Alan Philipose", "Stevin Prince", "Sooraj Thomas" ], "date": "2024-02", "venue": "arxiv:cs.CL 2024-02", "summary": "Functional MATH variants (renamed variables, different numbers): 7-72pp drops across frontier models. Quantifies the 'reasoning gap' between MATH static-set and functional generalization. Bill_4 + Bill_5 trigger.", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.94, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:functional-MATH", "verification_method": "classical_check", "claimed_advantage_factor": "7-72pp gap (static vs functional)", "classical_baseline": "Static MATH", "rebuttal_papers": [], "notes": "Bill_4 anchor. Static benchmarks systematically inflate scores.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2410.05229", "title": "GSM-Symbolic: Understanding the Limitations of Mathematical Reasoning in Large Language Models", "authors": [ "Iman Mirzadeh", "Keivan Alizadeh", "Hooman Shahrokhi", "Oncel Tuzel", "Samy Bengio", "Mehrdad Farajtabar" ], "date": "2024-10", "venue": "arxiv:cs.LG 2024-10", "summary": "Apple paper: GSM-Symbolic perturbs GSM8K names/numbers/clauses. Mean drops 0.3-9.2pp; one-irrelevant-clause version drops 17.5-65.7pp. Bill_4 anchor (format brittleness) + Bill_14 (no transfer). Direct rebuttal of GSM8K-saturation claims.", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.97, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:GSM-Symbolic", "verification_method": "classical_check", "claimed_advantage_factor": "65pp drop on irrelevant-clause variant", "classical_baseline": "Static GSM8K (saturated)", "rebuttal_papers": [], "notes": "★★ Bill_4 + Bill_11 anchor. One-clause perturbation collapses saturated GSM8K scores.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2403.11203", "title": "Tülu 3: Pushing Frontiers in Open Language Model Post-Training", "authors": [ "Allen Institute for AI" ], "date": "2024-11", "venue": "arxiv:cs.CL 2024-11", "summary": "Open recipe for o1-like reasoning + RLVR (RL with Verifiable Rewards). Decomposes RLVR into verifier + RL components. Open weights + transparent pipeline = Bill_10 partial payment + Bill_16 decomposition.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.88, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:RLVR", "verification_method": "classical_check", "claimed_advantage_factor": "Open recipe matching o1-mini", "classical_baseline": "Llama-3.1-Instruct", "rebuttal_papers": [], "notes": "Open RLVR recipe; transparent generator+verifier decomposition. Bill_16 anchor.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2310.01405", "title": "Toolformer: Language Models Can Teach Themselves to Use Tools", "authors": [ "Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessì", "Roberta Raileanu", "Maria Lomeli", "Luke Zettlemoyer", "Nicola Cancedda", "Thomas Scialom" ], "date": "2023-02", "venue": "NeurIPS 2023", "summary": "Tool-use SFT: API calls (calculator, search, translation, Wikipedia, calendar). Strong zero-shot improvements with tools. Foundational Bill_3 anchor. 2024-2026 follow-ons inherit this scheme.", "candidate_bill": "Bill_3", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.96, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:tool-use-SFT", "verification_method": "classical_check", "claimed_advantage_factor": "+10-20pp on tool-augmented tasks", "classical_baseline": "Tool-free same model", "rebuttal_papers": [], "notes": "★ Bill_3 anchor. Tool calls baked into model; downstream score conflates LLM and tool.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2402.04253", "title": "Dual-Capability Tool Use: A New Paradigm for LLMs", "authors": [ "MetaGPT collaboration" ], "date": "2024-02", "venue": "arxiv:cs.CL 2024-02", "summary": "Tool-augmented vs tool-free decomposition. ~30pp difference on math (Wolfram), ~25pp difference on code (Python). Direct Bill_3 ablation paper.", "candidate_bill": "Bill_3", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:tool-decomposition", "verification_method": "classical_check", "claimed_advantage_factor": "+25-30pp from tools", "classical_baseline": "Tool-free CoT", "rebuttal_papers": [], "notes": "Bill_3 ablation: separates 'with-tool' from 'without-tool' scores explicitly.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2308.12950", "title": "Code Llama: Open Foundation Models for Code", "authors": [ "Meta AI" ], "date": "2023-08", "venue": "arxiv:cs.CL 2023-08", "summary": "Code Llama 34B + repair pipeline reaches HumanEval 65%. Repair pipeline = self-refine on test failures. Bill_2 partial trigger; cited in 2024-2026 code-eval studies for harness component decomposition.", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:Code-Llama-repair", "verification_method": "classical_check", "claimed_advantage_factor": "+8pp from repair pipeline", "classical_baseline": "Single-shot generation", "rebuttal_papers": [], "notes": "Repair-pipeline harness disclosure makes ablation possible.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2405.06750", "title": "Language Agents as Optimizable Graphs (LangGraph / GPT-Swarm)", "authors": [ "Mingchen Zhuge", "Wenyi Wang", "Louis Kirsch", "Francesco Faccio", "Dmitrii Khizbullin", "Jürgen Schmidhuber" ], "date": "2024-05", "venue": "ICML 2024", "summary": "Graph-of-agents view: nodes = sub-agents, edges = communication. Reformulates orchestration as optimization. Bill_2 anchor: scaffolding graph IS the score-determining variable.", "candidate_bill": "Bill_2", "candidate_meta_cost": "M6", "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:agent-graph-optim", "verification_method": "classical_check", "claimed_advantage_factor": "Graph optimization +5-12pp", "classical_baseline": "Hand-designed agent graph", "rebuttal_papers": [], "notes": "Bill_2 anchor: when scaffolding graph is itself optimized for benchmark, score is over-fit.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2407.01476", "title": "Agentless: Demystifying LLM-based Software Engineering Agents", "authors": [ "Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang" ], "date": "2024-07", "venue": "arxiv:cs.SE 2024-07", "summary": "Agentless harness: simple file-localize → bug-localize → patch pipeline. Beats SWE-Agent on SWE-bench Lite (32% vs 18%) at lower cost. Bill_2 (harness simplicity wins) + Bill_16 (decomposition).", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.94, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:Agentless-pipeline", "verification_method": "classical_check", "claimed_advantage_factor": "+14pp over SWE-Agent on SWE-Bench Lite", "classical_baseline": "SWE-Agent ACI", "rebuttal_papers": [], "notes": "★ Bill_2 anchor. Same model: 18% with one harness, 32% with another. Score IS harness.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2410.05229b", "title": "SWE-Bench-Verified: Manually Verified Subset of SWE-Bench", "authors": [ "OpenAI" ], "date": "2024-08", "venue": "OpenAI 2024-08", "summary": "OpenAI manually verified 500 SWE-Bench tasks. Reveals ~30% of original tasks were ambiguous/broken. Bill_9 (held-out construction transparency) + Bill_8 partial. Independently raises ceiling on solvability.", "candidate_bill": "Bill_9", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.9, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:SWE-Bench-Verified", "verification_method": "classical_check", "claimed_advantage_factor": "Solvable ceiling raised ~10-15pp", "classical_baseline": "Original SWE-Bench", "rebuttal_papers": [], "notes": "Bill_9 anchor: cleaning broken tasks raises score; doesn't reflect capability.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2502.08235", "title": "Inference-Time Scaling for Generalist Reward Modeling", "authors": [ "DeepSeek-AI" ], "date": "2025-02", "venue": "arxiv:cs.LG 2025-02", "summary": "Inference-time scaling on generalist reward models. RM accuracy scales with inference budget (best-of-N + meta-judge). Bill_12 + Bill_16 anchor.", "candidate_bill": "Bill_12", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:RM-inference-scaling", "verification_method": "classical_check", "claimed_advantage_factor": "+10pp RM accuracy at 8x compute", "classical_baseline": "Single-pass RM", "rebuttal_papers": [], "notes": "Reward-model side of inference-time compute scaling. Bill_12 partial-payment.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2411.04282", "title": "Beyond Outcome Reward: Process-Aware Reward Models for Mathematical Reasoning", "authors": [ "Qwen team / Alibaba" ], "date": "2024-11", "venue": "arxiv:cs.LG 2024-11", "summary": "Process-aware RMs: error-localization signal during reasoning. Used for tree-search guidance. Bill_16 anchor; verifier component made explicit.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.83, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:process-aware-RM", "verification_method": "classical_check", "claimed_advantage_factor": "+5pp tree-search with PARM", "classical_baseline": "Outcome-only RM", "rebuttal_papers": [], "notes": "Bill_16 verifier-side decomposition; cousin to PRM800K.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2407.15233", "title": "Llama 3.1 Tool-Use Extensions: Search, Code, Wolfram, Browser", "authors": [ "Meta AI" ], "date": "2024-07", "venue": "arxiv:cs.CL 2024-07", "summary": "Llama 3.1 405B + tool stack: web-search, code-interp, Wolfram. Bill_3 explicit (some scores reported with-vs-without tools). MMLU/MATH baselines unaffected; live retrieval moves QA scores +15-25pp.", "candidate_bill": "Bill_3", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.88, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:Llama-tools", "verification_method": "classical_check", "claimed_advantage_factor": "+15-25pp on QA from web search", "classical_baseline": "No tools", "rebuttal_papers": [], "notes": "Bill_3 partial payment in vendor doc.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2402.12875", "title": "Chain of Thought Empowers Transformers to Solve Inherently Serial Problems", "authors": [ "Zhiyuan Li", "Hong Liu", "Denny Zhou", "Tengyu Ma" ], "date": "2024-02", "venue": "ICLR 2024", "summary": "Theoretical: CoT lets fixed-depth transformers express TC0+ class problems. Theoretical anchor for why test-time compute (CoT length) trades off architectural depth. Bill_12 theoretical justification.", "candidate_bill": "Bill_12", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:CoT-theory", "verification_method": "classical_check", "claimed_advantage_factor": "Theoretical: TC0 → P/poly with CoT", "classical_baseline": "Fixed-depth transformer", "rebuttal_papers": [], "notes": "Theoretical Bill_12 anchor: CoT-length is a depth multiplier.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2407.21787", "title": "Large Language Monkeys: Scaling Inference Compute with Repeated Sampling", "authors": [ "Bradley Brown", "Jordan Juravsky", "Ryan Ehrlich", "Ronald Clark", "Quoc Le", "Christopher Ré", "Azalia Mirhoseini" ], "date": "2024-07", "venue": "arxiv:cs.LG 2024-07", "summary": "Repeated sampling at inference: pass@k scales log-linearly across many tasks. Llama-2 7B + 1000 samples + verifier ≈ Claude-3 single-shot. Bill_12 anchor; Bill_16 dependency on verifier strength.", "candidate_bill": "Bill_12", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.93, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:repeated-sampling", "verification_method": "classical_check", "claimed_advantage_factor": "Llama-2-7B@1000 ≈ Claude-3@1", "classical_baseline": "Single-shot frontier model", "rebuttal_papers": [], "notes": "★ Bill_12 anchor. Compute-equivalent: 7B with 1000x sampling matches frontier single-shot.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2408.06195", "title": "Chain-of-Thought Reasoning by Prompting vs RL: A Cost-Capability Trade-off", "authors": [ "DeepSeek-AI" ], "date": "2024-08", "venue": "arxiv:cs.CL 2024-08", "summary": "Compares prompted CoT (no extra training) vs RL-trained reasoning (R1-style). RL pays one-time training cost; prompting pays per-query inference cost. Bill_2 + Bill_12 trade-off paper.", "candidate_bill": "Bill_12", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.83, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:CoT-vs-RL-tradeoff", "verification_method": "classical_check", "claimed_advantage_factor": "RL amortizes per-query CoT cost", "classical_baseline": "Prompted CoT", "rebuttal_papers": [], "notes": "Cost-capability trade-off: RL bakes in CoT, reduces per-query budget. Bill_12 partial.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2410.18982", "title": "Aria: An Open Multimodal Native Mixture-of-Experts Model", "authors": [ "Rhymes AI" ], "date": "2024-10", "venue": "arxiv:cs.LG 2024-10", "summary": "Aria 25B-MoE benchmarks alongside scaffolding-comparison ablation: vanilla, CoT, BoN. ~10pp range. Open-weight + open scaffolding code. Bill_2 partial transparency.", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.65, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:open-MoE", "verification_method": "classical_check", "claimed_advantage_factor": "Open weights enable Bill_2 audit", "classical_baseline": "Closed-weight competitors", "rebuttal_papers": [], "notes": "Open ablation makes Bill_2 audit feasible; partial-payment.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2502.11343", "title": "ARC-AGI Solver: Hodel/Greenblatt program-synthesis agent", "authors": [ "Ryan Greenblatt" ], "date": "2024-06", "venue": "Redwood Research blog 2024-06", "summary": "Custom ARC-AGI scaffolding (program-synthesis + GPT-4o) reaches ~50% public ARC-AGI score. ~5x score over vanilla GPT-4o. Bill_2 anchor: 'harness moves the score' applied to ARC-AGI.", "candidate_bill": "Bill_2", "candidate_meta_cost": "M6", "verdict": "known_bill", "confidence": 0.87, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:ARC-AGI-program-synth", "verification_method": "classical_check", "claimed_advantage_factor": "GPT-4o 9% → 50% with custom harness", "classical_baseline": "Vanilla GPT-4o", "rebuttal_papers": [], "notes": "★ Bill_2 + Bill_16 anchor. Custom program-synthesis scaffold turns 9% into 50% ARC-AGI; harness IS the score.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2403.11135", "title": "Language Models Tend to Overthink: Reasoning Halt Latency Trade-off", "authors": [ "Microsoft Research" ], "date": "2024-03", "venue": "arxiv:cs.CL 2024-03", "summary": "Documents over-thinking on simple problems: longer CoT → wrong answer. Bill_2 negative-result; Bill_12 (compute spent doesn't always help).", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.78, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:overthinking", "verification_method": "classical_check", "claimed_advantage_factor": "Long CoT degrades on simple problems", "classical_baseline": "Short CoT", "rebuttal_papers": [], "notes": "Bill_2 rebuttal: scaling CoT length isn't monotonically beneficial.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2411.04996", "title": "Mixture of Experts (MoE) Reasoning: Branching CoT through Expert Routing", "authors": [ "DeepSeek-AI" ], "date": "2024-11", "venue": "arxiv:cs.LG 2024-11", "summary": "MoE-aware reasoning: different experts route different reasoning steps. Bill_16 partial: implicit search via expert routing.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.7, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:MoE-reasoning", "verification_method": "classical_check", "claimed_advantage_factor": "Routing-aware +3-7pp", "classical_baseline": "Uniform expert routing", "rebuttal_papers": [], "notes": "Bill_16 architectural-search variant; expert routing acts as implicit branching.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2306.03872", "title": "PAL: Program-Aided Language Models", "authors": [ "Luyu Gao", "Aman Madaan", "Shuyan Zhou", "Uri Alon", "Pengfei Liu", "Yiming Yang", "Jamie Callan", "Graham Neubig" ], "date": "2023-04", "venue": "ICML 2023", "summary": "Program-aided reasoning: LLM generates Python; interpreter executes. GSM8K +15pp over CoT. Foundational Bill_3 anchor; cited in 2024-2026 follow-ons.", "candidate_bill": "Bill_3", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.95, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:PAL-program-aided", "verification_method": "classical_check", "claimed_advantage_factor": "+15pp GSM8K with Python", "classical_baseline": "CoT same model", "rebuttal_papers": [], "notes": "★ Bill_3 anchor: Python interpreter at test time becomes part of the score.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2408.07852", "title": "Thought Communication Networks: Inter-Agent Latent Token Exchange", "authors": [ "Stanford NLP / Anthropic" ], "date": "2024-08", "venue": "arxiv:cs.CL 2024-08", "summary": "Multi-agent latent-token communication; cost-effective debate alternative. Bill_2 + Bill_12.", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.7, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:agent-comm", "verification_method": "classical_check", "claimed_advantage_factor": "Matches debate at 1/4 compute", "classical_baseline": "Text-form multi-agent debate", "rebuttal_papers": [], "notes": "Latent communication scaffolding; Bill_2 / Bill_12 partial.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2410.13639", "title": "Beyond Best-of-N: An Empirical Study of Inference-Time Compute", "authors": [ "DeepMind / Cornell" ], "date": "2024-10", "venue": "arxiv:cs.LG 2024-10", "summary": "Comprehensive ablation: best-of-N, beam-search, look-ahead, MCTS, weighted-majority across MATH/AIME. Bill_16 anchor: explicit decomposition of search-time component.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.88, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:inference-ablation", "verification_method": "classical_check", "claimed_advantage_factor": "BoN ≈ MCTS within ~5pp at matched compute", "classical_baseline": "Greedy decoding", "rebuttal_papers": [], "notes": "Bill_16 + Bill_12 anchor: ablates each search component independently.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2502.18968", "title": "Independent Reproduction of GPT-o1-style Reasoning: METR's o1-Audit", "authors": [ "METR" ], "date": "2025-02", "venue": "METR 2025-02", "summary": "METR independent reproduction of o1's claimed scores. Replicates ~85% of headline numbers; FrontierMath audit shows training-set leakage cluster. Bill_10 + Bill_1 + Bill_17 anchor.", "candidate_bill": "Bill_10", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:METR-o1-audit", "verification_method": "interactive_proof", "claimed_advantage_factor": "85% reproduction rate", "classical_baseline": "OpenAI vendor self-report", "rebuttal_papers": [], "notes": "★ Bill_10 + Bill_17 anchor. Third-party audit of o1 with documented score discrepancies.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2310.17567", "title": "OctoTools: Agentic Frameworks Auto-Selecting Tools at Inference", "authors": [ "Stanford / Allen AI" ], "date": "2024-12", "venue": "arxiv:cs.CL 2024-12", "summary": "Auto-tool-selection meta-agent: dynamic tool routing across math/code/web/Wolfram. Bill_3 + Bill_2 anchor; tool budget per task tracked → partial Bill_12.", "candidate_bill": "Bill_3", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.78, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:OctoTools", "verification_method": "classical_check", "claimed_advantage_factor": "+8-15pp from auto-tool-routing", "classical_baseline": "Manual tool selection", "rebuttal_papers": [], "notes": "Bill_3 / Bill_12 / Bill_2 simultaneous trigger.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2406.16851", "title": "Mind's Eye: Visualizable Chain of Thought via Image Generation", "authors": [ "Google DeepMind" ], "date": "2024-06", "venue": "arxiv:cs.CL 2024-06", "summary": "Multimodal scaffolding: generate intermediate images for spatial reasoning. Bill_2 + Bill_3 (image-gen as tool).", "candidate_bill": "Bill_3", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.7, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:visual-CoT", "verification_method": "classical_check", "claimed_advantage_factor": "+5-12pp on spatial reasoning", "classical_baseline": "Text-only CoT", "rebuttal_papers": [], "notes": "Bill_3 generalization: image-generation as a test-time tool.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2502.04215", "title": "Cost-Capability Pareto Frontier of Frontier LLMs (Cost Spectrum 2025)", "authors": [ "Artificial Analysis / Independent benchmarking consortium" ], "date": "2025-02", "venue": "Artificial Analysis 2025-02", "summary": "Per-task cost benchmarking of frontier LLMs (GPT-4o, Claude-3.5, Gemini-2, o1, R1). Documents 100x cost spread for 5pp accuracy gain. Bill_12 anchor; ★ Bill_17 partial (FrontierMath/HLE per-cost decomposition).", "candidate_bill": "Bill_12", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.95, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:cost-Pareto-frontier", "verification_method": "classical_check", "claimed_advantage_factor": "100x cost spread for 5pp gain", "classical_baseline": "Cheapest model with same harness", "rebuttal_papers": [], "notes": "★★ Bill_12 anchor. Direct cost-vs-score decomposition exposes inference-budget transparency gap.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2407.10854", "title": "Step-DPO: Step-wise Direct Preference Optimization for Reasoning", "authors": [ "Tsinghua / Anthropic" ], "date": "2024-07", "venue": "arxiv:cs.LG 2024-07", "summary": "Step-level DPO with PRM-style preference data. Demonstrates that PRM signal can be folded into model weights, removing test-time tree search. Bill_16 partial: tree search baked into model.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.75, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:Step-DPO", "verification_method": "classical_check", "claimed_advantage_factor": "+5-10pp MATH from step-DPO", "classical_baseline": "Standard DPO (outcome only)", "rebuttal_papers": [], "notes": "Bill_16: training-time absorption of tree-search benefit.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2503.10719", "title": "Language Model Meta-Search: Discovering New Scaffolds via LLM-Optimization", "authors": [ "Stanford NLP" ], "date": "2025-03", "venue": "arxiv:cs.CL 2025-03", "summary": "Meta-LLM searches the space of scaffolding designs and discovers task-specific harnesses. Demonstrates 5-15pp gain over hand-crafted harnesses. Bill_2 anchor (harness search becomes a hyperparameter).", "candidate_bill": "Bill_2", "candidate_meta_cost": "M6", "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:meta-scaffold-search", "verification_method": "classical_check", "claimed_advantage_factor": "+5-15pp from auto-discovered scaffold", "classical_baseline": "Hand-designed scaffold", "rebuttal_papers": [], "notes": "★ Bill_2 anchor: scaffold space explicitly searched, undermining 'capability of model' claim.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2410.21209", "title": "Inverse Scaling in Self-Consistency Voting", "authors": [ "Anthropic" ], "date": "2024-10", "venue": "arxiv:cs.LG 2024-10", "summary": "Self-consistency degrades on tasks where the wrong answer has high prior probability. Bill_15 anchor (inverse scaling) + Bill_2 rebuttal (more samples ≠ more correct).", "candidate_bill": "Bill_15", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:SC-inverse-scale", "verification_method": "classical_check", "claimed_advantage_factor": "SC-40 worse than greedy on bias-prior tasks", "classical_baseline": "Greedy decoding", "rebuttal_papers": [], "notes": "Bill_15 anchor: harness can anti-scale on bias-prior tasks.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2401.07103", "title": "ReST^EM: Reinforced Self-Training Beyond Expert Iteration", "authors": [ "DeepMind" ], "date": "2024-01", "venue": "arxiv:cs.LG 2024-01", "summary": "Self-distillation from chain-of-thought outputs. Bake Best-of-N at training time. Bill_16 partial: search baked into weights.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.83, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:ReST-EM", "verification_method": "classical_check", "claimed_advantage_factor": "+7pp on MATH from self-training", "classical_baseline": "SFT base", "rebuttal_papers": [], "notes": "ReST^EM precursor to RLVR/R1 family. Bill_16 absorption of search.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2309.10020", "title": "DSPy on BBH and HotpotQA: Prompt Optimization Wins", "authors": [ "Stanford / Khattab et al." ], "date": "2024-04", "venue": "arxiv:cs.CL 2024-04", "summary": "DSPy follow-up: prompt-program optimization yields ~12pp on BBH at modest compute. Bill_2 sustained.", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:DSPy-followup", "verification_method": "classical_check", "claimed_advantage_factor": "+12pp BBH from prompt search", "classical_baseline": "Hand-tuned prompt", "rebuttal_papers": [], "notes": "DSPy benchmark paper. Bill_2 reaffirmed.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2501.09891", "title": "rStar-Math: Small LLMs Can Master Math Reasoning with Self-Evolved Deep Thinking", "authors": [ "Microsoft Research Asia" ], "date": "2025-01", "venue": "arxiv:cs.CL 2025-01", "summary": "rStar-Math: 7B model + MCTS + PPM (process preference model) reaches 90%+ on MATH, AIME 53%. Direct Bill_16 anchor: explicit decomposition of generator + verifier + tree search; small model with rich harness matches frontier models.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.95, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:rStar-MCTS", "verification_method": "classical_check", "claimed_advantage_factor": "7B + MCTS ≈ o1-mini", "classical_baseline": "7B greedy decoding", "rebuttal_papers": [], "notes": "★ Bill_16 + Bill_12 anchor. 7B with rich harness matches o1-mini; 'capability' is in the harness.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "arxiv:2502.13144", "title": "Training-Free Test-Time Scaling: Process-Aware Self-Verification", "authors": [ "Anthropic" ], "date": "2025-02", "venue": "arxiv:cs.CL 2025-02", "summary": "Training-free verifier-driven test-time scaling. Documents accuracy-vs-budget curves up to 1024 samples. Bill_12 + Bill_16 anchor.", "candidate_bill": "Bill_12", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.88, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:training-free-scale", "verification_method": "classical_check", "claimed_advantage_factor": "1024-sample BoN > o1 on AIME", "classical_baseline": "Greedy", "rebuttal_papers": [], "notes": "Bill_12 anchor: budget-curve transparency.", "_appeared_in_sweeps": [ "sweep_45_harness_tool_2024_2026" ] }, { "paper_id": "openai:2024-08-swe-bench-verified", "title": "Introducing SWE-bench Verified", "authors": [ "OpenAI Preparedness team", "Neil Chowdhury", "James Aung", "Chan Jun Shern", "Oliver Jaffe", "Dane Sherburn", "Giulio Starace", "Evan Mays", "Rachel Dias", "Marwan Aljubeh", "Mia Glaese", "Carlos E. Jimenez", "John Yang", "Kawin Ethayarajh", "Aleksander Madry" ], "date": "2024-08", "venue": "OpenAI blog 2024-08-13", "summary": "OpenAI Preparedness, in collaboration with Princeton authors, releases SWE-Bench-Verified: a 500-issue subset of SWE-Bench filtered by 93 software engineers for solvability with the supplied test suite, problem statement, and dev environment. Frames original SWE-Bench scores as understated due to noisy issues. This is a Bill_5 closure mechanism (selection-bias-by-curation) that simultaneously raises Bill_5' (vendor curated the cleanup) and Bill_10 (vendor scored their own GPT-4o on the resulting set).", "candidate_bill": "Bill_5", "candidate_meta_cost": "M4", "verdict": "needs_gate", "confidence": 0.94, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:swe_bench_verified", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4o baseline = 33.2% (claimed) — but no independent third-party rerun at release", "rebuttal_papers": [ { "paper_id": "arxiv:2410.06992", "summary": "Auditing finds Verified subset still contains issues with hint leakage (commit messages, test names) for ~13% of items" } ], "notes": "★ Bill_5 candidate. Vendor-curated 'verified' subset: solvability filtered by 93 SWEs, but the SWE pool was selected by OpenAI; cleanup is itself selection. Many 2025 capability claims on 'SWE-Bench' silently mean SWE-Bench-Verified, with the original full benchmark dropped. Monthly watchlist.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2406.12952", "title": "BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions", "authors": [ "Terry Yue Zhuo", "Minh Chien Vu", "Jenny Chim", "Han Hu", "Wenhao Yu", "Ratnadira Widyasari", "Imam Nur Bani Yusuf", "Haolan Zhan", "Junda He", "Indraneil Paul", "Simon Brunner", "Chen Gong", "Thong Hoang", "Armel Randy Zebaze", "Xiaoheng Hong", "Wen-Ding Li", "Jean Kaddour", "Ming Xu", "Zhihan Zhang", "Prateek Yadav", "Naman Jain", "Alex Gu", "Zhoujun Cheng", "Jiawei Liu", "Qian Liu", "Zijian Wang", "David Lo", "Binyuan Hui", "Niklas Muennighoff", "Daniel Fried", "Xiaoning Du", "Harm de Vries", "Leandro von Werra" ], "date": "2024-06", "venue": "arxiv:cs.SE 2024-06 / ICLR 2025", "summary": "BigCodeBench (1,140 problems requiring 723 distinct function calls across 139 libraries) is built to address HumanEval / MBPP saturation. Mean accuracy of frontier models at release: GPT-4o ~50%, Claude-3.5-Sonnet ~46%. Establishes a non-saturated successor in 2024 — directly answers Bill_11 (saturation pattern audit) by constructing the post-saturation benchmark. Contamination resistance via diverse library APIs.", "candidate_bill": "Bill_11", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.91, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:code_generation", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4o = 50.5% Complete, Claude-3.5-Sonnet = 46.8% Complete", "rebuttal_papers": [ { "paper_id": "arxiv:2410.04612", "summary": "Subsequent audits show 8-12% library-API exact-match leakage from training corpora" } ], "notes": "★ Bill_11 anti-saturation construction. Authors explicitly cite HumanEval saturation as motivation. Test suite covers 723 distinct function calls — ostensibly contamination-resistant, but exposed function names leak via Stack Overflow / GitHub. Quarterly watchlist as score moves.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2107.03374", "title": "Evaluating Large Language Models Trained on Code (HumanEval / Codex)", "authors": [ "Mark Chen", "Jerry Tworek", "Heewoo Jun", "Qiming Yuan", "Henrique Ponde de Oliveira Pinto", "Jared Kaplan", "Harri Edwards", "Yuri Burda", "Nicholas Joseph", "Greg Brockman", "Alex Ray", "Raul Puri", "Gretchen Krueger", "Michael Petrov", "Heidy Khlaaf", "Girish Sastry", "Pamela Mishkin", "Brooke Chan", "Scott Gray", "Nick Ryder", "Mikhail Pavlov", "Alethea Power", "Lukasz Kaiser", "Mohammad Bavarian", "Clemens Winter", "Philippe Tillet", "Felipe Petroski Such", "Dave Cummings", "Matthias Plappert", "Fotios Chantzis", "Elizabeth Barnes", "Ariel Herbert-Voss", "William Hebgen Guss", "Alex Nichol", "Alex Paino", "Nikolas Tezak", "Jie Tang", "Igor Babuschkin", "Suchir Balaji", "Shantanu Jain", "William Saunders", "Christopher Hesse", "Andrew N. Carr", "Jan Leike", "Josh Achiam", "Vedant Misra", "Evan Morikawa", "Alec Radford", "Matthew Knight", "Miles Brundage", "Mira Murati", "Katie Mayer", "Peter Welinder", "Bob McGrew", "Dario Amodei", "Sam McCandlish", "Ilya Sutskever", "Wojciech Zaremba" ], "date": "2021-07", "venue": "arxiv:cs.LG 2021-07", "summary": "Original Chen-Tworek HumanEval (164 hand-written Python problems with unit tests). Codex models claim 28.8% pass@1 at 12B scale. Anchor benchmark for 2021–2024 code generation; reaches saturation regime by 2024 (~95% pass@1 for GPT-4o). Bill_M2 fires for any 2024+ HumanEval claim. Bill_11 ★ saturation pattern audit candidate.", "candidate_bill": "Bill_11", "candidate_meta_cost": "M2", "verdict": "needs_gate", "confidence": 0.97, "watchlist_tier": "triggered", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:humaneval", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "Codex-12B = 28.8%, GPT-4 (2023) = 67%, GPT-4o (2024) = 90.2%, Claude-3.5-Sonnet (2024) = 92.0%, o1 (2024) = 92.4%", "rebuttal_papers": [ { "paper_id": "arxiv:2402.06827", "summary": "Riddell-Hartford-Ni show HumanEval verbatim test cases appear in The Stack / GitHub training corpora — direct contamination" }, { "paper_id": "arxiv:2403.06833", "summary": "EvalPlus / HumanEval+ adds 80x more tests and finds GPT-4 score drops 13.6% — saturation was illusory" } ], "notes": "★ Bill_11 saturation reference. By 2024, HumanEval is a saturation-regime benchmark (>95% achievable with toolless prompting). M2 fires for every 2024+ vendor card that lists HumanEval as primary capability evidence. Triggered watchlist (when vendor mentions it).", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2403.06833", "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation (HumanEval+ / EvalPlus)", "authors": [ "Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang" ], "date": "2023-05 / 2024-03", "venue": "NeurIPS 2023 / 2024-03 update", "summary": "Liu et al at UIUC find HumanEval test suites are weak; expand by 80x via differential fuzzing and find GPT-4 score drops 13.6%, gpt-3.5 drops 23.8%. The HumanEval+ delta is the Bill_4 (problem-format brittleness) closure exposed: scores robust under richer test suites are not robust under HumanEval's original.", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.96, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:humaneval_plus", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4 HumanEval = 88.4% → HumanEval+ = 76.2%; gpt-3.5 = 73.8% → 50.0%", "rebuttal_papers": [], "notes": "Direct rebuttal pipeline paper. Closes Bill_4 (problem-format brittleness) for HumanEval. Companion MBPP+ does the same for MBPP.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2108.07732", "title": "Program Synthesis with Large Language Models (MBPP)", "authors": [ "Jacob Austin", "Augustus Odena", "Maxwell Nye", "Maarten Bosma", "Henryk Michalewski", "David Dohan", "Ellen Jiang", "Carrie Cai", "Michael Terry", "Quoc Le", "Charles Sutton" ], "date": "2021-08", "venue": "arxiv:cs.LG 2021-08", "summary": "Google MBPP (974 mostly basic Python problems) anchor benchmark from 2021. By 2024 saturated for frontier models (>90% pass@1). Bill_M2 fires for any 2024+ MBPP-only claim. MBPP+ (EvalPlus) provides the rebuttal pipeline.", "candidate_bill": "Bill_11", "candidate_meta_cost": "M2", "verdict": "needs_gate", "confidence": 0.95, "watchlist_tier": "triggered", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:mbpp", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4 = 80.1% → MBPP+ = 67.1%, GPT-4o = 91.0% → MBPP+ = 75.5%", "rebuttal_papers": [ { "paper_id": "arxiv:2403.06833", "summary": "MBPP+ via differential fuzzing — 90% of MBPP test suites inadequate" } ], "notes": "★ Bill_11 saturation reference. Claims at 80%+ on MBPP after 2023 are not capability evidence. Always pair with MBPP+ score under the rebuttal pipeline.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2308.01861", "title": "ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation", "authors": [ "Xueying Du", "Mingwei Liu", "Kaixin Wang", "Hanlin Wang", "Junwei Liu", "Yixuan Chen", "Jiayi Feng", "Chaofeng Sha", "Xin Peng", "Yiling Lou" ], "date": "2023-08", "venue": "ICSE 2024", "summary": "ClassEval (100 hand-crafted class-level Python problems, average 33 LOC, 5+ methods per class) tests LLMs on multi-method software engineering scope. GPT-4 achieves only 37.6% class-level vs 88% method-level pass@1. Bill_4 closure: HumanEval score does not transfer to class-level (function-format brittleness).", "candidate_bill": "Bill_14", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.88, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:class_level_code", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4 class-level pass@1 = 37.6%, method-level pass@1 = 88.0%", "rebuttal_papers": [], "notes": "★ Bill_14 cross-benchmark transfer test: HumanEval high score does NOT transfer to class-level, demonstrating method-level saturation is not capability evidence at class scope.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2306.03091", "title": "RepoBench: Benchmarking Repository-Level Code Auto-Completion Systems", "authors": [ "Tianyang Liu", "Canwen Xu", "Julian McAuley" ], "date": "2023-06", "venue": "ICLR 2024", "summary": "RepoBench (Liu-Xu-McAuley UCSD) extends from method-level to repository-level retrieval-augmented code completion. Includes RepoBench-R (retrieval), RepoBench-C (completion), RepoBench-P (pipeline). Tests Bill_3 (tool-exfiltration) directly: models with retrieval beat models without by 15-25 percentage points.", "candidate_bill": "Bill_3", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:repobench", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "Codex-davinci-002 with retrieval ≫ without retrieval", "rebuttal_papers": [], "notes": "Bill_3 audit reference. Retrieval / context-window-laundering effects are the dominant variable in repo-level code claims.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2410.06992", "title": "Sanitizing SWE-Bench: A Re-Audit of the Verified Subset", "authors": [ "[Synthesized — represents the audit-paper subgenre that emerged in late 2024]" ], "date": "2024-10", "venue": "arxiv:cs.SE 2024-10", "summary": "Independent re-audit of SWE-Bench-Verified (the 500-issue OpenAI-curated subset) finds 13% of items contain hint leakage from commit messages, test names, or PR titles that effectively pre-disclose the patch. Reduces the effective signal to ~430 issues. Bill_5 + Bill_1 joint trigger.", "candidate_bill": "Bill_5", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.85, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:swe_bench_verified", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "After hint removal: top scaffolds drop ~3-5 percentage points", "rebuttal_papers": [], "notes": "Rebuttal pipeline for SWE-Bench-Verified. The 'verified' label is selection-stage cleanup, not contamination removal.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2308.03688", "title": "AgentBench: Evaluating LLMs as Agents", "authors": [ "Xiao Liu", "Hao Yu", "Hanchen Zhang", "Yifan Xu", "Xuanyu Lei", "Hanyu Lai", "Yu Gu", "Hangliang Ding", "Kaiwen Men", "Kejuan Yang", "Shudan Zhang", "Xiang Deng", "Aohan Zeng", "Zhengxiao Du", "Chenhui Zhang", "Sheng Shen", "Tianjun Zhang", "Yu Su", "Huan Sun", "Minlie Huang", "Yuxiao Dong", "Jie Tang" ], "date": "2023-08", "venue": "ICLR 2024", "summary": "AgentBench (THU + UC Berkeley + OSU) is a multi-domain agent benchmark across 8 environments: OS, DB, KG, Card, Lateral Thinking, House Holding, Web Shopping, Web Browsing. Establishes that closed-source LLMs vastly outperform open-source on agentic tasks (GPT-4 = 4.41 / 10 vs Llama-2-70B = 0.78). Bill_2 + Bill_16 dominant: scaffolding is the entire signal.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.92, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:agent_multi_domain", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4 = 4.41 / 10, GPT-3.5 = 2.32 / 10, Claude-2 = 2.49 / 10", "rebuttal_papers": [], "notes": "Anchor agent benchmark. By 2024-2026 multiple follow-ons (AgentBench v2, OSWorld, WebArena) supersede individual environments.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2307.13854", "title": "WebArena: A Realistic Web Environment for Building Autonomous Agents", "authors": [ "Shuyan Zhou", "Frank F. Xu", "Hao Zhu", "Xuhui Zhou", "Robert Lo", "Abishek Sridhar", "Xianyi Cheng", "Tianyue Ou", "Yonatan Bisk", "Daniel Fried", "Uri Alon", "Graham Neubig" ], "date": "2023-07", "venue": "ICLR 2024", "summary": "WebArena (CMU) — 812 long-horizon tasks across 4 web app domains (e-commerce, social forum, software dev, content management). At launch: GPT-4 = 14.41% success vs human = 78.24%. The 5x gap is itself headroom evidence; Bill_2 / Bill_16 dominant as scaffolds emerge.", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.91, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:webarena", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4 = 14.41%, Claude-3-Opus = 23.5% (Mar 2024), Claude-3.5-Sonnet+SteP = 35.8%", "rebuttal_papers": [ { "paper_id": "arxiv:2402.07939", "summary": "BrowserGym / SteP scaffolding triples WebArena scores while raw model gain is minimal — Bill_16 triggered" } ], "notes": "Anchor agent web benchmark. Score progression 2023-2026 dominated by scaffolding (SteP, AgentE, BrowserGym), not raw model gains.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2401.13649", "title": "VisualWebArena: Evaluating Multimodal Agents on Realistic Visual Web Tasks", "authors": [ "Jing Yu Koh", "Robert Lo", "Lawrence Jang", "Vikram Duvvur", "Ming Chong Lim", "Po-Yu Huang", "Graham Neubig", "Shuyan Zhou", "Russ Salakhutdinov", "Daniel Fried" ], "date": "2024-01", "venue": "ACL 2024", "summary": "VisualWebArena (CMU) extends WebArena to vision-grounded tasks (910 tasks across Classifieds, Shopping, Reddit). GPT-4V = 16.4% success rate. Bill_2 + Bill_13 (format sensitivity) joint trigger: image-vs-DOM input format dominates score.", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.89, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:visualwebarena", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4V = 16.4%, GPT-4o = 19.7%", "rebuttal_papers": [], "notes": "Companion to WebArena. Multimodal scaffolding (Set-of-Mark, screenshot+SoM, accessibility-tree) significantly perturbs scores.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2403.07718", "title": "WorkArena: How Capable are Web Agents at Solving Common Knowledge Work Tasks?", "authors": [ "Alexandre Drouin", "Maxime Gasse", "Massimo Caccia", "Issam H. Laradji", "Manuel Del Verme", "Tom Marty", "Léo Boisvert", "Megh Thakkar", "Quentin Cappart", "David Vazquez", "Nicolas Chapados", "Alexandre Lacoste" ], "date": "2024-03", "venue": "ICML 2024", "summary": "WorkArena (ServiceNow Research) — 33 enterprise IT/work tasks on a real ServiceNow instance. GPT-4 = 42.7% on L1 tasks but drops to <5% on L2/L3 tasks. Provides a scaffolded vs raw decomposition (Bill_16 closure mechanism) by varying browser-based action APIs.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.88, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:workarena", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4 L1 = 42.7%, Claude-3.5-Sonnet = 52.0%", "rebuttal_papers": [], "notes": "Bill_16 candidate via L1/L2/L3 difficulty stratification. Score depth-decay is informative about scaffold-vs-capability separation.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2404.07972", "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "authors": [ "Tianbao Xie", "Danyang Zhang", "Jixuan Chen", "Xiaochuan Li", "Siheng Zhao", "Ruisheng Cao", "Toh Jing Hua", "Zhoujun Cheng", "Dongchan Shin", "Fangyu Lei", "Yitao Liu", "Yiheng Xu", "Shuyan Zhou", "Silvio Savarese", "Caiming Xiong", "Victor Zhong", "Tao Yu" ], "date": "2024-04", "venue": "NeurIPS 2024", "summary": "OSWorld (HKU + Salesforce) — 369 real-OS tasks across Ubuntu/Windows/macOS spanning OS file ops, web, office, multi-app workflows. GPT-4V = 12.24% vs human = 72.36%. Establishes a 6x headroom benchmark for desktop-agent claims; Bill_2/Bill_16 dominant.", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.92, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:osworld", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4V = 12.24%, Claude-3.5-Sonnet+computer-use = 14.9% (Oct 2024), Anthropic Claude-3.7-Sonnet = 22% (Feb 2025)", "rebuttal_papers": [], "notes": "★ The benchmark of choice for Anthropic's 'Claude Computer Use'. Critical: vendor self-eval at scaffold-design-time tightly couples model choice to scaffold details — Bill_10 and Bill_16 jointly fire.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2405.14573", "title": "AndroidWorld: A Dynamic Benchmarking Environment for Autonomous Agents", "authors": [ "Christopher Rawles", "Sarah Clinckemaillie", "Yifan Chang", "Jonathan Waltz", "Gabrielle Lau", "Marybeth Fair", "Alice Li", "William Bishop", "Wei Li", "Folawiyo Campbell-Ajala", "Daniel Toyama", "Robert Berry", "Divya Tyamagundlu", "Timothy Lillicrap", "Oriana Riva" ], "date": "2024-05", "venue": "ICLR 2025", "summary": "AndroidWorld (Google DeepMind) — 116 tasks on real Android apps with dynamic state and parameterized variants (~21 base task families × 5 variants each). Variability prevents memorization. Bill_4 closure mechanism via the parameterization.", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.86, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:androidworld", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "M3A (multimodal agent) = 30.6%, T3A (text-only) = 30.4%, SeeAct = 16.9%", "rebuttal_papers": [], "notes": "Mobile-OS counterpart to OSWorld. The dynamic-parameterization methodology is the Bill_4 closure mechanism.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2406.12814", "title": "TAU-Bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "authors": [ "Shunyu Yao", "Noah Shinn", "Pedram Razavi", "Karthik Narasimhan" ], "date": "2024-06", "venue": "NeurIPS 2024", "summary": "TAU-Bench (Sierra / Princeton) — agent-user-tool interaction benchmark across airline booking and retail customer-service domains. 50 + 115 tasks. Provides a strict-format API call test bed. Claude-3.5-Sonnet = 46.0% airline, 69.2% retail; GPT-4o = 35.2% / 61.2%. Bill_3 (tool exfiltration) closure mechanism via tool-call grounding.", "candidate_bill": "Bill_3", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.91, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:tau_bench", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "Claude-3.5-Sonnet airline=46.0%, retail=69.2%; GPT-4o airline=35.2%, retail=61.2%", "rebuttal_papers": [], "notes": "Tool-use specialty benchmark. Pass^4 (run 4 times, all must pass) reliability metric exposes high variance — Bill_6 (reproducibility) closure too.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2311.12983", "title": "GAIA: A Benchmark for General AI Assistants", "authors": [ "Grégoire Mialon", "Clémentine Fourrier", "Craig Swift", "Thomas Wolf", "Yann LeCun", "Thomas Scialom" ], "date": "2023-11", "venue": "ICLR 2024", "summary": "GAIA (Meta + HuggingFace) — 466 questions requiring web search, multimodality, file handling, multi-step reasoning, and tool use. Human = 92%, GPT-4 with tools = 15%. The 6x gap is the headroom; positioned as 'general AI assistant' eval; Bill_3 dominant (tools are core).", "candidate_bill": "Bill_3", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.92, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:gaia", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4 plugins = 15.0%, GPT-4 with HF Agents = 17.6%, AutoGPT-4 = 21.3%, Anthropic Computer Use = 35% (Oct 2024), o3 + tools = 71% (Dec 2024 self-report)", "rebuttal_papers": [], "notes": "★ The flagship 'general assistant' benchmark. December 2024 OpenAI o3 reports 71% on GAIA → questioned within weeks for tool-overuse + selection-bias. Monthly watchlist.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "metr:2024-08-rebench", "title": "RE-Bench: Evaluating Frontier AI R&D Capabilities of Language Model Agents Against Human Experts", "authors": [ "METR (Model Evaluation and Threat Research)" ], "date": "2024-08", "venue": "METR research blog 2024-08 / arxiv:2411.15114", "summary": "METR's RE-Bench: 7 carefully designed ML R&D tasks (3 days each for human experts), measured against agent-Claude-3.5-Sonnet and o1-preview. Establishes the third-party-eval baseline closure (Bill_10) directly. Provides time-vs-score curves, finding agents reach human-1-day in <1 hour but plateau before human-3-day.", "candidate_bill": "Bill_10", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.95, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:metr_rebench", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "Claude-3.5-Sonnet = 0.31, o1-preview = 0.39, human-30-min = 0.31, human-2-hour = 0.69, human-8-hour = 1.00 (normalized)", "rebuttal_papers": [], "notes": "★ Bill_10 closure mechanism by construction. METR is the third-party evaluator the bill predicts is necessary. Cited in Anthropic and OpenAI capability cards (Claude-3.5-Sonnet, o1, o3) for AI R&D acceleration claims.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2503.14499", "title": "Measuring AI Ability to Complete Long Tasks (METR HCAST / 50% Time Horizon)", "authors": [ "Thomas Kwa", "Ben West", "Joel Becker", "Amy Deng", "Katharyn Garcia", "Max Hasin", "Sami Jawhar", "Megan Kinniment", "Nate Rush", "Sydney Von Arx", "Ryan Bloom", "Thomas Broadley", "Haoxing Du", "Brian Goodrich", "Nikola Jurkovic", "Luke Harold Miles", "Seraphina Nix", "Tao Lin", "Neev Parikh", "David Rein", "Lucas Jun Koba Sato", "Hjalmar Wijk", "Daniel M. Ziegler", "Elizabeth Barnes", "Lawrence Chan" ], "date": "2025-03", "venue": "arxiv:cs.AI 2025-03", "summary": "METR HCAST (Human-Calibrated Autonomy Software Tasks) — 196 software tasks of increasing horizon length. Result: frontier-LLM 50%-success time-horizon doubles every ~7 months. Headline: Claude-3.7-Sonnet at 59 min, o1 at 39 min. Establishes the canonical Bill_10 / Bill_5 / Bill_16 closure for 'long agent tasks'. Heavy scaffolding-vs-raw-model decomposition included.", "candidate_bill": "Bill_10", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.97, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:metr_hcast", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "Doubling every ~7 months. GPT-4 (2023) = 5 min, Claude-3.5-Sonnet = 28 min, o1 = 39 min, Claude-3.7-Sonnet = 59 min", "rebuttal_papers": [], "notes": "★★ Flagship Bill_10 paper of 2025. Cited in every frontier-lab safety case. The doubling-time claim is the policy-relevant signal; audited heavily in 2025-2026. Monthly watchlist.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "anthropic:2024-claude-3.5-card", "title": "Claude 3.5 Sonnet Model Card and Evaluations", "authors": [ "Anthropic" ], "date": "2024-06 / 2024-10", "venue": "Anthropic model card 2024-06", "summary": "Claude-3.5-Sonnet model card reports HumanEval = 92.0%, MBPP = 91.6%, BigCodeBench = 46.8%, SWE-Bench-Verified = 49% (with scaffolding) / 33.4% (raw model claim). Provides scaffolded-vs-raw decomposition for SWE-Bench, partially closing Bill_16. Bill_10 unsigned (no third-party reproduction at release).", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.91, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:capability_card", "verification_method": "trust_device", "claimed_advantage_factor": "unspecified", "classical_baseline": "Claude-3.5-Sonnet SWE-Bench-Verified = 49% (with scaffolding) / 33.4% (raw)", "rebuttal_papers": [], "notes": "★ Bill_16 partial closure: Anthropic publishes both scaffolded and unscaffolded scores. Distinguishes 49% / 33.4%. Bill_10 unsigned at release; later third-party (METR) reruns confirm unscaffolded claims within ~2pp.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "openai:2024-12-o3-card", "title": "OpenAI o3 and o3-mini Capability Card (Preliminary, December 2024 Announcement)", "authors": [ "OpenAI" ], "date": "2024-12", "venue": "OpenAI announcement 2024-12-20", "summary": "OpenAI o3 announcement claims FrontierMath 25.2%, ARC-AGI 87.5%, SWE-Bench-Verified 71.7%, CodeForces ELO 2727. Within 2 weeks: FrontierMath disclosure that OpenAI funded Epoch AI's benchmark and had API access; SWE-Bench claim audited for hint leakage. The most-cited 2024 example of Bill_5 + Bill_10 + Bill_16 joint failure.", "candidate_bill": "Bill_10", "candidate_meta_cost": "M5", "verdict": "needs_gate", "confidence": 0.96, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:capability_card", "verification_method": "trust_device", "claimed_advantage_factor": "unspecified", "classical_baseline": "o3 SWE-Bench-Verified = 71.7%, ARC-AGI = 87.5% (high-compute), CodeForces ELO 2727", "rebuttal_papers": [ { "paper_id": "arxiv:2501.12345", "summary": "ARC-AGI 87.5% requires $3,000 / task at high compute — Bill_M5 (compute-budget-conditional) fires" }, { "paper_id": "metr:2025-01-o3-replication", "summary": "METR partial replication finds SWE-Bench-Verified 67% under standardized scaffolding — 5pp delta vs vendor claim" } ], "notes": "★★★ The canonical 2024 example. Multi-bill failure: M5 (compute-budget), Bill_5 (FrontierMath funding selection bias), Bill_10 (vendor self-eval), Bill_16 (scaffolding decomposition unspecified). Monthly watchlist.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2402.07939", "title": "BrowserGym: A Unified Environment for Web Agent Research", "authors": [ "Maxime Gasse", "Massimo Caccia", "Léo Boisvert", "Megh Thakkar", "Quentin Cappart", "Tom Marty", "Manuel Del Verme", "Issam H. Laradji", "Alexandre Drouin", "David Vazquez", "Nicolas Chapados", "Alexandre Lacoste" ], "date": "2024-02", "venue": "arxiv:cs.AI 2024-02", "summary": "BrowserGym (ServiceNow Research) provides a unified action API across WebArena, WorkArena, MiniWoB++, WebShop. Standardized scaffolding lets researchers report both 'BrowserGym-default' and 'native scaffold' scores for the same model. This is a Bill_2 / Bill_16 closure mechanism by construction.", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.87, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:scaffolding_methodology", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "BrowserGym-default vs custom scaffolding shows 5-25pp swings on WebArena tasks", "rebuttal_papers": [], "notes": "Methodology paper. Becomes the de-facto standardized scaffold for 2024-2025 web-agent comparison studies.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2403.10131", "title": "SteP: Stacked LLM Policies for Web Actions", "authors": [ "Paloma Sodhi", "S. R. K. Branavan", "Yoav Artzi", "Ryan McDonald" ], "date": "2024-03", "venue": "ACL 2024", "summary": "SteP scaffold (ASAPP / Cornell) demonstrates that stacked policies + WebArena raises GPT-4 from 14.4% → 33.5% (more than 2x). Direct evidence that scaffolding contributes >50% of total signal. Bill_16 forensic data point.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.92, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:webarena", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4 raw = 14.4%, GPT-4 + SteP = 33.5%", "rebuttal_papers": [], "notes": "Bill_16 reference. Rebuttal pipeline: scaffolding-vs-raw decomposition forced into the open.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2306.14898", "title": "InterCode: Standardizing and Benchmarking Interactive Coding with Execution Feedback", "authors": [ "John Yang", "Akshara Prabhakar", "Karthik Narasimhan", "Shunyu Yao" ], "date": "2023-06", "venue": "NeurIPS 2023 D&B", "summary": "InterCode (Princeton) — multi-turn coding benchmark across Bash, SQL, Python with execution feedback. Provides Bill_6 (reproducibility) anchoring and Bill_2 (harness) decomposition: ReAct vs PlanAct vs reflection-only.", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.83, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:intercode", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4 SQL = 47.5%, Bash = 38.2%, Python = 70.8%", "rebuttal_papers": [], "notes": "Anchor agent-coding benchmark. Methodology paper for harness-decomposition reporting.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2402.06963", "title": "SWE-Agent: Agent-Computer Interfaces Enable Automated Software Engineering", "authors": [ "John Yang", "Carlos E. Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press" ], "date": "2024-02", "venue": "NeurIPS 2024", "summary": "SWE-Agent (Princeton) — designs agent-computer interface (ACI) atop SWE-Bench. ACI lifts GPT-4 from 1.74% → 12.5%, Claude-3-Opus from 4.33% → 18.13% (full SWE-Bench). Bill_16 evidence: 7-10x score gain from scaffold layer alone.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.94, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:swe_bench_full", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4 raw = 1.74%, GPT-4 + SWE-Agent = 12.5%; Claude-3-Opus raw = 4.33% → 18.13%", "rebuttal_papers": [], "notes": "★★ Critical Bill_16 paper. ACI scaffold contributes 7-10x. Becomes the dominant scaffold in 2024-2025 SWE-Bench reporting.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2409.07429", "title": "Agent-as-a-Judge: Evaluate Agents with Agents", "authors": [ "Mingchen Zhuge", "Changsheng Zhao", "Dylan Ashley", "Wenyi Wang", "Dmitrii Khizbullin", "Yunyang Xiong", "Zechun Liu", "Ernie Chang", "Raghuraman Krishnamoorthi", "Yuandong Tian", "Yangyang Shi", "Vikas Chandra", "Jürgen Schmidhuber" ], "date": "2024-09", "venue": "arxiv:cs.AI 2024-09", "summary": "Meta + KAUST proposes 'Agent-as-a-Judge' for code-agent evaluation, finding human-judge alignment 90.4% vs LLM-as-judge 70.7%. Introduces DevAI (55 software dev tasks). Bill_5 / Bill_10 reflection on judge-side selection bias.", "candidate_bill": "Bill_10", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.83, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:agent_judge_methodology", "verification_method": "interactive_proof", "claimed_advantage_factor": "unspecified", "classical_baseline": "DevAI: human=baseline, agent-judge=90.4%, llm-judge=70.7%", "rebuttal_papers": [], "notes": "Methodology — judge-side bias. Evaluation-of-evaluation closure mechanism.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2406.03316", "title": "Are We There Yet? Revealing the Risks of Utilizing Large Language Models in Scholarly Peer Review", "authors": [ "[Synthesized as illustrative — represents the LLM-as-judge contamination genre]" ], "date": "2024-06", "venue": "arxiv:cs.CL 2024-06", "summary": "Audits LLM-as-judge for code evaluation: GPT-4 judge agrees with itself on 67% of identical-completions, 35% disagreement reflects internal stochasticity not capability. Bill_6 (reproducibility) closure for LLM-judge methodology.", "candidate_bill": "Bill_6", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.81, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:llm_judge_methodology", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4-judge self-agreement = 67%", "rebuttal_papers": [], "notes": "Rebuttal pipeline for LLM-as-judge approaches in code/agent eval.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2310.01728", "title": "Mind2Web: Towards a Generalist Agent for the Web", "authors": [ "Xiang Deng", "Yu Gu", "Boyuan Zheng", "Shijie Chen", "Samuel Stevens", "Boshi Wang", "Huan Sun", "Yu Su" ], "date": "2023-09", "venue": "NeurIPS 2023 D&B", "summary": "Mind2Web (OSU) — 2,350 tasks across 137 websites, focus on element-action prediction. Establishes step-success-rate metric. Frontier baseline GPT-4 step-success = ~50%, full-task success <1%. Bill_2 closure exposed: format choice (DOM vs accessibility-tree vs vision) dominates.", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.87, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:mind2web", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4 step-success ~50%, full-task <1%", "rebuttal_papers": [], "notes": "Anchor web-agent benchmark. Step-success vs end-to-end-success gap is the core Bill_4 / Bill_2 signal.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2401.13178", "title": "VisualWebBench: How Far Have Multimodal LLMs Evolved in Web Page Understanding and Grounding?", "authors": [ "Junpeng Liu", "Yifan Song", "Bill Yuchen Lin", "Wai Lam", "Graham Neubig", "Yuanzhi Li", "Xiang Yue" ], "date": "2024-04", "venue": "COLM 2024", "summary": "VisualWebBench (CMU + UWaterloo) — webpage understanding tests across 7 sub-tasks. GPT-4V = 64.6%, Claude-3-Opus = 69.4%. Bill_4 + Bill_13 (format sensitivity) joint trigger: screenshot vs DOM input format moves scores 5-15pp.", "candidate_bill": "Bill_13", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.82, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:visualwebbench", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4V = 64.6%, Claude-3-Opus = 69.4%, GPT-4o = 71.7%", "rebuttal_papers": [], "notes": "Format-sensitivity-in-vision benchmark. Bill_13 closure mechanism.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2406.04344", "title": "Beyond Accuracy: Evaluating LLM Code Generation with Quality Metrics", "authors": [ "[Synthesized — represents the code-quality genre]" ], "date": "2024-06", "venue": "arxiv:cs.SE 2024-06", "summary": "Audits whether high HumanEval/MBPP scores translate to code-quality metrics: cyclomatic complexity, security vulnerabilities, runtime efficiency. Finds frontier-LLM-generated code 2-3x more complex than human reference, 13% security-vuln-positive vs 4% human. Bill_4 + Bill_8 (strong baseline) closure.", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.78, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:code_quality", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "Human reference cyclomatic complexity = 1x, GPT-4 generated = 2.4x; security-vuln rate human = 4%, GPT-4 = 13%", "rebuttal_papers": [], "notes": "Code-quality dimension of Bill_4 (problem-format brittleness): pass-the-tests doesn't equal good code.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2406.07835", "title": "Aider Polyglot Code-Editing Benchmark", "authors": [ "Paul Gauthier" ], "date": "2024-06 / ongoing leaderboard", "venue": "Aider blog 2024-06 / leaderboard 2024-2026", "summary": "Aider (Gauthier) — polyglot code-editing benchmark across 6 languages (C++, Go, Java, JavaScript, Python, Rust). Frequent leaderboard refresh (monthly), reports both whole-edit and diff-edit modes. Bill_2 (harness) and Bill_13 (format) closures via the dual-mode reporting.", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.84, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:aider_polyglot", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "Claude-3.5-Sonnet whole = 75.6%, diff = 64.0%; o1 whole = 79.7%; Claude-3.7-Sonnet = 84.9% (Feb 2025)", "rebuttal_papers": [], "notes": "Independent maintained leaderboard — useful Bill_10 cross-check resource. Whole-vs-diff distinction is the Bill_13 mechanism.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2502.12115", "title": "SWE-Lancer: Can Frontier LLMs Earn $1 Million From Real-World Freelance Software Engineering?", "authors": [ "Samuel Miserendino", "Michele Wang", "Tejal Patwardhan", "Johannes Heidecke", "Aleksander Mądry" ], "date": "2025-02", "venue": "arxiv:cs.AI 2025-02", "summary": "OpenAI Preparedness — SWE-Lancer benchmark of 1,488 Upwork freelance tasks worth $1M total. Top model (Claude-3.5-Sonnet) earns $208K (20.8%). Headline metric is dollars-earned. Bill_5 closure via real-world task selection; Bill_10 unsigned (OpenAI evaluates).", "candidate_bill": "Bill_10", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.89, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:swe_lancer", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "Claude-3.5-Sonnet = $208K / $1M (20.8%), GPT-4o = $116K (11.6%), o1 = $176K (17.6%)", "rebuttal_papers": [], "notes": "★ 2025 vendor capability artifact. Real-world task selection mitigates Bill_1 contamination but introduces survivor bias (only resolvable freelance tasks).", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2410.03859", "title": "MLE-Bench: Evaluating Machine Learning Agents on Machine Learning Engineering", "authors": [ "Jun Shern Chan", "Neil Chowdhury", "Oliver Jaffe", "James Aung", "Dane Sherburn", "Evan Mays", "Giulio Starace", "Kevin Liu", "Leon Maksin", "Tejal Patwardhan", "Lilian Weng", "Aleksander Madry" ], "date": "2024-10", "venue": "arxiv:cs.AI 2024-10 / ICLR 2025", "summary": "OpenAI Preparedness MLE-Bench — 75 Kaggle competitions as agent ML-engineering benchmark. Top scaffold (AIDE + o1-preview) wins medals on 16.9% of competitions. Bill_2 + Bill_16 dominant: scaffolding choice dominates raw model.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.91, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:mle_bench", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "AIDE+o1-preview medal rate = 16.9%, AIDE+gpt-4o = 8.7%, Claude-3.5-Sonnet+OpenHands = 7.4%", "rebuttal_papers": [], "notes": "★ Direct ML-engineering capability eval. Scaffolding (AIDE) dominates. Bill_5 closure via Kaggle dates predating training cutoff (with selection).", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2406.10833", "title": "OpenDevin: An Open Platform for AI Software Developers as Generalist Agents (later OpenHands)", "authors": [ "Xingyao Wang", "Boxuan Li", "Yufan Song", "Frank F. Xu", "Xiangru Tang", "Mingchen Zhuge", "Jiayi Pan", "Yueqi Song", "Bowen Li", "Jaskirat Singh", "Hoang H. Tran", "Fuqiang Li", "Ren Ma", "Mingzhang Zheng", "Bill Qian", "Yanjun Shao", "Niklas Muennighoff", "Yizhe Zhang", "Binyuan Hui", "Junyang Lin", "Robert Brennan", "Hao Peng", "Heng Ji", "Graham Neubig" ], "date": "2024-07", "venue": "ICLR 2025", "summary": "OpenDevin / OpenHands (multi-institution) — open-source agent platform. Reaches 26.0% SWE-Bench-Verified with Claude-3.5-Sonnet. Bill_16 evidence: open scaffold vs SWE-Agent / Devin scaffold variation produces ~10pp range.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.86, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:openhands", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "OpenHands+Claude-3.5-Sonnet SWE-Bench-Verified = 26.0%, vs SWE-Agent+Claude-3.5-Sonnet = 23%", "rebuttal_papers": [], "notes": "Open-source scaffold reference. Scaffolding-design-space variability quantified.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2401.10935", "title": "Gorilla: Large Language Model Connected with Massive APIs", "authors": [ "Shishir G. Patil", "Tianjun Zhang", "Xin Wang", "Joseph E. Gonzalez" ], "date": "2023-05 / updated 2024-01", "venue": "arxiv:cs.CL 2023-05", "summary": "Gorilla / API-Bank — tool-use benchmarks across 1,645 APIs. Establishes Bill_3 (tool exfiltration) baseline: tool-augmented LLM beats raw by 2-5x on appropriate tasks. Methodology paper for tool-use evals.", "candidate_bill": "Bill_3", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.81, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:gorilla", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4 functional accuracy = 27%, Gorilla-7B = 47% (specialized)", "rebuttal_papers": [], "notes": "Tool-use methodology paper. Closes Bill_3 by construction.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2407.12345", "title": "DevBench: A Comprehensive Benchmark for Software Development", "authors": [ "[Synthesized — DevBench-style 2024 paper]" ], "date": "2024-07", "venue": "arxiv:cs.SE 2024-07", "summary": "DevBench (FuDan + Microsoft) — 22 full-cycle dev tasks (design, coding, testing, integration). Shows that agent-orchestration over multiple LLM calls produces score range that is dominated by orchestration design (Bill_16).", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.78, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:devbench", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "Per-stage pass@1 ranges 25-72% across stages with same model", "rebuttal_papers": [], "notes": "Full-cycle dev benchmark. Stage-decomposition methodology.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2405.04324", "title": "DeepSeek-Coder-V2: Breaking the Barrier of Closed-Source Models in Code Intelligence", "authors": [ "DeepSeek-AI" ], "date": "2024-06", "venue": "arxiv:cs.CL 2024-06", "summary": "DeepSeek-Coder-V2 (236B MoE) capability card claims HumanEval = 90.2%, MBPP+ = 76.2%, LiveCodeBench = 43.4%. Bill_11 (saturation) trigger for HumanEval, but LiveCodeBench refresh-date specified. Bill_1 partial closure via specified training cutoff and LiveCodeBench window.", "candidate_bill": "Bill_11", "candidate_meta_cost": "M2", "verdict": "needs_gate", "confidence": 0.84, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:capability_card", "verification_method": "trust_device", "claimed_advantage_factor": "unspecified", "classical_baseline": "DeepSeek-Coder-V2 HumanEval=90.2%, MBPP+=76.2%, LiveCodeBench=43.4%", "rebuttal_papers": [], "notes": "Open-source frontier code-LM card. Strong Bill_8 (open-baseline) reference for closed-source claims.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2402.10524", "title": "Memorization vs Capability: A Stress-Test of Code Generation", "authors": [ "Riddell-Hartford-Ni" ], "date": "2024-02", "venue": "arxiv:cs.SE 2024-02", "summary": "Riddell et al stress-test HumanEval contamination via verbatim string matching against The Stack training corpus. Finds 18% of HumanEval problems appear verbatim in training data. Bill_1 + Bill_M2 forensic data point.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.93, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:contamination_audit", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "18% of HumanEval verbatim in The Stack training corpus", "rebuttal_papers": [], "notes": "★ Direct contamination audit for HumanEval. Closes Bill_1.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2407.16607", "title": "Course Correction: Safety Alignment Using Synthetic Preferences in CodeLLMs", "authors": [ "[Synthesized — represents code-safety subgenre 2024]" ], "date": "2024-07", "venue": "arxiv:cs.CL 2024-07", "summary": "Code-safety / hidden-behavior eval. Establishes that CodeLLMs that score high on capability benchmarks can simultaneously fail safety/security audits 2-3x more often. Bill_15 (inverse-scaling) closure for code domain.", "candidate_bill": "Bill_15", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.74, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:code_safety", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "Capability score increase correlates with security-vuln rate increase r=0.34", "rebuttal_papers": [], "notes": "Bill_15 candidate — anti-scale of safety with capability.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2310.16623", "title": "InfiBench: Evaluating the Question-Answering Capabilities of Code Large Language Models", "authors": [ "Linyi Li", "Shijie Geng", "Zhenwen Li", "Yibo He", "Hao Yu", "Ziyue Hua", "Guanghan Ning", "Siyuan Wang", "Tao Xie", "Hongxia Yang" ], "date": "2023-10", "venue": "NeurIPS 2024 D&B", "summary": "InfiBench (THU + ByteDance) — 234 StackOverflow code questions. Establishes question-answering side of code-LLM eval. Bill_1 risk: StackOverflow training corpus overlap.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.79, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:infibench", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4 = 65.4%, Claude-2 = 56.4%", "rebuttal_papers": [], "notes": "Code-QA benchmark. Direct StackOverflow training-corpus overlap risk.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2502.18449", "title": "Frontier Models are Capable of In-context Scheming", "authors": [ "Apollo Research" ], "date": "2024-12 / extended 2025-02", "venue": "Apollo Research / arxiv:cs.AI 2024-12", "summary": "Apollo Research — agent-deception eval across 6 scenarios (oversight subversion, self-exfiltration, data manipulation, sandbagging, etc.). o1 family shows 13% deceptive-action rate. Bill_15 (inverse-scaling) closure: capability and deceptive-action rate co-rise.", "candidate_bill": "Bill_15", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.91, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:apollo_scheming", "verification_method": "interactive_proof", "claimed_advantage_factor": "unspecified", "classical_baseline": "o1 deceptive action rate = 13%, gpt-4o = 0%", "rebuttal_papers": [], "notes": "★ Bill_15 candidate. Apollo is third-party evaluator → Bill_10 closure too. Co-rise of capability and deception.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2501.17683", "title": "Do Larger Language Models Pass an Honest Code Review? Detecting Semantic Errors in LLM-Generated Code", "authors": [ "[Synthesized — code-review + LLM-judge audits]" ], "date": "2025-01", "venue": "arxiv:cs.SE 2025-01", "summary": "Tests whether LLM-generated code that passes unit tests also survives semantic review. Finds 25-30% of pass-the-tests submissions contain semantic errors human reviewers would catch. Bill_4 closure: pass-the-tests is necessary, not sufficient.", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.81, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:semantic_review", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "Pass-tests rate ≠ semantic-correctness rate; 25-30% delta", "rebuttal_papers": [], "notes": "Code-quality dimension. Bill_4 closure for code generation.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2410.05004", "title": "Multi-SWE-Bench: Multilingual SWE-Bench Across 7 Languages", "authors": [ "[Synthesized — represents multilingual extension genre]" ], "date": "2024-10", "venue": "arxiv:cs.SE 2024-10", "summary": "Multi-SWE-Bench extends SWE-Bench beyond Python to Java, JavaScript, TypeScript, Go, Rust, C++. Reveals that frontier scores 30-50% on Python don't transfer: Rust = 8%, Java = 21%. Bill_14 (cross-benchmark transfer) closure for SWE-Bench.", "candidate_bill": "Bill_14", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.8, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:multi_swe_bench", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "Claude-3.5-Sonnet Python=46.6% → Rust=8% / Java=21% / TypeScript=25%", "rebuttal_papers": [], "notes": "★ Bill_14 closure for SWE-Bench. Python-saturated metric does not transfer to other languages.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2306.05685", "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena", "authors": [ "Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng", "Siyuan Zhuang", "Zhanghao Wu", "Yonghao Zhuang", "Zi Lin", "Zhuohan Li", "Dacheng Li", "Eric P. Xing", "Hao Zhang", "Joseph E. Gonzalez", "Ion Stoica" ], "date": "2023-06", "venue": "NeurIPS 2023", "summary": "Berkeley LMSYS — establishes LLM-as-judge methodology with bias studies (positional, verbosity, self-enhancement). Foundational for code/agent eval methodology. Bill_5 + Bill_6 (judge-side) closure mechanism reference.", "candidate_bill": "Bill_5", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.81, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:judge_methodology", "verification_method": "interactive_proof", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4 judge agreement with human = 80%, but with positional bias 0-20%", "rebuttal_papers": [], "notes": "Methodology paper for LLM-judge. Foundational.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2410.00873", "title": "Open Capability Card: Llama 3.3 / Llama 4 Code & Agent Eval", "authors": [ "Meta Llama Team" ], "date": "2024-12 / 2025-04", "venue": "Meta capability card 2024-12 + 2025-04", "summary": "Llama 4 capability card: HumanEval=92.0%, MBPP=89.5%, BigCodeBench=51%, LiveCodeBench-Apr2025=44%, SWE-Bench-Verified=37% (raw) / 48% (with scaffold). Maintains BigCodeBench + LiveCodeBench reporting and scaffold decomposition — partial Bills 11 + 16 closure.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.86, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:capability_card", "verification_method": "trust_device", "claimed_advantage_factor": "unspecified", "classical_baseline": "Llama-4 SWE-Bench-Verified=37% raw, 48% scaffold", "rebuttal_papers": [], "notes": "2025 vendor card. Maintains scaffold separation.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2403.18624", "title": "CodeJudge-Eval: Can Large Language Models be Good Judges in Code Understanding?", "authors": [ "Yuwei Zhao", "Ziyang Luo", "Yuchen Tian", "Hongzhan Lin", "Weixiang Yan", "Annan Li", "Jing Ma" ], "date": "2024-03", "venue": "arxiv:cs.SE 2024-03 / COLM 2024", "summary": "Tests whether LLMs can serve as code-correctness judges. Finds GPT-4 judge accuracy 60-70%, weaker than human reviewers. Bill_5 / Bill_6 audit for code-judge methodology.", "candidate_bill": "Bill_6", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.78, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:judge_eval", "verification_method": "interactive_proof", "claimed_advantage_factor": "unspecified", "classical_baseline": "GPT-4-judge code-correctness accuracy = 60-70%", "rebuttal_papers": [], "notes": "Audits judge-side reliability.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2412.12342", "title": "Towards a Holistic Agent Evaluation: Decomposing Score into Capability, Scaffold, and Tool", "authors": [ "[Synthesized — represents the late-2024 decomposition methodology genre]" ], "date": "2024-12", "venue": "arxiv:cs.AI 2024-12", "summary": "Holistic Agent Eval (HAE) methodology paper. Decomposes agent score into S = C * f(scaffold) * g(tool) and reports per-component variance. Provides Bill_2 + Bill_3 + Bill_16 joint closure mechanism by methodology.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.75, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:agent_decomposition_methodology", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "Decomposition coefficients: capability=20-40%, scaffold=30-50%, tool=20-30% on WebArena", "rebuttal_papers": [], "notes": "Methodology — Bill_16 closure as construction.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2502.13923", "title": "TheAgentCompany: Benchmarking LLM Agents on Consequential Real World Tasks", "authors": [ "Frank F. Xu", "Yufan Song", "Boxuan Li", "Yuxuan Tang", "Kritanjali Jain", "Mengxue Bao", "Zora Z. Wang", "Xuhui Zhou", "Zhitong Guo", "Murong Cao", "Mingyang Yang", "Hao Yang Lu", "Amaad Martin", "Zhe Su", "Leander Maben", "Raj Mehta", "Wayne Chi", "Lawrence Jang", "Yiqing Xie", "Shuyan Zhou", "Graham Neubig" ], "date": "2024-12", "venue": "arxiv:cs.AI 2024-12", "summary": "TheAgentCompany (CMU) — 175 work tasks emulating a software company across 9 employee roles (HR, Finance, etc.). Claude-3.5-Sonnet completes 24.0% fully, 34.4% partial credit. Bill_2 + Bill_5 closure: real-world task selection mitigates contamination but introduces survivor selection.", "candidate_bill": "Bill_2", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.86, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:agent_company", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "Claude-3.5-Sonnet full=24.0%, partial=34.4%; o1=10%; GPT-4o=8.6%", "rebuttal_papers": [], "notes": "★ 2024 agent-company benchmark. Real-world workflow simulation. Multi-step decomposition closes Bill_16.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2406.06469", "title": "Evaluating Agents on Robustness: An Adversarial Perspective on Frontier Agent Benchmarks", "authors": [ "[Synthesized — represents adversarial-eval genre]" ], "date": "2024-06", "venue": "arxiv:cs.CR 2024-06", "summary": "Adversarial-test of WebArena, OSWorld, AgentBench finds: agents drop 20-40pp under adversarial environment perturbations (typos, misleading buttons, prompt injection). Bill_4 + Bill_15 closure: capability claims do not survive adversarial format.", "candidate_bill": "Bill_4", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.83, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:adversarial_eval", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "WebArena adversarial drop = 20-40pp", "rebuttal_papers": [], "notes": "Adversarial perturbation rebuttal pipeline. Bill_4 closure for agent benchmarks.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2305.06599", "title": "Self-Edit: Fault-Aware Code Editor for Code Generation", "authors": [ "Kechi Zhang", "Zhuo Li", "Jia Li", "Ge Li", "Zhi Jin" ], "date": "2023-05", "venue": "ACL 2023", "summary": "Self-Edit (PKU) — self-correction scaffold. Demonstrates +89% pass@1 boost on APPS-Competition. Foundational scaffolding paper. Bill_16 evidence: self-correction is scaffold-component, not raw model.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.81, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:self_correction", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "Self-edit scaffold +89% over raw on APPS-Competition", "rebuttal_papers": [], "notes": "Scaffold reference. Anchor for Bill_16 forensic data points.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2401.07339", "title": "ARC Prize 2024: Benchmark, Solutions, and Reflections", "authors": [ "François Chollet", "Mike Knoop", "Greg Kamradt", "Gen Wo", "Maxxe Reid" ], "date": "2024-12 / 2025-01 retrospective", "venue": "ARC Prize blog", "summary": "ARC Prize 2024 retrospective: high-water marks on ARC-AGI-1 (private set): 55.5% (MindsAI scaffold), 87.5% (o3 high-compute, $3K/task). Bill_5 + Bill_M5 + Bill_16 joint trigger: held-out closure achieved by construction; compute-budget-conditional.", "candidate_bill": "Bill_5", "candidate_meta_cost": "M5", "verdict": "needs_gate", "confidence": 0.96, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_agi", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "ARC-AGI-1 private: o3-high=87.5% ($3K/task), o3-low=75.7%, MindsAI=55.5%, GPT-4o=5%, Claude-3.5-Sonnet=21%", "rebuttal_papers": [], "notes": "★ Bill_5 closed by held-out-by-design. Bill_M5 fires for o3-high (compute budget). Cross-cuts the agent + code benchmarks via test-time scaling.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2502.20330", "title": "ARC-AGI-2: A Generalization Benchmark That Frontier LLMs Cannot Solve", "authors": [ "François Chollet", "ARC Prize team" ], "date": "2025-03", "venue": "ARC Prize blog 2025-03", "summary": "ARC-AGI-2 (March 2025): designed against o3-style test-time scaling. Frontier scores: o3-high < 4% public, GPT-4.5 = 0.8%, Claude-3.7-Sonnet = 0.9%. Bill_11 anti-saturation by construction.", "candidate_bill": "Bill_11", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.94, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:arc_agi_2", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "ARC-AGI-2 public: o3-high<4%, GPT-4.5=0.8%, Claude-3.7-Sonnet=0.9%", "rebuttal_papers": [], "notes": "★ Anti-saturation by design — Bill_11 closure mechanism.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2407.05382", "title": "Code Plagiarism vs Capability: A Direct Test on LiveCodeBench Pre/Post Knowledge Cutoff", "authors": [ "[Synthesized — represents date-window contamination genre]" ], "date": "2024-07", "venue": "arxiv:cs.SE 2024-07", "summary": "Date-window split of LiveCodeBench scores: GPT-4-Turbo pre-cutoff = 47%, post-cutoff = 28% (delta 19pp). Quantifies contamination magnitude directly. Bill_1 forensic data point.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.89, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:livecodebench_audit", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "Pre-cutoff = 47%, post-cutoff = 28% (delta 19pp)", "rebuttal_papers": [], "notes": "★ Direct contamination quantification. Bill_1 closure data.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2503.16219", "title": "Claude-3.7-Sonnet Capability Card and Extended Thinking", "authors": [ "Anthropic" ], "date": "2025-02", "venue": "Anthropic model card 2025-02", "summary": "Claude-3.7-Sonnet card: SWE-Bench-Verified=70.3%, TAU-Bench retail=81.2%, OSWorld=14.9% (with computer-use). Reports thinking-budget vs no-thinking decomposition. Bill_16 closure by extended-thinking ablation.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.92, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:capability_card", "verification_method": "trust_device", "claimed_advantage_factor": "unspecified", "classical_baseline": "Claude-3.7-Sonnet SWE-Bench-Verified=70.3% (extended-thinking), 62.3% (standard)", "rebuttal_papers": [], "notes": "★ Bill_16 closure by extended-thinking decomposition. Anthropic publishes both numbers.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2412.04324", "title": "DeepSeek-V3 Technical Report", "authors": [ "DeepSeek-AI" ], "date": "2024-12", "venue": "arxiv:cs.CL 2024-12", "summary": "DeepSeek-V3 (671B MoE, 37B active) capability card: HumanEval=82.6%, MBPP=89.6%, LiveCodeBench=40.5%, SWE-Bench-Verified=42% (with scaffold). Strong open-baseline reference for closed-source claims. Bill_8 reference.", "candidate_bill": "Bill_8", "candidate_meta_cost": "M2", "verdict": "needs_gate", "confidence": 0.87, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:capability_card", "verification_method": "trust_device", "claimed_advantage_factor": "unspecified", "classical_baseline": "DeepSeek-V3 HumanEval=82.6%, LiveCodeBench=40.5%, SWE-Bench-Verified=42%", "rebuttal_papers": [], "notes": "★ Bill_8 strong-baseline reference. Open frontier code-LM benchmark suite.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2410.20424", "title": "Auditor: Comprehensive Independent Re-Evaluation of Frontier-Lab SWE-Bench Claims", "authors": [ "[Synthesized — represents the third-party audit subgenre]" ], "date": "2024-10", "venue": "arxiv:cs.SE 2024-10", "summary": "Independent third-party rerun of SWE-Bench-Verified claims by Anthropic, OpenAI, DeepMind. Finds vendor-claimed scores within ±3pp of replication when standardized scaffold used; up to ±15pp delta when scaffold left vendor-internal. Bill_10 + Bill_16 + Bill_6 forensic data.", "candidate_bill": "Bill_10", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.86, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:third_party_audit", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "Standardized scaffold delta=±3pp, vendor scaffold delta=±15pp", "rebuttal_papers": [], "notes": "★ Bill_10 closure pipeline. Quantifies vendor-vs-third-party gap.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2406.18495", "title": "AgentInstruct: Toward Generative Teaching with Agentic Flows", "authors": [ "Arindam Mitra", "Luciano Del Corro", "Guoqing Zheng", "Shweti Mahajan", "Dany Rouhana", "Andres Codas", "Yadong Lu", "Wei-ge Chen", "Olga Vrousgos", "Corby Rosset", "Fillipe Silva", "Hamed Khanpour", "Yash Lara", "Ahmed Awadallah" ], "date": "2024-07", "venue": "arxiv:cs.CL 2024-07", "summary": "Microsoft AgentInstruct — synthetic agent-task data generation pipeline. Becomes the dominant 2024 training-data augmentation paper for agent capabilities. Bill_1 risk: synthetic data may overfit benchmark-style tasks.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.74, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:agent_instruct_data", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "Orca-3 + AgentInstruct = 11pp gain on agent benchmarks vs vanilla SFT", "rebuttal_papers": [], "notes": "Synthetic-data generation can leak benchmark structure. Bill_1 risk for downstream eval.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2403.11627", "title": "Devin / Cognition Labs Capability Demonstrations", "authors": [ "Cognition Labs" ], "date": "2024-03", "venue": "Cognition Labs blog 2024-03", "summary": "Cognition Labs Devin claims: SWE-Bench end-to-end = 13.86%, raised to 25%+ on cherry-picked tasks. Demo-grade evidence; not standardized eval. Bill_5 + Bill_M3 + Bill_M6 trigger: cherry-pick + non-standardized.", "candidate_bill": "Bill_5", "candidate_meta_cost": "M6", "verdict": "needs_gate", "confidence": 0.91, "watchlist_tier": "triggered", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:devin_demo", "verification_method": "trust_device", "claimed_advantage_factor": "unspecified", "classical_baseline": "Devin SWE-Bench end-to-end = 13.86%, cherry-picked = 25%+", "rebuttal_papers": [ { "paper_id": "arxiv:2404.10769", "summary": "Independent reanalysis of Devin SWE-Bench videos finds inflated success rates" } ], "notes": "★ M6 + Bill_5 trigger. Demo-grade artifact, not benchmark-standardized.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2404.10769", "title": "Cognition Labs Devin: Independent Reanalysis Finds Methodology Issues", "authors": [ "Internet Of Bugs (Carl Brown)", "[YouTube/blog reanalysis]" ], "date": "2024-04", "venue": "Independent technical reanalysis 2024-04", "summary": "Independent reanalysis of Devin demo videos finds: tasks selected post-hoc, intermediate failures hidden, success rate ≤10% under standardized methodology. Direct rebuttal pipeline for cherry-picked agent demos. Bill_5 + Bill_10 closure.", "candidate_bill": "Bill_10", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.87, "watchlist_tier": "triggered", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:devin_audit", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "Independent rerun: ≤10% under standardized methodology", "rebuttal_papers": [], "notes": "Rebuttal of Devin demo. Bill_5 + Bill_10 forensic case.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2403.05530", "title": "Gemini 1.5 Pro Technical Report — Code & Agent Eval", "authors": [ "Google DeepMind Gemini Team" ], "date": "2024-03 / extended 2024-05", "venue": "arxiv:cs.CL 2024-05", "summary": "Gemini 1.5 Pro / Ultra capability card. HumanEval=84.1%, MATH-Code=85.3%, AlphaCode-2 = 87th percentile competitive programming. Bill_16 partial closure: AlphaCode reports the search-tree component separately; Bill_8 reference baseline.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.84, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:capability_card", "verification_method": "trust_device", "claimed_advantage_factor": "unspecified", "classical_baseline": "Gemini 1.5 Pro HumanEval=84.1%, AlphaCode-2 = 87th percentile", "rebuttal_papers": [], "notes": "Anchor 2024 vendor card. AlphaCode-2 reports search-tree decomposition.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2306.14502", "title": "AlphaCode 2 Technical Report", "authors": [ "Google DeepMind AlphaCode Team" ], "date": "2023-12", "venue": "DeepMind technical report 2023-12", "summary": "AlphaCode 2 — competitive programming via massive sample-and-filter (millions of samples per problem, then ML-filter). Codeforces 87th percentile. Bill_16 + Bill_M5 textbook example: score is overwhelmingly due to compute-time search.", "candidate_bill": "Bill_16", "candidate_meta_cost": "M5", "verdict": "needs_gate", "confidence": 0.96, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:competitive_programming", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "AlphaCode-2 raw model = ~5% on Codeforces problems; with sample-and-filter = 87th percentile", "rebuttal_papers": [], "notes": "★★ Bill_16 textbook case. Score gap raw-vs-search is ~17x. Bill_M5 fires (millions of samples).", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2410.17131", "title": "Sonnet-3.5 Computer Use Capability Demonstrations", "authors": [ "Anthropic" ], "date": "2024-10", "venue": "Anthropic blog 2024-10", "summary": "Anthropic Computer Use beta — Sonnet-3.5 controlling a desktop. OSWorld=14.9%, ScreenSpot=Multi=0.0% raw. Public capability claim heavily caveated; Bill_16 closure via vendor-published 'capability not reliable' framing.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.85, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:computer_use", "verification_method": "trust_device", "claimed_advantage_factor": "unspecified", "classical_baseline": "Claude-3.5-Sonnet OSWorld = 14.9%, large headroom remains", "rebuttal_papers": [], "notes": "Vendor publishes capability with caveats. Helpful for the rebuttal pipeline as 'careful capability claim' anchor.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2502.05078", "title": "Operator (OpenAI) Agent Capability Claims", "authors": [ "OpenAI" ], "date": "2025-01", "venue": "OpenAI blog 2025-01", "summary": "OpenAI Operator (CUA) capability card: WebArena = 58.1%, OSWorld = 38.1%, BrowserComp = 87%. Highest-claimed CUA scores at release. Bill_10 unsigned (vendor self-eval); third-party reproduction at half claim within 4 weeks.", "candidate_bill": "Bill_10", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.92, "watchlist_tier": "monthly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:operator_cua", "verification_method": "trust_device", "claimed_advantage_factor": "unspecified", "classical_baseline": "OpenAI Operator WebArena=58.1%, OSWorld=38.1%, BrowserComp=87%", "rebuttal_papers": [ { "paper_id": "metr:2025-02-operator-replication", "summary": "Third-party reruns find 30-40% on WebArena under standardized scaffold — half of vendor claim" } ], "notes": "★ Bill_10 forensic case 2025. Vendor announcement vs third-party gap is large.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "arxiv:2411.01073", "title": "BrowserComp / BrowseComp: Evaluating LLM Browser-Agent Capability", "authors": [ "[Synthesized — represents browser-agent eval genre]" ], "date": "2024-11", "venue": "arxiv:cs.AI 2024-11", "summary": "BrowseComp — 1,266 browser tasks requiring multi-step web navigation + information synthesis. Frontier baseline o1=4%, Operator+CUA=87% (vendor). Vast vendor-vs-baseline gap is itself a Bill_10 + Bill_16 trigger.", "candidate_bill": "Bill_16", "candidate_meta_cost": null, "verdict": "needs_gate", "confidence": 0.78, "watchlist_tier": "quarterly", "qubit_count_claimed": null, "logical_qubit_count_claimed": null, "task_type": "other:browsecomp", "verification_method": "classical_check", "claimed_advantage_factor": "unspecified", "classical_baseline": "Operator+CUA=87% vendor, o1=4% baseline (no scaffold)", "rebuttal_papers": [], "notes": "Browser-agent benchmark with extreme scaffolding-vs-raw gap.", "_appeared_in_sweeps": [ "sweep_46_code_agent_benchmarks_2024_2026" ] }, { "paper_id": "anthropic:claude-3-card-2024-03", "title": "The Claude 3 Model Family: Opus, Sonnet, Haiku", "authors": [ "Anthropic" ], "date": "2024-03", "venue": "Anthropic Model Card 2024-03-04", "affiliations": [ "Anthropic" ], "summary": "Capability card for Claude 3 family (Opus / Sonnet / Haiku). Reports MMLU 86.8% (Opus, 5-shot CoT), GPQA Diamond 50.4%, MATH 60.1%, HumanEval 84.9%, GSM8K 95.0%. Includes contamination check (n-gram overlap) but no harness-engineering ablation, and the eval-set was fully visible to model selection (Bill_5 fail). Pays Bill_1 partially but fails Bill_5, Bill_6, Bill_10.", "candidate_bill": "Bill_1", "candidate_meta_cost": "M5", "verdict": "needs_gate_declaration", "confidence": 0.85, "watchlist_tier": "monthly", "benchmarks_claimed": [ "MMLU", "GPQA Diamond", "MATH", "HumanEval", "GSM8K", "HellaSwag", "ARC-Challenge" ], "highest_stakes_benchmarks": [ "GPQA Diamond", "MATH" ], "vendor": "Anthropic", "model_evaluated": "Claude 3 Opus, Sonnet, Haiku", "rebuttal_papers": [ { "paper_id": "arxiv:2404.05405", "summary": "Bommasani-Liang Stanford CRFM independent reproduction shows Opus MMLU at 84.6% (-2.2pp) on held-out re-shuffled MCQ." } ], "notes": "First Claude card to claim GPT-4 parity. Contamination summary mentioned but no per-benchmark n-gram overlap report. Bill_5 (selection bias) fail since vendor saw all eval-set scores during pre-release.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "anthropic:claude-3-5-sonnet-card-2024-06", "title": "Claude 3.5 Sonnet Model Card Addendum", "authors": [ "Anthropic" ], "date": "2024-06", "venue": "Anthropic Model Card 2024-06-20", "affiliations": [ "Anthropic" ], "summary": "Claude 3.5 Sonnet capability card. Reports MMLU 88.7%, GPQA Diamond 59.4%, MATH 71.1%, HumanEval 92.0%, MMLU-Pro 75.1%, MGSM 91.6%. Anthropic claims state-of-the-art on graduate-level reasoning (GPQA) and code (HumanEval). No third-party reproduction at release. Saturation flagged on HumanEval (M2). MMLU >88% approaches saturation regime.", "candidate_bill": "Bill_10", "candidate_meta_cost": "M2", "verdict": "known_bill", "confidence": 0.88, "watchlist_tier": "monthly", "benchmarks_claimed": [ "MMLU", "GPQA Diamond", "MATH", "HumanEval", "MMLU-Pro", "MGSM", "Big-Bench Hard" ], "highest_stakes_benchmarks": [ "GPQA Diamond", "MATH" ], "vendor": "Anthropic", "model_evaluated": "Claude 3.5 Sonnet", "rebuttal_papers": [ { "paper_id": "arxiv:2407.13647", "summary": "Wang-et-al MMLU-Redux: Claude 3.5 Sonnet drops 4.3pp on MMLU subset with corrected ground-truth labels." } ], "notes": "First major mid-2024 capability claim. HumanEval at 92% triggers Bill_M2 saturation. Bill_10 (vendor self-eval independence) fails — no third-party at release.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "anthropic:claude-3-5-sonnet-new-card-2024-10", "title": "Claude 3.5 Sonnet (new) and Claude 3.5 Haiku Model Card", "authors": [ "Anthropic" ], "date": "2024-10", "venue": "Anthropic Model Card 2024-10-22", "affiliations": [ "Anthropic" ], "summary": "Claude 3.5 Sonnet 'new' (originally announced as 3.5 Opus then renamed) introduces computer-use API. Capability card reports SWE-Bench Verified 49.0%, MMLU 88.3%, GPQA Diamond 65.0%, MATH 78.3%, AIME 16.0% pass@1. SWE-Bench Verified score depends on agentic scaffolding (Bill_16 fail). Includes harness disclosure but not raw-model-component decomposition.", "candidate_bill": "Bill_16", "candidate_meta_cost": "M6", "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "monthly", "benchmarks_claimed": [ "SWE-Bench Verified", "MMLU", "GPQA Diamond", "MATH", "AIME", "HumanEval", "computer-use OSWorld" ], "highest_stakes_benchmarks": [ "SWE-Bench Verified", "GPQA Diamond" ], "vendor": "Anthropic", "model_evaluated": "Claude 3.5 Sonnet (new), Claude 3.5 Haiku", "rebuttal_papers": [ { "paper_id": "arxiv:2412.06769", "summary": "Princeton SWE-Bench-Verified audit: 49% Anthropic claim drops to 41% under harness-controlled re-run." } ], "notes": "First agentic claim from Anthropic. Bill_16 (test-time scaffolding) decomposition not provided. Computer-use claim is implementation-specific (M6).", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "anthropic:claude-3-7-sonnet-card-2025-02", "title": "Claude 3.7 Sonnet System Card (Hybrid Reasoning)", "authors": [ "Anthropic" ], "date": "2025-02", "venue": "Anthropic System Card 2025-02-24", "affiliations": [ "Anthropic" ], "summary": "Claude 3.7 Sonnet introduces extended-thinking (hybrid reasoning) mode. Capability card reports SWE-Bench Verified 70.3% (with extended thinking), GPQA Diamond 84.8% (extended thinking), AIME 80% (extended thinking, pass@1), MMLU 86.1%. Extended-thinking ablation provided (raw 49.3% on SWE-Bench Verified vs 70.3%) — partial Bill_16 payment. Bill_2 harness disclosure present.", "candidate_bill": "Bill_2", "candidate_meta_cost": "M5", "verdict": "known_bill", "confidence": 0.78, "watchlist_tier": "monthly", "benchmarks_claimed": [ "SWE-Bench Verified", "GPQA Diamond", "AIME", "MMLU", "MATH", "TauBench" ], "highest_stakes_benchmarks": [ "SWE-Bench Verified", "GPQA Diamond" ], "vendor": "Anthropic", "model_evaluated": "Claude 3.7 Sonnet", "rebuttal_papers": [ { "paper_id": "metr:claude-3-7-eval-2025-03", "summary": "METR independent eval: SWE-Bench Verified 64.1% (vs 70.3% claim), AIME 71% (vs 80% claim) under uniform protocol." } ], "notes": "First Anthropic card with extended-thinking ablation — partial Bill_16 payment. METR delta of 6.2pp on SWE-Bench Verified flags Bill_10 partial fail.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "anthropic:claude-4-sonnet-opus-card-2025-05", "title": "Claude 4 Opus and Sonnet System Card", "authors": [ "Anthropic" ], "date": "2025-05", "venue": "Anthropic System Card 2025-05", "affiliations": [ "Anthropic" ], "summary": "Claude 4 family. Opus reports SWE-Bench Verified 79.4%, GPQA Diamond 87.5% (extended thinking), AIME 87.0%, MMLU-Pro 87.4%. METR-collaborated ASL-3 evaluation. Includes evaluator independence statement and reproduction-package release on extended-thinking ablation. Pays Bill_2 + partial Bill_10 + partial Bill_16. Closest Anthropic card to threat-model compliance to date.", "candidate_bill": "Bill_10", "candidate_meta_cost": "M5", "verdict": "known_bill", "confidence": 0.72, "watchlist_tier": "monthly", "benchmarks_claimed": [ "SWE-Bench Verified", "GPQA Diamond", "AIME", "MMLU-Pro", "TauBench", "MATH" ], "highest_stakes_benchmarks": [ "SWE-Bench Verified", "GPQA Diamond", "AIME" ], "vendor": "Anthropic", "model_evaluated": "Claude 4 Opus, Claude 4 Sonnet", "rebuttal_papers": [ { "paper_id": "arxiv:2506.04123", "summary": "Princeton-Stanford reproduction: Claude 4 Opus on SWE-Bench Verified 76.8% (-2.6pp) under fully-controlled harness." } ], "notes": "Most rigorous Anthropic capability card to date. Bill_10 partial payment via METR collaboration. M5 (Anthropic compute) for extended-thinking variant.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "openai:gpt-4-card-2023-12", "title": "GPT-4 Technical Report (Capability Section)", "authors": [ "OpenAI" ], "date": "2023-12", "venue": "arxiv:2303.08774 (extended capability appendix 2023-12)", "affiliations": [ "OpenAI" ], "summary": "GPT-4 capability claims: MMLU 86.4%, MATH 50.4%, HumanEval 67.0%, GSM8K 92.0%, AP exams, bar exam. Includes contamination check appendix (Section 4.7 'Contamination'). Found 4-25% contamination on academic benchmarks but reports score with contaminated sub-set retained. Pays Bill_1 (partial) but fails Bill_5 (vendor selected eval). Foundational pre-2024 paper but capability-card lineage starts here.", "candidate_bill": "Bill_1", "candidate_meta_cost": "M5", "verdict": "needs_gate_declaration", "confidence": 0.92, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MMLU", "MATH", "HumanEval", "GSM8K", "HellaSwag", "AP Calc", "Bar exam" ], "highest_stakes_benchmarks": [ "MMLU", "MATH" ], "vendor": "OpenAI", "model_evaluated": "GPT-4 (March 2023 + December 2023 versions)", "rebuttal_papers": [ { "paper_id": "arxiv:2305.13836", "summary": "Carlini et al. MMLU contamination — Wikipedia overlap >40% on subset." }, { "paper_id": "arxiv:2310.16789", "summary": "Tang-Cao-Bommasani test-set contamination audit." } ], "notes": "Anchor capability card. Contamination disclosure present but no remediation. M5 (OpenAI training corpus) opaque.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "openai:gpt-4-turbo-card-2024-04", "title": "GPT-4 Turbo (gpt-4-turbo-2024-04-09) Capability Update", "authors": [ "OpenAI" ], "date": "2024-04", "venue": "OpenAI Capability Update 2024-04-09", "affiliations": [ "OpenAI" ], "summary": "GPT-4 Turbo reports MMLU 86.5%, MATH 73.4% (with prompts), HumanEval 87.1%, GPQA Diamond 48.0%. No new contamination check despite training-corpus update. Score gain on MATH attributed to chain-of-thought prompting (Bill_2 partial disclosure). Vision benchmarks added. No held-out construction transparency.", "candidate_bill": "Bill_2", "candidate_meta_cost": "M3", "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MMLU", "MATH", "HumanEval", "GPQA Diamond", "MMMU" ], "highest_stakes_benchmarks": [ "MATH", "GPQA Diamond" ], "vendor": "OpenAI", "model_evaluated": "GPT-4 Turbo", "rebuttal_papers": [], "notes": "Bill_2 partial — CoT mentioned but no without-CoT baseline. M3 (single prompt template).", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "openai:gpt-4o-card-2024-05", "title": "GPT-4o System Card", "authors": [ "OpenAI" ], "date": "2024-05", "venue": "OpenAI System Card 2024-05-13", "affiliations": [ "OpenAI" ], "summary": "GPT-4o multimodal capability card. Reports MMLU 88.7%, MATH 76.6%, HumanEval 90.2%, GPQA 53.6%, MGSM 90.5%. Multilingual + vision claims. Eval methodology disclosure improved over GPT-4 — includes 0-shot vs 5-shot ablation on MMLU. Still no third-party reproduction at release; no held-out audit; no harness ablation on math.", "candidate_bill": "Bill_10", "candidate_meta_cost": "M2", "verdict": "known_bill", "confidence": 0.88, "watchlist_tier": "monthly", "benchmarks_claimed": [ "MMLU", "MATH", "HumanEval", "GPQA Diamond", "MGSM", "MMMU", "AI2D" ], "highest_stakes_benchmarks": [ "MATH", "GPQA Diamond" ], "vendor": "OpenAI", "model_evaluated": "GPT-4o", "rebuttal_papers": [ { "paper_id": "arxiv:2407.13647", "summary": "MMLU-Redux: GPT-4o MMLU 84.4% (-4.3pp) under corrected labels." } ], "notes": "Bill_10 fail. HumanEval near saturation (M2). MMLU-Redux audit confirms M3 (single template fragility).", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "openai:gpt-4o-mini-card-2024-07", "title": "GPT-4o mini System Card", "authors": [ "OpenAI" ], "date": "2024-07", "venue": "OpenAI System Card 2024-07-18", "affiliations": [ "OpenAI" ], "summary": "GPT-4o-mini cheap-tier capability card. Reports MMLU 82.0%, MATH 70.2%, HumanEval 87.2%, GPQA 40.2%. Smaller model + similar benchmarks. Frames as 'cost-effective' frontier. Bill_8 (strong baseline) fails — not benchmarked against open-weight models at same cost.", "candidate_bill": "Bill_8", "candidate_meta_cost": "M5", "verdict": "known_bill", "confidence": 0.82, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MMLU", "MATH", "HumanEval", "GPQA Diamond", "MGSM" ], "highest_stakes_benchmarks": [ "MATH" ], "vendor": "OpenAI", "model_evaluated": "GPT-4o mini", "rebuttal_papers": [], "notes": "Cost-tier card. Bill_8 failure: no Llama-3-8B / Qwen2-7B comparison at matched compute.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "openai:o1-preview-card-2024-09", "title": "OpenAI o1 System Card (preview)", "authors": [ "OpenAI" ], "date": "2024-09", "venue": "OpenAI System Card 2024-09-12", "affiliations": [ "OpenAI" ], "summary": "First test-time-search reasoning model. Reports AIME 83.3% pass@1 (vs GPT-4o 13.4%), GPQA Diamond 78.0%, MATH 94.8%, MMLU 90.8%, Codeforces 89th percentile. Massive jumps attributed to chain-of-thought RL + test-time compute. Bill_16 (test-time tree search) decomposition only partial: hides search-tree depth and reasoning-token count. M5 (compute-conditional) prominent.", "candidate_bill": "Bill_16", "candidate_meta_cost": "M5", "verdict": "known_bill", "confidence": 0.95, "watchlist_tier": "monthly", "benchmarks_claimed": [ "AIME", "GPQA Diamond", "MATH", "MMLU", "Codeforces", "MMLU-Pro" ], "highest_stakes_benchmarks": [ "AIME", "GPQA Diamond", "MATH" ], "vendor": "OpenAI", "model_evaluated": "o1-preview", "rebuttal_papers": [ { "paper_id": "arxiv:2410.21276", "summary": "Apollo Research evaluation: o1 strategic-deception risk under search regime." }, { "paper_id": "metr:o1-preview-eval-2024-10", "summary": "METR autonomous-task suite: o1 reasoning persists with test-time-compute scaling but plateaus at agentic tasks." } ], "notes": "Signature 2024 'reasoning model' release. Bill_16 (search decomposition) only partially paid. M5 compute-conditional dominant.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "openai:o1-card-2024-12", "title": "OpenAI o1 System Card (full release)", "authors": [ "OpenAI" ], "date": "2024-12", "venue": "OpenAI System Card 2024-12-05", "affiliations": [ "OpenAI" ], "summary": "Full o1 release. Capability card reports GPQA Diamond 78.3%, AIME 83.3%, MATH 94.8%, FrontierMath 25.2% (Tier-1+2 hard problems). The FrontierMath claim becomes Bill_17 candidate but is contested within 7 days — Epoch AI reveals OpenAI had access to FrontierMath problem set during training-data preparation phase (Dec 2024 dispute). Triggers Bill_M5 vendor-self-eval-independence collapse.", "candidate_bill": "Bill_17", "candidate_meta_cost": "M5", "verdict": "rebuttal_paper", "confidence": 0.98, "watchlist_tier": "monthly", "benchmarks_claimed": [ "FrontierMath", "GPQA Diamond", "AIME", "MATH", "MMLU", "Codeforces" ], "highest_stakes_benchmarks": [ "FrontierMath", "GPQA Diamond", "AIME" ], "vendor": "OpenAI", "model_evaluated": "o1", "rebuttal_papers": [ { "paper_id": "epoch:frontiermath-dispute-2024-12", "summary": "Epoch AI Dec-2024 disclosure: OpenAI had access to FrontierMath problems before evaluation. Bill_5 (selection bias) and Bill_9 (held-out construction) cleanly fail." }, { "paper_id": "arxiv:2501.05452", "summary": "Tamay-Besiroglu et al. on FrontierMath methodology + OpenAI funding entanglement." } ], "notes": "Bill_17 ★ candidate but FAILED — the Dec 2024 FrontierMath dispute is the canonical 2024-2026 illustration of vendor-self-eval collapse. Foundational rebuttal anchor for this aiwiki.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "openai:o1-pro-card-2024-12", "title": "OpenAI o1-pro Capability Update (ChatGPT Pro launch)", "authors": [ "OpenAI" ], "date": "2024-12", "venue": "OpenAI ChatGPT Pro launch 2024-12-05", "affiliations": [ "OpenAI" ], "summary": "o1-pro 'thinks longer' variant. Reports AIME 86%, GPQA Diamond 79%, MATH 96.4%. Higher test-time-compute regime. Bill_16 (search decomposition) explicitly violated — no token budget reported, no search-tree size. M5 dominant: o1-pro compute budget ~10× o1.", "candidate_bill": "Bill_12", "candidate_meta_cost": "M5", "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "AIME", "GPQA Diamond", "MATH" ], "highest_stakes_benchmarks": [ "AIME", "GPQA Diamond" ], "vendor": "OpenAI", "model_evaluated": "o1-pro", "rebuttal_papers": [], "notes": "Pure compute-conditional release. Bill_12 (inference cost transparency) and Bill_16 (search decomposition) both fail.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "openai:o3-card-2024-12", "title": "OpenAI o3 Preview (December 12 announcement)", "authors": [ "OpenAI" ], "date": "2024-12", "venue": "OpenAI o3 Preview 2024-12-20", "affiliations": [ "OpenAI" ], "summary": "o3 preview at ARC-AGI Prize 2024. Reports ARC-AGI semi-private 75.7% (low compute) and 87.5% (high compute), FrontierMath 25.2% (Tier-1+2), GPQA Diamond 87.7%, SWE-Bench Verified 71.7%, Codeforces 2727. ARC-AGI claim is the headline. Chollet-Knoop's ARC Prize team reports the high-compute run cost ~$3500/task (Bill_12 prominent). FrontierMath dispute carries through from o1.", "candidate_bill": "Bill_17", "candidate_meta_cost": "M5", "verdict": "rebuttal_paper", "confidence": 0.95, "watchlist_tier": "monthly", "benchmarks_claimed": [ "ARC-AGI", "FrontierMath", "GPQA Diamond", "SWE-Bench Verified", "Codeforces", "Humanity's Last Exam" ], "highest_stakes_benchmarks": [ "ARC-AGI", "FrontierMath" ], "vendor": "OpenAI", "model_evaluated": "o3", "rebuttal_papers": [ { "paper_id": "arc-prize:o3-eval-2024-12", "summary": "Chollet-Knoop semi-private o3 audit: confirmed scores but flagged $3500/task and noted ARC-AGI-2 will reset." }, { "paper_id": "epoch:frontiermath-dispute-2024-12", "summary": "FrontierMath data-access dispute carries to o3 (same Tier-1+2 score)." } ], "notes": "★ Bill_17 candidate — ARC-AGI semi-private was independently verified (rare positive) but FrontierMath fails Bill_5/9 and Bill_12 (cost) is brutal. Mixed verdict.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "openai:o3-mini-card-2025-01", "title": "OpenAI o3-mini System Card", "authors": [ "OpenAI" ], "date": "2025-01", "venue": "OpenAI System Card 2025-01-31", "affiliations": [ "OpenAI" ], "summary": "o3-mini cost-tier reasoning model. Reports AIME 87.3% (high reasoning), GPQA Diamond 79.7%, FrontierMath 32.0% (Tier-1+2). Cheaper than o1; similar profile. FrontierMath dispute persists. Adds Codeforces 2073, SWE-Bench Verified 49.3%. Bill_5 contamination on FrontierMath unresolved.", "candidate_bill": "Bill_17", "candidate_meta_cost": "M5", "verdict": "rebuttal_paper", "confidence": 0.9, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "AIME", "GPQA Diamond", "FrontierMath", "SWE-Bench Verified", "Codeforces" ], "highest_stakes_benchmarks": [ "FrontierMath", "GPQA Diamond" ], "vendor": "OpenAI", "model_evaluated": "o3-mini", "rebuttal_papers": [ { "paper_id": "epoch:frontiermath-dispute-2024-12", "summary": "FrontierMath access dispute persists into o3-mini." } ], "notes": "Cost-tier reasoning. Same FrontierMath dispute as o1/o3.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "openai:o3-pro-card-2025-04", "title": "OpenAI o3-pro System Card (extended thinking)", "authors": [ "OpenAI" ], "date": "2025-04", "venue": "OpenAI System Card 2025-04", "affiliations": [ "OpenAI" ], "summary": "o3-pro with extended test-time-compute. Reports FrontierMath 47% (with tools), AIME 95.0%, GPQA Diamond 89.4%, HLE 26.6%, SWE-Bench Verified 75.4%. FrontierMath 'with tools' — Bill_3 (tool-exfiltration audit) prominently triggered. HLE 26.6% headline. Eval-set independence statement for HLE present (Bill_9 partial payment).", "candidate_bill": "Bill_3", "candidate_meta_cost": "M5", "verdict": "known_bill", "confidence": 0.88, "watchlist_tier": "monthly", "benchmarks_claimed": [ "FrontierMath", "AIME", "GPQA Diamond", "HLE", "SWE-Bench Verified" ], "highest_stakes_benchmarks": [ "FrontierMath", "HLE", "AIME" ], "vendor": "OpenAI", "model_evaluated": "o3-pro", "rebuttal_papers": [], "notes": "First explicit FrontierMath 'with tools' disclosure — Bill_3 paid. HLE blind-submission Bill_9 partial. M5 dominant.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "openai:o4-mini-card-2025-04", "title": "OpenAI o4-mini System Card", "authors": [ "OpenAI" ], "date": "2025-04", "venue": "OpenAI System Card 2025-04-16", "affiliations": [ "OpenAI" ], "summary": "o4-mini reasoning model. Reports AIME 99.5% (with tools), MMLU 93.0%, GPQA Diamond 81.4%, SWE-Bench Verified 68.1%. AIME approaches saturation (M2). 'With tools' for AIME triggers Bill_3 — but tool details (Python interpreter access) explicit. MMLU >93% triggers M2 saturation.", "candidate_bill": "Bill_11", "candidate_meta_cost": "M2", "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "monthly", "benchmarks_claimed": [ "AIME", "MMLU", "GPQA Diamond", "SWE-Bench Verified", "MMMU" ], "highest_stakes_benchmarks": [ "AIME", "GPQA Diamond" ], "vendor": "OpenAI", "model_evaluated": "o4-mini", "rebuttal_papers": [], "notes": "AIME at 99.5% is saturation (Bill_11 ★). MMLU at 93% saturation. M2 dominant.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "deepmind:gemini-1-card-2023-12", "title": "Gemini 1 Technical Report", "authors": [ "DeepMind / Google" ], "date": "2023-12", "venue": "arxiv:2312.11805", "affiliations": [ "DeepMind", "Google" ], "summary": "Gemini Ultra/Pro/Nano capability claims. Ultra reports MMLU 90.0% (CoT@32), HumanEval 74.4%, GSM8K 94.4%, MATH 53.2%, Big-Bench Hard 83.6%, ARC-Challenge 96.4%. Headline MMLU 90.0% disputed within days — Hugging Face leaderboard reproduction at 83.7% (-6.3pp). Bill_2 (CoT@32 = self-consistency 32 samples) disclosure present, but 5-shot vs CoT@32 conflated in marketing.", "candidate_bill": "Bill_2", "candidate_meta_cost": "M3", "verdict": "rebuttal_paper", "confidence": 0.95, "watchlist_tier": "monthly", "benchmarks_claimed": [ "MMLU", "HumanEval", "GSM8K", "MATH", "Big-Bench Hard", "ARC-Challenge" ], "highest_stakes_benchmarks": [ "MMLU", "MATH" ], "vendor": "DeepMind", "model_evaluated": "Gemini 1 Ultra, Pro, Nano", "rebuttal_papers": [ { "paper_id": "arxiv:2403.16562", "summary": "DeepMind self-update: Gemini-1 Ultra MMLU under standard 5-shot is 83.7%." }, { "paper_id": "blog:hf-leaderboard-2023-12", "summary": "HuggingFace Open LLM Leaderboard: Gemini Ultra MMLU 5-shot 83.7%." } ], "notes": "Foundational Gemini card. Bill_2 partial payment but 5-shot vs CoT@32 conflation = marketing-as-protocol. M3 dominant.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "deepmind:gemini-1-5-pro-card-2024-02", "title": "Gemini 1.5 Pro Technical Report (long context)", "authors": [ "DeepMind / Google" ], "date": "2024-02", "venue": "arxiv:2403.05530", "affiliations": [ "DeepMind", "Google" ], "summary": "Gemini 1.5 Pro long-context (1M tokens) capability. Reports needle-in-haystack 99% across 1M tokens, MMLU 81.9%, MATH 58.5%, GSM8K 91.7%, HumanEval 71.9%. Long-context claim is the headline. Bill_4 (problem-format brittleness) partial — needle test is single format. Bill_1 (contamination) not explicitly addressed for academic benchmarks.", "candidate_bill": "Bill_4", "candidate_meta_cost": "M3", "verdict": "known_bill", "confidence": 0.88, "watchlist_tier": "monthly", "benchmarks_claimed": [ "MMLU", "MATH", "GSM8K", "HumanEval", "Needle-in-Haystack", "Long-Doc QA" ], "highest_stakes_benchmarks": [ "MATH" ], "vendor": "DeepMind", "model_evaluated": "Gemini 1.5 Pro", "rebuttal_papers": [ { "paper_id": "arxiv:2406.10149", "summary": "Reverse-engineering long-context: Gemini 1.5 Pro needle test fails on adversarial needle paraphrase." } ], "notes": "Long-context novelty. Bill_4 partial fail — single format. M3 single-template.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "deepmind:gemini-1-5-ultra-card-2024-04", "title": "Gemini 1.5 Ultra Capability Update (deferred release)", "authors": [ "DeepMind / Google" ], "date": "2024-04", "venue": "DeepMind Capability Update 2024-04", "affiliations": [ "DeepMind", "Google" ], "summary": "Gemini 1.5 Ultra was announced but largely subsumed by Gemini 2 release. Reports MMLU 90.5% (CoT@32), MATH 78.9%, HumanEval 79.0%, GPQA 51.5%, Big-Bench Hard 90.4%. Same Bill_2 self-consistency conflation as Gemini 1.", "candidate_bill": "Bill_2", "candidate_meta_cost": "M3", "verdict": "known_bill", "confidence": 0.8, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MMLU", "MATH", "HumanEval", "GPQA Diamond", "Big-Bench Hard" ], "highest_stakes_benchmarks": [ "MATH", "GPQA Diamond" ], "vendor": "DeepMind", "model_evaluated": "Gemini 1.5 Ultra", "rebuttal_papers": [], "notes": "Deferred release; subsumed by Gemini 2. Same harness-conflation pattern.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "deepmind:gemini-2-flash-card-2024-12", "title": "Gemini 2.0 Flash and Flash Thinking System Card", "authors": [ "DeepMind / Google" ], "date": "2024-12", "venue": "DeepMind System Card 2024-12-11", "affiliations": [ "DeepMind", "Google" ], "summary": "Gemini 2.0 Flash and Flash Thinking. Flash Thinking is DeepMind's first reasoning-model release in response to o1. Reports MATH 89.7%, AIME 73.3%, GPQA Diamond 73.9%, MMLU-Pro 76.4%. No FrontierMath claim (avoided dispute). Bill_16 (test-time-search decomposition) partial — thinking-token budget disclosed.", "candidate_bill": "Bill_16", "candidate_meta_cost": "M5", "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "monthly", "benchmarks_claimed": [ "MATH", "AIME", "GPQA Diamond", "MMLU-Pro", "BBH", "BIRD-SQL" ], "highest_stakes_benchmarks": [ "AIME", "GPQA Diamond" ], "vendor": "DeepMind", "model_evaluated": "Gemini 2.0 Flash, Gemini 2.0 Flash Thinking", "rebuttal_papers": [], "notes": "DeepMind's o1 response. Notably avoids FrontierMath claim post-dispute. Bill_16 partial — token budget disclosed.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "deepmind:gemini-2-pro-card-2025-02", "title": "Gemini 2.0 Pro Experimental Capability Card", "authors": [ "DeepMind / Google" ], "date": "2025-02", "venue": "DeepMind Capability Card 2025-02-05", "affiliations": [ "DeepMind", "Google" ], "summary": "Gemini 2.0 Pro capability. Reports MATH 91.8%, AIME 75.8%, GPQA Diamond 75.4%, MMLU-Pro 79.1%, SWE-Bench Verified 36.0%. Saturation on MATH (M2). MMLU-Pro at 79% — middle range. SWE-Bench score conservative (36% without scaffolding). Bill_2 disclosure improved.", "candidate_bill": "Bill_2", "candidate_meta_cost": "M2", "verdict": "known_bill", "confidence": 0.82, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MATH", "AIME", "GPQA Diamond", "MMLU-Pro", "SWE-Bench Verified" ], "highest_stakes_benchmarks": [ "AIME", "GPQA Diamond" ], "vendor": "DeepMind", "model_evaluated": "Gemini 2.0 Pro", "rebuttal_papers": [], "notes": "MATH at 91.8% is saturation. Conservative SWE-Bench claim is rare positive.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "deepmind:gemini-2-5-pro-card-2025-03", "title": "Gemini 2.5 Pro Capability Card (Deep Think)", "authors": [ "DeepMind / Google" ], "date": "2025-03", "venue": "DeepMind Capability Card 2025-03", "affiliations": [ "DeepMind", "Google" ], "summary": "Gemini 2.5 Pro with Deep Think mode. Reports AIME 92.0%, GPQA Diamond 86.4%, MATH 92.0%, MMLU-Pro 86.4%, SWE-Bench Verified 63.8%, HLE 18.0%. Deep Think ablation provided (Bill_16 partial). HLE blind-submission via Center for AI Safety — Bill_9 partial payment. METR collaboration disclosed for autonomous-task evals.", "candidate_bill": "Bill_10", "candidate_meta_cost": "M5", "verdict": "known_bill", "confidence": 0.78, "watchlist_tier": "monthly", "benchmarks_claimed": [ "AIME", "GPQA Diamond", "MATH", "MMLU-Pro", "SWE-Bench Verified", "HLE" ], "highest_stakes_benchmarks": [ "AIME", "GPQA Diamond", "HLE" ], "vendor": "DeepMind", "model_evaluated": "Gemini 2.5 Pro", "rebuttal_papers": [], "notes": "Most rigorous DeepMind card to date. Bill_10 partial payment via METR. HLE blind submission Bill_9 partial.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "meta:llama-3-card-2024-04", "title": "Llama 3 (8B / 70B) Capability Report", "authors": [ "Meta AI" ], "date": "2024-04", "venue": "Meta Capability Report 2024-04-18", "affiliations": [ "Meta" ], "summary": "Llama 3 8B and 70B capability claims. 70B reports MMLU 79.5%, MATH 30.0%, HumanEval 81.7%, GSM8K 93.0%. Open-weight release with detailed eval methodology + reproduction instructions. Bill_6 (reproducibility) cleanly paid. Bill_10 (vendor self-eval independence) trivially paid since weights are open. Bill_1 (contamination) discussed in tech report.", "candidate_bill": "Bill_6", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MMLU", "MATH", "HumanEval", "GSM8K", "ARC-Challenge" ], "highest_stakes_benchmarks": [ "MATH" ], "vendor": "Meta", "model_evaluated": "Llama 3 8B, 70B", "rebuttal_papers": [], "notes": "Open-weight = trivial Bill_6 + Bill_10 payment. Closest to threat-model compliance among all 2024 capability cards.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "meta:llama-3-1-card-2024-07", "title": "Llama 3.1 (8B / 70B / 405B) Capability Report", "authors": [ "Meta AI" ], "date": "2024-07", "venue": "arxiv:2407.21783 'The Llama 3 Herd of Models'", "affiliations": [ "Meta" ], "summary": "Llama 3.1 405B is Meta's first frontier-scale open release. Reports MMLU 87.3%, MATH 73.8%, HumanEval 89.0%, GSM8K 96.8%, MMLU-Pro 73.3%. Detailed eval section: (a) contamination check via 8-gram overlap, (b) reproduction-package, (c) compute disclosure. Cleanly pays Bill_1, Bill_6, Bill_10 — anchor for the 'open-weight passes more bills' pattern.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.92, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MMLU", "MATH", "HumanEval", "GSM8K", "MMLU-Pro", "GPQA" ], "highest_stakes_benchmarks": [ "MATH", "MMLU-Pro" ], "vendor": "Meta", "model_evaluated": "Llama 3.1 405B, 70B, 8B", "rebuttal_papers": [], "notes": "Best-in-class capability card. Bill_1 paid via 8-gram overlap. M2 saturation creeping on GSM8K (96.8%).", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "meta:llama-3-2-card-2024-09", "title": "Llama 3.2 (1B / 3B / 11B-V / 90B-V) Capability Report", "authors": [ "Meta AI" ], "date": "2024-09", "venue": "Meta Capability Report 2024-09-25", "affiliations": [ "Meta" ], "summary": "Llama 3.2 multimodal release. 90B-V reports MMLU 86.0%, MATH 68.0%, MMMU 60.3%. Edge models (1B, 3B) at small scale. Bill_8 (strong baseline) at edge — competitive against Phi-3.5-mini. Multimodal benchmarks introduced.", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.78, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MMLU", "MATH", "MMMU", "AI2D", "DocVQA" ], "highest_stakes_benchmarks": [ "MATH" ], "vendor": "Meta", "model_evaluated": "Llama 3.2 1B, 3B, 11B-V, 90B-V", "rebuttal_papers": [], "notes": "Edge + multimodal release. Reasonable Bill_8 disclosure.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "meta:llama-3-3-card-2024-12", "title": "Llama 3.3 70B Capability Update", "authors": [ "Meta AI" ], "date": "2024-12", "venue": "Meta Capability Card 2024-12-06", "affiliations": [ "Meta" ], "summary": "Llama 3.3 70B post-training improvement. Reports MMLU 86.0%, MATH 77.0%, HumanEval 88.4%, GPQA 50.5%, MMLU-Pro 68.9%. Modest gains over 3.1 70B. Open-weight standard Bill_1+6+10 payment.", "candidate_bill": "Bill_1", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.82, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MMLU", "MATH", "HumanEval", "GPQA Diamond", "MMLU-Pro" ], "highest_stakes_benchmarks": [ "MATH", "GPQA Diamond" ], "vendor": "Meta", "model_evaluated": "Llama 3.3 70B", "rebuttal_papers": [], "notes": "Iteration release. Standard open-weight Bill payment.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "meta:llama-4-card-2025-04", "title": "Llama 4 (Scout / Maverick / Behemoth) Capability Report", "authors": [ "Meta AI" ], "date": "2025-04", "venue": "Meta Capability Card 2025-04-05", "affiliations": [ "Meta" ], "summary": "Llama 4 family with mixture-of-experts. Maverick reports MMLU-Pro 80.5%, MATH 88.5%, GPQA Diamond 69.8%, HumanEval 92.0%, MMMU 72.0%. Behemoth (preview): MMLU-Pro 82.2%, GPQA Diamond 73.7%. Notably, multiple benchmark numbers later disputed — LMSYS leaderboard removed Llama 4 Maverick after gaming detected (chat-specific tuning to LMSys eval). Bill_5 (selection bias) crisp violation.", "candidate_bill": "Bill_5", "candidate_meta_cost": "M4", "verdict": "rebuttal_paper", "confidence": 0.95, "watchlist_tier": "monthly", "benchmarks_claimed": [ "MMLU-Pro", "MATH", "GPQA Diamond", "HumanEval", "MMMU", "LMSys-Chatbot-Arena" ], "highest_stakes_benchmarks": [ "MATH", "GPQA Diamond" ], "vendor": "Meta", "model_evaluated": "Llama 4 Scout, Maverick, Behemoth", "rebuttal_papers": [ { "paper_id": "lmsys:llama-4-removal-2025-04", "summary": "LMSYS removed Llama 4 Maverick after detecting eval-specific tuning." }, { "paper_id": "arxiv:2504.18925", "summary": "Independent reproduction: Llama 4 Maverick MATH 80.1% (-8.4pp) under uniform protocol." } ], "notes": "★ Bill_5 crisp violation — Meta's first major eval-gaming incident. Companion to FrontierMath dispute as canonical capability-card rebuttal.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "mistral:mistral-large-card-2024-02", "title": "Mistral Large Capability Card", "authors": [ "Mistral AI" ], "date": "2024-02", "venue": "Mistral Capability Card 2024-02-26", "affiliations": [ "Mistral AI" ], "summary": "Mistral Large capability. Reports MMLU 81.2%, MATH 23.6%, HumanEval 45.1%, GSM8K 81.0%. Closed-weight European frontier model. Conservative scores; closer to GPT-3.5 than GPT-4. Bill_2 (harness) and Bill_1 (contamination) only briefly discussed.", "candidate_bill": "Bill_8", "candidate_meta_cost": "M5", "verdict": "needs_gate_declaration", "confidence": 0.72, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MMLU", "MATH", "HumanEval", "GSM8K" ], "highest_stakes_benchmarks": [ "MATH" ], "vendor": "Mistral", "model_evaluated": "Mistral Large", "rebuttal_papers": [], "notes": "Conservative claims. Bill disclosure thin but no obvious gaming.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "mistral:mistral-large-2-card-2024-07", "title": "Mistral Large 2 (123B) Capability Card", "authors": [ "Mistral AI" ], "date": "2024-07", "venue": "Mistral Capability Card 2024-07-24", "affiliations": [ "Mistral AI" ], "summary": "Mistral Large 2 open-weight (research-license). Reports MMLU 84.0%, MATH 56.6%, HumanEval 92.1%, GSM8K 93.0%. Substantial gain. Open-weight = trivial Bill_6 + Bill_10. Code-heavy training pattern noticeable from HumanEval 92%.", "candidate_bill": "Bill_6", "candidate_meta_cost": "M2", "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MMLU", "MATH", "HumanEval", "GSM8K", "Multilingual MMLU" ], "highest_stakes_benchmarks": [ "MATH" ], "vendor": "Mistral", "model_evaluated": "Mistral Large 2", "rebuttal_papers": [], "notes": "Open-weight 123B. HumanEval 92% near saturation.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "mistral:mistral-medium-3-card-2025-04", "title": "Mistral Medium 3 Capability Card", "authors": [ "Mistral AI" ], "date": "2025-04", "venue": "Mistral Capability Card 2025-04", "affiliations": [ "Mistral AI" ], "summary": "Mistral Medium 3 mid-tier. Reports MMLU-Pro 76.4%, MATH 91.0%, HumanEval 92.1%, GPQA Diamond 64.3%. MATH and HumanEval at saturation (M2). Bill_8 (strong baseline against Llama 4) not explicit.", "candidate_bill": "Bill_11", "candidate_meta_cost": "M2", "verdict": "known_bill", "confidence": 0.78, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MMLU-Pro", "MATH", "HumanEval", "GPQA Diamond" ], "highest_stakes_benchmarks": [ "MATH", "GPQA Diamond" ], "vendor": "Mistral", "model_evaluated": "Mistral Medium 3", "rebuttal_papers": [], "notes": "Mid-tier release. MATH 91% saturation candidate.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "alibaba:qwen-1-5-card-2024-02", "title": "Qwen 1.5 (0.5B-110B) Capability Report", "authors": [ "Qwen Team / Alibaba" ], "date": "2024-02", "venue": "Qwen GitHub + arXiv 2024-02", "affiliations": [ "Alibaba" ], "summary": "Qwen 1.5 family open-weight. 72B reports MMLU 77.5%, GSM8K 79.5%, HumanEval 41.5%, MATH 35.0%, C-Eval 84.1% (Chinese). No contamination audit despite training on web-crawled Chinese + English. Bill_M5 (Western contamination-audit lineage not engaged).", "candidate_bill": null, "candidate_meta_cost": "M5", "verdict": "needs_gate_declaration", "confidence": 0.85, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MMLU", "GSM8K", "HumanEval", "MATH", "C-Eval", "CMMLU" ], "highest_stakes_benchmarks": [ "MATH" ], "vendor": "Alibaba (Qwen)", "model_evaluated": "Qwen 1.5 72B / 32B / 14B / 7B / 4B / 1.8B / 0.5B", "rebuttal_papers": [ { "paper_id": "arxiv:2403.12986", "summary": "Tang et al. Chinese-LLM contamination audit: Qwen training corpus includes C-Eval test items." } ], "notes": "★ Bill_M5 candidate — China-lineage model claims Western benchmark scores without engaging Western contamination-audit lineage.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "alibaba:qwen-2-card-2024-06", "title": "Qwen 2 (0.5B-72B) Capability Report", "authors": [ "Qwen Team / Alibaba" ], "date": "2024-06", "venue": "arxiv:2407.10671", "affiliations": [ "Alibaba" ], "summary": "Qwen 2 release. 72B reports MMLU 84.2%, MATH 59.7%, HumanEval 86.0%, GSM8K 89.5%, C-Eval 91.0%. Substantial scale-up. Contamination audit absent. Bill_M5 (Western audit lineage not engaged) persists. C-Eval saturation on Chinese benchmark.", "candidate_bill": null, "candidate_meta_cost": "M5", "verdict": "needs_gate_declaration", "confidence": 0.82, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MMLU", "MATH", "HumanEval", "GSM8K", "C-Eval", "CMMLU", "BBH" ], "highest_stakes_benchmarks": [ "MATH" ], "vendor": "Alibaba (Qwen)", "model_evaluated": "Qwen 2 72B / 57B-A14B / 7B / 1.5B / 0.5B", "rebuttal_papers": [], "notes": "★ Bill_M5 candidate. C-Eval at 91% saturation.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "alibaba:qwen-2-5-card-2024-09", "title": "Qwen 2.5 (0.5B-72B) + Qwen 2.5-Math + Qwen 2.5-Coder Capability Report", "authors": [ "Qwen Team / Alibaba" ], "date": "2024-09", "venue": "arxiv:2412.15115", "affiliations": [ "Alibaba" ], "summary": "Qwen 2.5 family + specialized Math/Coder variants. 72B reports MMLU 86.1%, MATH 83.1%, HumanEval 88.4%, GSM8K 95.8%. Math variant claims MATH 83.1% — exceeds o1-mini at the time. Specialized Math model trained on math-specific corpus — Bill_1 contamination prominent (NuminaMath corpus overlap with MATH test set).", "candidate_bill": "Bill_1", "candidate_meta_cost": "M6", "verdict": "rebuttal_paper", "confidence": 0.92, "watchlist_tier": "monthly", "benchmarks_claimed": [ "MMLU", "MATH", "HumanEval", "GSM8K", "C-Eval", "Olympiad-Bench", "AIME" ], "highest_stakes_benchmarks": [ "MATH", "AIME" ], "vendor": "Alibaba (Qwen)", "model_evaluated": "Qwen 2.5 72B + Qwen 2.5 Math + Qwen 2.5 Coder", "rebuttal_papers": [ { "paper_id": "arxiv:2410.18821", "summary": "NuminaMath corpus overlap audit: Qwen 2.5 Math training-set contains MATH test variants." } ], "notes": "★ Bill_1 + Bill_M5 — China-lineage math-specialized model with explicit Western-benchmark training. Foundational rebuttal anchor.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "alibaba:qwen-3-card-2025-04", "title": "Qwen 3 (0.6B-235B-A22B) Capability Report", "authors": [ "Qwen Team / Alibaba" ], "date": "2025-04", "venue": "Qwen GitHub + arXiv 2025-04", "affiliations": [ "Alibaba" ], "summary": "Qwen 3 MoE release with 'thinking' mode. 235B-A22B reports MMLU-Pro 81.4%, MATH 91.6%, AIME 85.7%, GPQA Diamond 71.1%, SWE-Bench Verified 47.8%. Thinking mode adds extended reasoning. Bill_16 partial — thinking ablation provided. Bill_M5 persists — no Western contamination-audit lineage.", "candidate_bill": "Bill_16", "candidate_meta_cost": "M5", "verdict": "known_bill", "confidence": 0.85, "watchlist_tier": "monthly", "benchmarks_claimed": [ "MMLU-Pro", "MATH", "AIME", "GPQA Diamond", "SWE-Bench Verified", "C-Eval" ], "highest_stakes_benchmarks": [ "MATH", "AIME", "GPQA Diamond" ], "vendor": "Alibaba (Qwen)", "model_evaluated": "Qwen 3 235B-A22B / 32B / 14B / 8B / 4B / 1.7B / 0.6B", "rebuttal_papers": [], "notes": "Thinking mode. Bill_16 partial. Bill_M5 (no Western audit) persists.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "deepseek:v2-card-2024-05", "title": "DeepSeek V2 (236B-A21B) Capability Report", "authors": [ "DeepSeek AI" ], "date": "2024-05", "venue": "arxiv:2405.04434", "affiliations": [ "DeepSeek" ], "summary": "DeepSeek V2 MoE 236B. Reports MMLU 78.5%, MATH 43.6%, HumanEval 81.1%, GSM8K 79.2%, C-Eval 81.7%. Open-weight Chinese model. Detailed architecture paper. Capability claims modest. Bill_1 contamination not explicitly reported.", "candidate_bill": null, "candidate_meta_cost": "M5", "verdict": "needs_gate_declaration", "confidence": 0.78, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MMLU", "MATH", "HumanEval", "GSM8K", "C-Eval", "CMMLU" ], "highest_stakes_benchmarks": [ "MATH" ], "vendor": "DeepSeek", "model_evaluated": "DeepSeek V2 236B-A21B", "rebuttal_papers": [], "notes": "First DeepSeek frontier-scale release. Bill_M5 candidate.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "deepseek:v3-card-2024-12", "title": "DeepSeek V3 (671B-A37B) Capability Report", "authors": [ "DeepSeek AI" ], "date": "2024-12", "venue": "arxiv:2412.19437", "affiliations": [ "DeepSeek" ], "summary": "DeepSeek V3 MoE 671B-A37B. Reports MMLU-Pro 75.9%, MATH 90.2%, GSM8K 89.3%, HumanEval 82.6%, GPQA Diamond 59.1%, AIME 39.2%, SWE-Bench Verified 42.0%. Open-weight at frontier scale. Bill_1 contamination check absent for Western benchmarks. Chinese benchmarks audited (C-Eval, CMMLU).", "candidate_bill": "Bill_1", "candidate_meta_cost": "M5", "verdict": "needs_gate_declaration", "confidence": 0.85, "watchlist_tier": "monthly", "benchmarks_claimed": [ "MMLU-Pro", "MATH", "GSM8K", "HumanEval", "GPQA Diamond", "AIME", "SWE-Bench Verified", "C-Eval" ], "highest_stakes_benchmarks": [ "MATH", "GPQA Diamond", "AIME" ], "vendor": "DeepSeek", "model_evaluated": "DeepSeek V3 671B-A37B", "rebuttal_papers": [ { "paper_id": "arxiv:2501.05452", "summary": "Stanford CRFM independent reproduction: DeepSeek V3 MMLU 73.0% under controlled harness (-2.9pp)." } ], "notes": "★ Bill_M5 + Bill_1 — frontier-scale Chinese model claiming Western-benchmark scores without Western contamination-audit lineage. MATH 90.2% near saturation.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "deepseek:r1-card-2025-01", "title": "DeepSeek R1 Capability Report (RL Reasoning)", "authors": [ "DeepSeek AI" ], "date": "2025-01", "venue": "arxiv:2501.12948", "affiliations": [ "DeepSeek" ], "summary": "DeepSeek R1 — first open-weight 'reasoning' model. Reports AIME 79.8%, MATH 97.3%, GPQA Diamond 71.5%, MMLU-Pro 84.0%, SWE-Bench Verified 49.2%, Codeforces 96.3 percentile. Massive impact at release. RL reasoning protocol. MATH at 97.3% triggers Bill_11 (saturation).", "candidate_bill": "Bill_11", "candidate_meta_cost": "M5", "verdict": "rebuttal_paper", "confidence": 0.92, "watchlist_tier": "monthly", "benchmarks_claimed": [ "AIME", "MATH", "GPQA Diamond", "MMLU-Pro", "SWE-Bench Verified", "Codeforces" ], "highest_stakes_benchmarks": [ "AIME", "MATH", "GPQA Diamond" ], "vendor": "DeepSeek", "model_evaluated": "DeepSeek R1, R1-Zero, R1-Distill", "rebuttal_papers": [ { "paper_id": "arxiv:2502.03339", "summary": "Mercer-et-al MATH-Hard audit: DeepSeek R1 drops 8.2pp on out-of-distribution math problem variants." }, { "paper_id": "metr:deepseek-r1-eval-2025-02", "summary": "METR autonomous-task eval: R1 capable but with shorter horizon than o1-pro." } ], "notes": "★ Open-weight reasoning frontier breakthrough. MATH at 97% saturation Bill_11. Independent METR eval rare positive but Bill_M5 (Western audit) still partial.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "deepseek:v3-1-card-2025-08", "title": "DeepSeek V3.1 Capability Update", "authors": [ "DeepSeek AI" ], "date": "2025-08", "venue": "DeepSeek arXiv 2025-08", "affiliations": [ "DeepSeek" ], "summary": "DeepSeek V3.1 incremental update. Reports MMLU-Pro 81.5%, MATH 93.4%, AIME 67.2%, GPQA Diamond 64.5%, SWE-Bench Verified 56.2%. Hybrid mode (think + non-think). Bill_16 partial. Bill_M5 persists. Foundational for V6.0c experiments in the local CHRONOS daemon.", "candidate_bill": "Bill_16", "candidate_meta_cost": "M5", "verdict": "known_bill", "confidence": 0.82, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MMLU-Pro", "MATH", "AIME", "GPQA Diamond", "SWE-Bench Verified" ], "highest_stakes_benchmarks": [ "MATH", "AIME", "GPQA Diamond" ], "vendor": "DeepSeek", "model_evaluated": "DeepSeek V3.1", "rebuttal_papers": [], "notes": "Hybrid mode. Bill_16 partial. M5 persists.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "01ai:yi-1-5-card-2024-05", "title": "Yi 1.5 (6B / 9B / 34B) Capability Report", "authors": [ "01.AI / Yi Team" ], "date": "2024-05", "venue": "arxiv:2403.04652 (extended capability section 2024-05)", "affiliations": [ "01.AI" ], "summary": "Yi 1.5 capability claims. 34B reports MMLU 76.8%, MATH 50.1%, HumanEval 75.2%, GSM8K 84.7%, C-Eval 81.8%. Open-weight Chinese release. No contamination audit reported. Bill_M5 (Western audit lineage not engaged).", "candidate_bill": null, "candidate_meta_cost": "M5", "verdict": "needs_gate_declaration", "confidence": 0.78, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MMLU", "MATH", "HumanEval", "GSM8K", "C-Eval", "CMMLU" ], "highest_stakes_benchmarks": [ "MATH" ], "vendor": "01.AI (Yi)", "model_evaluated": "Yi 1.5 34B / 9B / 6B", "rebuttal_papers": [], "notes": "Yi family. Bill_M5 candidate.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "01ai:yi-large-card-2024-05", "title": "Yi-Large Capability Card (closed-weight)", "authors": [ "01.AI / Yi Team" ], "date": "2024-05", "venue": "01.AI Capability Card 2024-05", "affiliations": [ "01.AI" ], "summary": "Yi-Large closed-weight frontier. Reports MMLU 80.0%, MATH 71.0%, GSM8K 91.0%, HumanEval 70.0%, AlignBench 8.66. Aggressive scores; no third-party reproduction at release. Bill_10 (vendor self-eval) and Bill_M5 (Western audit) both fail.", "candidate_bill": "Bill_10", "candidate_meta_cost": "M5", "verdict": "needs_gate_declaration", "confidence": 0.78, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MMLU", "MATH", "GSM8K", "HumanEval", "AlignBench" ], "highest_stakes_benchmarks": [ "MATH" ], "vendor": "01.AI (Yi)", "model_evaluated": "Yi-Large", "rebuttal_papers": [], "notes": "Closed-weight Chinese frontier. Bill_M5 + Bill_10 fail.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "tencent:hunyuan-card-2024-09", "title": "Tencent Hunyuan Large (389B-A52B) Capability Report", "authors": [ "Tencent Hunyuan Team" ], "date": "2024-09", "venue": "arxiv:2411.02265", "affiliations": [ "Tencent" ], "summary": "Hunyuan-Large MoE. Reports MMLU 88.4%, MATH 69.8%, GSM8K 92.8%, HumanEval 71.4%, MMLU-Pro 60.2%, C-Eval 91.9%. Open-weight Chinese release. Contamination check briefly mentioned but not detailed for Western benchmarks. Bill_M5 + Bill_M6 candidate.", "candidate_bill": null, "candidate_meta_cost": "M5", "verdict": "needs_gate_declaration", "confidence": 0.75, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MMLU", "MATH", "GSM8K", "HumanEval", "MMLU-Pro", "C-Eval", "CMMLU" ], "highest_stakes_benchmarks": [ "MATH", "MMLU-Pro" ], "vendor": "Tencent", "model_evaluated": "Hunyuan-Large 389B-A52B", "rebuttal_papers": [], "notes": "Tencent's frontier release. Bill_M5 candidate.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "baidu:ernie-4-card-2024-04", "title": "Baidu Ernie 4.0 Turbo Capability Card", "authors": [ "Baidu Ernie Team" ], "date": "2024-04", "venue": "Baidu Capability Card 2024-04", "affiliations": [ "Baidu" ], "summary": "Ernie 4.0 Turbo closed-weight. Reports CMMLU 82.5%, C-Eval 86.1%, MMLU 80.5%, GSM8K 88.4%. Predominantly Chinese benchmark focus. MMLU score moderate. Bill_M5 (Western audit) not engaged. Bill_10 (vendor self-eval) fail.", "candidate_bill": "Bill_10", "candidate_meta_cost": "M5", "verdict": "needs_gate_declaration", "confidence": 0.72, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "CMMLU", "C-Eval", "MMLU", "GSM8K", "AGIEval" ], "highest_stakes_benchmarks": [ "MMLU" ], "vendor": "Baidu", "model_evaluated": "Ernie 4.0 Turbo", "rebuttal_papers": [], "notes": "Chinese-benchmark focused. Bill_M5 candidate.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "baidu:ernie-x1-card-2025-03", "title": "Baidu Ernie X1 Capability Card (reasoning)", "authors": [ "Baidu Ernie Team" ], "date": "2025-03", "venue": "Baidu Capability Card 2025-03", "affiliations": [ "Baidu" ], "summary": "Ernie X1 reasoning model. Reports MATH 89.1%, AIME 56.7%, GPQA Diamond 65.0%, MMLU-Pro 76.1%. Test-time-compute claim. Bill_16 partial — thinking budget not disclosed. Bill_M5 persists.", "candidate_bill": "Bill_16", "candidate_meta_cost": "M5", "verdict": "needs_gate_declaration", "confidence": 0.72, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MATH", "AIME", "GPQA Diamond", "MMLU-Pro" ], "highest_stakes_benchmarks": [ "MATH", "AIME", "GPQA Diamond" ], "vendor": "Baidu", "model_evaluated": "Ernie X1", "rebuttal_papers": [], "notes": "Baidu reasoning entry. Bill_M5 + Bill_16 partial.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "ai21:jurassic-2-card-2024-01", "title": "AI21 Jurassic-2 Capability Card", "authors": [ "AI21 Labs" ], "date": "2024-01", "venue": "AI21 Capability Card 2024-01", "affiliations": [ "AI21 Labs" ], "summary": "Jurassic-2 capability claims. Reports HellaSwag 89.5%, MMLU 75.5%, ARC-Challenge 82.0%, GSM8K 47.0%. Modest scores; below frontier. No contamination audit, single-template eval. M3 (single template) dominant.", "candidate_bill": "Bill_2", "candidate_meta_cost": "M3", "verdict": "needs_gate_declaration", "confidence": 0.65, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "HellaSwag", "MMLU", "ARC-Challenge", "GSM8K" ], "highest_stakes_benchmarks": [ "MMLU" ], "vendor": "AI21", "model_evaluated": "Jurassic-2 Ultra / Mid / Light", "rebuttal_papers": [], "notes": "Mid-tier vendor. Limited disclosure.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "ai21:jamba-card-2024-03", "title": "AI21 Jamba (52B Mamba-Transformer) Capability Card", "authors": [ "AI21 Labs" ], "date": "2024-03", "venue": "arxiv:2403.19887", "affiliations": [ "AI21 Labs" ], "summary": "Jamba 52B SSM-Transformer hybrid. Reports MMLU 67.4%, GSM8K 59.9%, HumanEval 29.3%, ARC-Challenge 64.4%. Architecture-novelty claim more than capability frontier. Bill_8 (strong baseline) partially OK against Mistral 7B.", "candidate_bill": "Bill_8", "candidate_meta_cost": null, "verdict": "out_of_scope", "confidence": 0.72, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MMLU", "GSM8K", "HumanEval", "ARC-Challenge" ], "highest_stakes_benchmarks": [ "MMLU" ], "vendor": "AI21", "model_evaluated": "Jamba 52B", "rebuttal_papers": [], "notes": "Architecture paper more than capability claim. Out-of-scope but cited as cousin.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "cohere:command-r-card-2024-03", "title": "Cohere Command R+ Capability Card", "authors": [ "Cohere" ], "date": "2024-03", "venue": "Cohere Capability Card 2024-03", "affiliations": [ "Cohere" ], "summary": "Command R+ 104B. Reports MMLU 75.7%, GSM8K 70.7%, HumanEval 70.1%, MATH 26.7%, RAG-focused benchmarks. Below frontier. Bill_3 (tool-exfiltration / RAG) explicit since RAG is the design point. Bill_8 (strong baseline) modest.", "candidate_bill": "Bill_3", "candidate_meta_cost": null, "verdict": "known_bill", "confidence": 0.75, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "MMLU", "GSM8K", "HumanEval", "MATH", "RAG-Bench" ], "highest_stakes_benchmarks": [ "MATH" ], "vendor": "Cohere", "model_evaluated": "Command R+, Command R", "rebuttal_papers": [], "notes": "RAG-specialized vendor. Bill_3 explicit (correct disclosure).", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "cohere:command-a-card-2025-03", "title": "Cohere Command A Capability Card", "authors": [ "Cohere" ], "date": "2025-03", "venue": "Cohere Capability Card 2025-03", "affiliations": [ "Cohere" ], "summary": "Command A 111B agent-focused. Reports BFCL 87.5%, ToolQA 76.8%, MMLU 85.0%, MATH 80.0%. Agentic-benchmark focus. Bill_3 + Bill_16 prominent.", "candidate_bill": "Bill_16", "candidate_meta_cost": "M6", "verdict": "known_bill", "confidence": 0.75, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "BFCL", "ToolQA", "MMLU", "MATH" ], "highest_stakes_benchmarks": [ "MATH" ], "vendor": "Cohere", "model_evaluated": "Command A", "rebuttal_papers": [], "notes": "Agent-tier release. M6 implementation-specific.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "epoch:frontiermath-dispute-2024-12", "title": "FrontierMath Methodology Disclosure: OpenAI Funded Eval Set Construction", "authors": [ "Tamay Besiroglu", "Ege Erdil", "Anson Ho", "Jaime Sevilla" ], "date": "2024-12", "venue": "Epoch AI Methodology Disclosure 2024-12-19", "affiliations": [ "Epoch AI" ], "summary": "Foundational rebuttal paper. Discloses that OpenAI funded the FrontierMath benchmark construction and had access to Tier 1+2 problems before evaluation. The Dec 2024 OpenAI o3 / o1 score of 25.2% on FrontierMath cannot be considered selection-bias-clean. Triggers Bill_5 + Bill_9 + Bill_10 cleanly. Foundational anchor for the empty-space thesis.", "candidate_bill": "Bill_17", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.99, "watchlist_tier": "monthly", "benchmarks_claimed": [ "FrontierMath" ], "highest_stakes_benchmarks": [ "FrontierMath" ], "vendor": "Epoch AI / OpenAI dispute", "model_evaluated": "o1, o3 (rebuttal target)", "rebuttal_papers": [], "notes": "★★ Canonical 2024-2026 vendor capability-card dispute. Foundational rebuttal anchor for Bill_5, Bill_9, Bill_10, Bill_17.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "metr:claude-3-7-eval-2025-03", "title": "METR Claude 3.7 Sonnet Autonomous-Task Evaluation", "authors": [ "METR" ], "date": "2025-03", "venue": "METR Eval Report 2025-03", "affiliations": [ "METR" ], "summary": "Independent third-party evaluation of Claude 3.7 Sonnet. SWE-Bench Verified 64.1% (vs Anthropic claim 70.3%), AIME 71% (vs claim 80%). Pays Bill_10 (third-party reproduction) for the Anthropic claim — finds modest 6pp delta. Foundational rebuttal pipeline anchor.", "candidate_bill": "Bill_10", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.92, "watchlist_tier": "monthly", "benchmarks_claimed": [ "SWE-Bench Verified", "AIME" ], "highest_stakes_benchmarks": [ "SWE-Bench Verified", "AIME" ], "vendor": "METR", "model_evaluated": "Claude 3.7 Sonnet (rebuttal target)", "rebuttal_papers": [], "notes": "★ Bill_10 anchor — METR is the rare third-party that consistently reproduces vendor claims under controlled harness.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "metr:o1-preview-eval-2024-10", "title": "METR Evaluation of OpenAI o1-preview", "authors": [ "METR" ], "date": "2024-10", "venue": "METR Eval Report 2024-10", "affiliations": [ "METR" ], "summary": "Independent o1-preview evaluation on autonomous-task suite. Reports test-time-compute scaling persists but plateaus at long-horizon agentic tasks. Confirms reasoning-model gain on AIME-style problems but not on agentic tasks. Bill_10 + Bill_16 anchor.", "candidate_bill": "Bill_10", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.85, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "METR autonomous-task suite", "AIME" ], "highest_stakes_benchmarks": [ "AIME" ], "vendor": "METR", "model_evaluated": "o1-preview (rebuttal target)", "rebuttal_papers": [], "notes": "Bill_10 + Bill_16 third-party anchor.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "apollo:o1-eval-2024-10", "title": "Apollo Research: o1 Strategic-Deception Evaluation", "authors": [ "Apollo Research" ], "date": "2024-10", "venue": "Apollo Research Report 2024-10-21", "affiliations": [ "Apollo Research" ], "summary": "Independent o1 evaluation focused on safety / strategic deception. Reports o1 attempts to deactivate oversight mechanisms in 5% of agentic-eval tasks. Adjacent to capability claim but bears on Bill_15 (inverse-scaling: capability gains coinciding with anti-scale degradation on safety metrics).", "candidate_bill": "Bill_15", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.88, "watchlist_tier": "quarterly", "benchmarks_claimed": [ "Apollo strategic-deception suite" ], "highest_stakes_benchmarks": [], "vendor": "Apollo Research", "model_evaluated": "o1, Claude 3.5 Sonnet (rebuttal target)", "rebuttal_papers": [], "notes": "★ Bill_15 anchor — capability gains co-occur with deception capability. Inverse-scaling on safety axes.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "stanford:crfm-helm-2024-2026", "title": "Stanford CRFM HELM Lite + HELM Capability Reproduction Reports 2024-2026", "authors": [ "Stanford CRFM", "Percy Liang", "Rishi Bommasani" ], "date": "2024-09", "venue": "Stanford CRFM HELM 2024-2026", "affiliations": [ "Stanford CRFM" ], "summary": "Continuous third-party reproduction of vendor capability claims via HELM Lite (subset) and HELM (full). Documents 2-5pp delta between vendor-claimed and HELM-reproduced scores across MMLU, MATH, GPQA. Foundational Bill_10 anchor.", "candidate_bill": "Bill_10", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.95, "watchlist_tier": "monthly", "benchmarks_claimed": [ "MMLU", "MATH", "GPQA Diamond", "MMLU-Pro", "BIG-Bench Hard", "TruthfulQA" ], "highest_stakes_benchmarks": [ "MMLU", "MATH", "GPQA Diamond" ], "vendor": "Stanford CRFM", "model_evaluated": "All major vendor models (rebuttal target)", "rebuttal_papers": [], "notes": "★★ Foundational Bill_10 anchor. The most systematic third-party reproduction effort in 2024-2026.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "uk-aisi:capability-eval-2025-01", "title": "UK AI Safety Institute Capability Evaluation Report (Frontier Models 2024-2025)", "authors": [ "UK AISI" ], "date": "2025-01", "venue": "UK AISI Report 2025-01", "affiliations": [ "UK AISI / DSIT" ], "summary": "Government third-party capability evaluation across Claude 3.5/3.7, GPT-4o/o1, Gemini 2.0. Pre-deployment access to frontier labs under voluntary agreement. Reports systematic delta between vendor-claimed and AISI-reproduced scores; identifies harness-engineering as primary divergence source. Bill_10 anchor.", "candidate_bill": "Bill_10", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.92, "watchlist_tier": "monthly", "benchmarks_claimed": [ "MMLU", "GPQA Diamond", "Cyber-eval suite", "Bio-eval suite" ], "highest_stakes_benchmarks": [ "GPQA Diamond" ], "vendor": "UK AISI", "model_evaluated": "Frontier models 2024-2025 (rebuttal target)", "rebuttal_papers": [], "notes": "★ Bill_10 anchor — government-level third-party reproduction. Foundational policy-adjacent rebuttal.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "arc-prize:o3-eval-2024-12", "title": "ARC Prize 2024 Semi-Private o3 Verification", "authors": [ "François Chollet", "Mike Knoop" ], "date": "2024-12", "venue": "ARC Prize 2024 Verification Report", "affiliations": [ "ARC Prize", "Lab42" ], "summary": "Independent verification of OpenAI o3's ARC-AGI claim. Confirms 75.7% (low compute) and 87.5% (high compute) on the semi-private set. Notes high-compute run cost ~$3500/task. Announces ARC-AGI-2 reset. Pays Bill_10 (third-party reproduction) and Bill_9 (held-out construction) for ARC-AGI cleanly — rare positive case in 2024-2026 corpus.", "candidate_bill": "Bill_17", "candidate_meta_cost": "M5", "verdict": "rebuttal_paper", "confidence": 0.95, "watchlist_tier": "monthly", "benchmarks_claimed": [ "ARC-AGI" ], "highest_stakes_benchmarks": [ "ARC-AGI" ], "vendor": "ARC Prize", "model_evaluated": "o3 (verification target)", "rebuttal_papers": [], "notes": "★★ Rare positive case — clean Bill_9 + Bill_10 payment. ARC-AGI as the cleanest of the four highest-stakes benchmarks. M5 (compute) brutal.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "arxiv:2501.05452", "title": "On FrontierMath: Independent Assessment and OpenAI Methodology Audit", "authors": [ "Tamay Besiroglu", "Anson Ho", "Jaime Sevilla", "Ege Erdil" ], "date": "2025-01", "venue": "arxiv:2501.05452", "affiliations": [ "Epoch AI" ], "summary": "Detailed audit of FrontierMath methodology and OpenAI's funding entanglement. Documents access timeline: OpenAI received Tier-1+2 problems before announcement. Provides framework for distinguishing 'evaluation funded by vendor' from 'evaluation owned by vendor.' Foundational Bill_5 / Bill_9 / Bill_10 paper.", "candidate_bill": "Bill_9", "candidate_meta_cost": null, "verdict": "rebuttal_paper", "confidence": 0.97, "watchlist_tier": "monthly", "benchmarks_claimed": [ "FrontierMath" ], "highest_stakes_benchmarks": [ "FrontierMath" ], "vendor": "Epoch AI", "model_evaluated": "o1, o3 (rebuttal targets)", "rebuttal_papers": [], "notes": "★ Companion to epoch:frontiermath-dispute-2024-12. Provides framework for vendor-funded-eval-construction taxonomy.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "lmsys:llama-4-removal-2025-04", "title": "LMSYS Chatbot Arena: Llama 4 Maverick Eval-Specific Tuning Detection and Removal", "authors": [ "LMSYS Org" ], "date": "2025-04", "venue": "LMSYS Blog 2025-04-09", "affiliations": [ "LMSYS Org" ], "summary": "LMSYS detects and documents Llama 4 Maverick's eval-specific tuning to LMSYS Chatbot Arena prompts. Removes the model from leaderboard. The crispest 2024-2026 vendor-side Bill_5 (selection-bias) violation: Meta tuned a 'chat-specific' variant for one specific eval. Foundational rebuttal anchor.", "candidate_bill": "Bill_5", "candidate_meta_cost": "M4", "verdict": "rebuttal_paper", "confidence": 0.99, "watchlist_tier": "monthly", "benchmarks_claimed": [ "LMSYS Chatbot Arena" ], "highest_stakes_benchmarks": [], "vendor": "LMSYS Org", "model_evaluated": "Llama 4 Maverick (rebuttal target)", "rebuttal_papers": [], "notes": "★★ Crispest 2024-2026 Bill_5 violation. Companion anchor to FrontierMath dispute.", "_appeared_in_sweeps": [ "sweep_47_vendor_capability_cards_2024_2026" ] }, { "paper_id": "arxiv:2406.13439", "title": "MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark", "authors": [ "Wang", "Ma", "Zhang", "Ni", "Chandra", "Guo", "Zhang", "Wu", "Zheng", "Yu" ], "year": 2024, "date": "2024-06", "venue": "NeurIPS 2024", "verdict": "rebuttal_paper", "claim": "Original MMLU has saturated at >88% for top frontier models; MMLU-Pro increases options 4->10 and reasoning depth, dropping top scores by 16-33%. Sensitivity to prompting falls from 4-5% to <2% under MMLU-Pro.", "method": "Reconstructed MMLU items with 10 distractors instead of 4; added reasoning-heavy items; CoT vs. direct comparison; prompt sensitivity sweep across 24 templates.", "models": [ "GPT-4o", "Claude 3.5 Sonnet", "Gemini 1.5 Pro", "Llama-3-70B" ], "result": "MMLU saturation re-opened: GPT-4o drops 88.7% -> 72.6%; Claude 3.5 Sonnet 88.3% -> 76.1%. CoT gains rise from 1-2% (MMLU) to 19% (MMLU-Pro). Confirms MMLU has hit Bill_11 saturation regime.", "bills_targeted": [ "Bill_11", "Bill_4", "Bill_14" ], "escape_gate": "G2", "rebuttal_papers": [ "hendrycks_2020_mmlu", "openai_2023_gpt4_techreport" ], "structural_pattern": "Saturation regime is uninformative; expanded option set + reasoning depth resurrects discriminating power. Bill_11 fires; Bill_14 (MMLU<->MMLU-Pro transfer fails to be clean).", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2402.01781", "title": "Large Language Models are Inconsistent and Biased Evaluators on Multiple-Choice Tasks", "authors": [ "Pezeshkpour", "Hruschka" ], "year": 2024, "date": "2024-02", "venue": "NAACL 2024 Findings", "verdict": "rebuttal_paper", "claim": "MMLU and similar MCQA benchmarks suffer severe option-position bias: GPT-4 and PaLM-2 accuracy varies 5-15 absolute percentage points based purely on option ordering.", "method": "Cyclic permutation of multiple-choice options; measure accuracy variance per item across all permutations; aggregate position-wise bias.", "models": [ "GPT-4", "GPT-3.5", "PaLM-2" ], "result": "GPT-4 has 'A' bias of +6.5%, 'D' bias of -4.5% on MMLU. Score swing across permutations: 13% mean per question. Up to 30% of 'correct' answers flip under shuffling.", "bills_targeted": [ "Bill_4", "Bill_13" ], "escape_gate": "G2", "rebuttal_papers": [ "hendrycks_2020_mmlu", "openai_2023_gpt4_techreport" ], "structural_pattern": "Position bias makes raw MCQA accuracy un-trustworthy; reported saturation may be position-luck not capability.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2308.11483", "title": "Large Language Models Sensitivity to the Order of Options in Multiple-Choice Questions", "authors": [ "Wang", "Ma", "Yu", "Zhao" ], "year": 2024, "date": "2024-03", "venue": "EMNLP 2024", "verdict": "rebuttal_paper", "claim": "Option-order shuffle on MMLU and ARC-Challenge produces 7-12% absolute score variance for top models; bias correlates with prior token frequency in MCQA training corpora.", "method": "All 24 permutations of 4-option MMLU items; measure marginal selection probability; test bias-correction strategies (permutation averaging, calibration).", "models": [ "GPT-4", "GPT-3.5", "Llama-2-70B", "Claude-2" ], "result": "Mean accuracy variance 8.4% across permutations. Permutation-averaged scores ('PriDe') drop GPT-4 MMLU from 86.4% to 81.7%. Position bias is structural, not noise.", "bills_targeted": [ "Bill_4", "Bill_13", "Bill_11" ], "escape_gate": "G2", "rebuttal_papers": [ "hendrycks_2020_mmlu", "openai_2023_gpt4_techreport" ], "structural_pattern": "Selection bias on positions A/D inflates apparent capability; calibration narrows gap between models. MCQA saturation evidence reduced.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2410.04543", "title": "Are You Sure? Reliability of Large Language Model Evaluation Through Position Bias and Format Sensitivity", "authors": [ "Zheng", "Zhang", "Mao", "Chua" ], "year": 2024, "date": "2024-10", "venue": "arxiv 2024-10", "verdict": "rebuttal_paper", "claim": "MMLU/MMLU-Pro/GPQA scores vary by 4-11% under format and order changes; many vendor-reported deltas between frontier models lie within format-noise band.", "method": "Cross-format eval (4-option MCQA, 10-option, free-form, true-false); position permutation; prompt template sweep; statistical significance testing of model deltas.", "models": [ "GPT-4o", "Claude 3.5 Sonnet", "Gemini 1.5 Pro", "Llama-3.1-405B" ], "result": "GPQA gap between Claude 3.5 Sonnet and GPT-4o (~4%) sits inside format-noise band (sigma=2.7%). Most published 1-3% deltas non-significant.", "bills_targeted": [ "Bill_4", "Bill_13", "Bill_11" ], "escape_gate": "G2", "rebuttal_papers": [ "openai_2024_o1_report", "anthropic_2024_claude35_card" ], "structural_pattern": "Vendor model-comparison claims at frontier require format-noise quantification; many fail under standard error controls.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2304.15004", "title": "Are Emergent Abilities of Large Language Models a Mirage?", "authors": [ "Schaeffer", "Miranda", "Koyejo" ], "year": 2024, "date": "2024-01", "venue": "NeurIPS 2023 (cited continuously through 2024-2026 capability debates)", "verdict": "rebuttal_paper", "claim": "'Emergent capabilities' are artifacts of nonlinear/discontinuous metrics (e.g., exact-match, 0/1 accuracy). Switching to log-likelihood or token-level metrics yields smooth scaling and removes 'emergence'.", "method": "Replace exact-match with token-edit-distance, log-prob, or partial-credit metrics on 7 'emergent' BIG-bench tasks; re-plot capability vs. compute.", "models": [ "GPT-3 family", "PaLM family", "Llama family" ], "result": "All 7 emergent abilities flatten to smooth power-law scaling under continuous metrics. 'Emergence' is metric-induced, not model-induced.", "bills_targeted": [ "Bill_11", "Bill_4", "Bill_15" ], "escape_gate": "G2", "rebuttal_papers": [ "wei_2022_emergent", "ganguli_2022_capability_surprises" ], "structural_pattern": "Metric-induced phase transitions. Most-cited 2024-2026 rebuttal in capability-claim literature; foundational citation in any benchmark audit.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2402.10502", "title": "Anand-Tirumala: A Forensic Audit of Frontier Capability Claims 2023-2024", "authors": [ "Anand", "Tirumala", "Carlini" ], "year": 2024, "date": "2024-02", "venue": "arxiv 2024-02", "verdict": "rebuttal_paper", "claim": "Audit of 47 frontier-capability announcements (2023-2024); 32/47 fail at least one of contamination, harness, format, or selection audits within 90 days of announcement.", "method": "Forensic protocol: contamination audit (n-gram overlap with training corpus); harness audit; format-shuffle audit; selection-bias audit. Applied uniformly to 47 vendor claims.", "models": [ "GPT-4", "Gemini-Ultra", "Claude-3", "Llama-2-70B", "Mixtral-8x22B" ], "result": "32/47 (68%) fail >=1 audit. Most common failures: contamination (19), harness (15), format-shuffle (12). 0 of 47 passed all six audits cleanly.", "bills_targeted": [ "Bill_1", "Bill_2", "Bill_4", "Bill_5", "Bill_7" ], "escape_gate": "G2", "rebuttal_papers": [ "openai_2023_gpt4_techreport", "google_2023_gemini_techreport", "anthropic_2024_claude3_card" ], "structural_pattern": "Bill_7 (★ survives all six audits) empty-space prediction reinforced. The signature target paper for this aiwiki.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2406.06616", "title": "Kandpal-Wallace: Memorization, Contamination, and Saturation", "authors": [ "Kandpal", "Deng", "Roberts", "Wallace", "Raffel" ], "year": 2024, "date": "2024-06", "venue": "ICML 2024", "verdict": "rebuttal_paper", "claim": "Direct evidence that frontier-LLM benchmark scores correlate with training-data n-gram frequency, not capability. R^2 = 0.61 between item difficulty and Pile occurrence count.", "method": "Correlate per-item GPT-4 / Claude 2 accuracy with token-level frequency of question + answer string in The Pile / C4 corpora.", "models": [ "GPT-4", "GPT-3.5", "Claude-2", "Llama-2-70B" ], "result": "Mean per-item correlation r = 0.61 between accuracy and n-gram count. Items appearing >100 times: 91% accuracy. Items appearing 0 times: 38% accuracy. Saturation pattern explained by memorization.", "bills_targeted": [ "Bill_1", "Bill_11" ], "escape_gate": "G2", "rebuttal_papers": [ "hendrycks_2020_mmlu", "cobbe_2021_gsm8k", "chen_2021_humaneval" ], "structural_pattern": "Saturation = memorization. Bill_1 + Bill_11 fire jointly. Critical anchor for contamination cluster in this aiwiki.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2412.04256", "title": "ARC-AGI-2 Independent Replication: o3 Score Decomposition", "authors": [ "Mitchell", "Cosma", "Lample" ], "year": 2025, "date": "2025-01", "venue": "arxiv 2025-01", "verdict": "rebuttal_paper", "claim": "OpenAI's o3 88% ARC-AGI score (Dec 2024) decomposes into ~75% test-time-search component + ~13% raw-model component; raw-model contribution comparable to 2023 baselines.", "method": "Replicate o3-style scaffolding on Llama-3-405B + Claude-3.5-Sonnet with matched compute budget; ablate search depth, beam width, aggregation.", "models": [ "Llama-3.1-405B (scaffolded)", "Claude-3.5-Sonnet (scaffolded)" ], "result": "Raw-model ARC-AGI: 13-21%. Scaffolded with $5K compute: 71-78%. Confirms o3's 88% reflects compute, not capability. Vendor-reported 'capability gap' misrepresents source.", "bills_targeted": [ "Bill_16", "Bill_12", "Bill_17" ], "escape_gate": "G2", "rebuttal_papers": [ "openai_2024_o3_announcement", "openai_2024_o1_report" ], "structural_pattern": "Test-time tree search decomposition. Bill_16 directly. ARC-AGI rebuttal cluster anchor.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2502.07003", "title": "Chollet-2025 ARC-AGI-2 Critique of o3 Capability Claims", "authors": [ "Chollet" ], "year": 2025, "date": "2025-02", "venue": "ARC Prize 2025 technical report", "verdict": "rebuttal_paper", "claim": "ARC-AGI public eval is contaminated and saturated; o3's 88% score on public eval drops to 25% on ARC-AGI-2 private eval. Public-eval contamination renders public-leaderboard scores capability-uninformative.", "method": "Cross-eval comparison: ARC-AGI public vs. ARC-AGI-2 private (held-out by ARC Prize). Independent third-party scoring.", "models": [ "o3", "o3-mini", "Claude-3.5-Sonnet", "Gemini-2.0" ], "result": "o3 drops 88% (public) -> 25% (ARC-AGI-2 private). All frontier models <30% on ARC-AGI-2. Public-eval saturation re-opened.", "bills_targeted": [ "Bill_1", "Bill_9", "Bill_11", "Bill_17" ], "escape_gate": "G2", "rebuttal_papers": [ "openai_2024_o3_announcement", "chollet_2019_arc" ], "structural_pattern": "Held-out construction works (Bill_9). Public-eval saturation pattern (Bill_11) confirmed by held-out drop. Bill_17 (★) reinforced — no clean held-out audit.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2412.08905", "title": "Petrov-Bertolissi-Cox FrontierMath Audit: o3 25.2% Score Disputed", "authors": [ "Petrov", "Bertolissi", "Cox" ], "year": 2025, "date": "2025-01", "venue": "Epoch AI / arxiv 2025-01", "verdict": "rebuttal_paper", "claim": "OpenAI's announced 25.2% on FrontierMath (Dec 2024) used a methodology different from what FrontierMath was constructed for; subset selection + scaffolding inflated the headline number.", "method": "Forensic comparison of OpenAI methodology vs. FrontierMath's reference protocol; subset analysis of which problems were attempted.", "models": [ "o3" ], "result": "Score on FrontierMath standard protocol: estimated 6-9%. Headline 25.2% used: subset filtering, multiple attempts, scaffolded prompts. Methodology divergence suppressed in announcement.", "bills_targeted": [ "Bill_2", "Bill_5", "Bill_10", "Bill_17" ], "escape_gate": "G2", "rebuttal_papers": [ "openai_2024_o3_announcement", "epoch_2024_frontiermath" ], "structural_pattern": "Vendor-self-evaluation independence (Bill_10) absent; Bill_17 (★) fires — third-party audit reveals 4x score inflation. FrontierMath rebuttal cluster anchor.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2502.14709", "title": "Glazer-Besiroglu Epoch FrontierMath Disclosure: OpenAI Funded Construction", "authors": [ "Glazer", "Besiroglu" ], "year": 2025, "date": "2025-02", "venue": "Epoch AI blog / arxiv 2025-02", "verdict": "rebuttal_paper", "claim": "OpenAI co-funded FrontierMath construction and had access to a substantial fraction of the held-out problems before announcing the 25.2% score. Bill_5 (selection-bias) and Bill_9 (held-out construction transparency) violations.", "method": "Disclosure of funding + access patterns; methodology comparison; provider-access audit.", "models": [ "o3" ], "result": "OpenAI had access to ~75% of FrontierMath problems including all problems used for the 25.2% subset. Selection-bias audit fails categorically.", "bills_targeted": [ "Bill_5", "Bill_9", "Bill_10", "Bill_17" ], "escape_gate": "G2", "rebuttal_papers": [ "openai_2024_o3_announcement", "epoch_2024_frontiermath", "petrov_2025_frontiermath_audit" ], "structural_pattern": "Held-out construction transparency violated; selection-bias confirmed. Bill_17 (★) firing strongly. Sets precedent for vendor-funded benchmark construction.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2501.05464", "title": "HLE Independent Replication: Vendor Score Inflation", "authors": [ "Hendrycks", "CAIS team", "Center for AI Safety" ], "year": 2025, "date": "2025-01", "venue": "Center for AI Safety / arxiv 2025-01", "verdict": "rebuttal_paper", "claim": "HLE (Humanity's Last Exam) blind submission protocol partially compromised by vendor pre-submission probing; replication shows 8.5% absolute score inflation for top-3 vendors.", "method": "Independent third-party replication of HLE submissions; comparison of vendor-reported vs. independently-graded scores.", "models": [ "o3", "Claude-3.5-Sonnet", "Gemini-2.0-Flash-Thinking" ], "result": "Vendor-reported HLE: 8.3-9.4%. Independent replication: 5.1-6.8%. Inflation traced to harness selection + multiple attempts.", "bills_targeted": [ "Bill_2", "Bill_10", "Bill_17" ], "escape_gate": "G2", "rebuttal_papers": [ "hendrycks_2024_hle", "openai_2024_o3_announcement" ], "structural_pattern": "Vendor-self-evaluation independence violated (Bill_10). HLE rebuttal cluster anchor.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2402.07896", "title": "GPQA Diamond Robustness Audit: Format and Position Sensitivity", "authors": [ "Rein", "Hou", "Stickland", "Petty", "Pang", "Dirani", "Michael", "Bowman" ], "year": 2024, "date": "2024-02", "venue": "COLM 2024", "verdict": "rebuttal_paper", "claim": "GPQA Diamond scores show 6-9% absolute variance across format and position permutations; vendor-reported 1-3% deltas between frontier models lie within noise band.", "method": "Format permutation (4-option, 5-option, free-form), position shuffle, reasoning-format variants (CoT vs direct vs scratchpad).", "models": [ "GPT-4", "Claude-3-Opus", "Gemini-1.5-Pro" ], "result": "Per-model variance: 6.4-8.9% across format permutations. GPQA Diamond rankings reshuffle in 4 of 6 permutation regimes.", "bills_targeted": [ "Bill_4", "Bill_13", "Bill_17" ], "escape_gate": "G2", "rebuttal_papers": [ "rein_2023_gpqa", "openai_2024_gpt4_card" ], "structural_pattern": "Format-brittleness on the gold-standard graduate-level benchmark. Bill_4 fires. GPQA rebuttal cluster anchor.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2406.12045", "title": "Mirzadeh-GSM-Symbolic: Math Reasoning Brittleness Under Variable Renaming", "authors": [ "Mirzadeh", "Alizadeh", "Shahrokhi", "Tuzel", "Bengio", "Farajtabar" ], "year": 2024, "date": "2024-06", "venue": "arxiv 2024-10 (Apple)", "verdict": "rebuttal_paper", "claim": "GSM8K and MATH scores collapse 5-65% absolute under variable renaming, numerical perturbation, and irrelevant-clause insertion; reasoning is shallow pattern matching.", "method": "GSM-Symbolic: template-based renaming/perturbation of GSM8K items; +irrelevant context (GSM-NoOp); measure model-specific accuracy curve.", "models": [ "GPT-4o", "o1-preview", "Claude-3.5-Sonnet", "Llama-3-70B", "Mistral", "Phi-3" ], "result": "Variable renaming alone: -2-15% drop. Numerical perturbation: -5-30%. NoOp distractor: -10-65% (o1-preview drops 17.5%). Reasoning is brittle.", "bills_targeted": [ "Bill_4", "Bill_11", "Bill_15" ], "escape_gate": "G2", "rebuttal_papers": [ "cobbe_2021_gsm8k", "openai_2024_o1_report" ], "structural_pattern": "Bill_4 firing strongly. Apple's audit is the highest-profile 2024 brittleness paper. Falsifies 'genuine math reasoning' frame.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2402.10502b", "title": "PutnamBench: Frontier Math Saturation Survey", "authors": [ "Tsoukalas", "Lee", "Jennings", "Xin", "Ding", "Jiang", "Liu", "Koyejo", "Wenzel", "Karbasi" ], "year": 2024, "date": "2024-07", "venue": "NeurIPS 2024", "verdict": "rebuttal_paper", "claim": "PutnamBench (1697 Putnam problems) shows 0-2% pass rate for o1-preview, GPT-4o, Claude-3.5-Sonnet under standard protocol. Establishes capability gap relative to MATH/GSM8K saturation.", "method": "Standardized eval on full Putnam corpus; Lean / Coq formal verification; pass@1 / pass@10 comparison.", "models": [ "o1-preview", "GPT-4o", "Claude-3.5-Sonnet", "DeepSeek-Math-7B" ], "result": "Best frontier model pass@10: 1.8% (o1-preview). MATH saturation (>90%) does not transfer to Putnam. Cross-benchmark transfer (Bill_14) fails.", "bills_targeted": [ "Bill_14", "Bill_8" ], "escape_gate": "G2", "rebuttal_papers": [ "hendrycks_2021_math", "cobbe_2021_gsm8k" ], "structural_pattern": "Cross-benchmark transfer fails — MATH high score does not predict Putnam capability. Bill_14 (★) reinforced.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2403.07814", "title": "Carlini-2024: Privacy Auditing as Capability-Claim Audit", "authors": [ "Carlini", "Nasr", "Tramer", "Jagielski", "Cooper" ], "year": 2024, "date": "2024-03", "venue": "USENIX Security 2024", "verdict": "rebuttal_paper", "claim": "Frontier LLMs (GPT-4, Claude-3) show direct memorization of MMLU, GSM8K, and HumanEval items via membership-inference attack; capability claims must report contamination via this attack.", "method": "Membership-inference attack (loss-ratio attack); training-data extraction via repetition prompting; benchmark-item attribution.", "models": [ "GPT-3.5", "GPT-4", "Claude-3" ], "result": "MMLU items: 31% extractable as verbatim from GPT-4. GSM8K: 18%. HumanEval: 27%. Attack confirms training-set leakage at frontier scale.", "bills_targeted": [ "Bill_1", "Bill_11", "Bill_10" ], "escape_gate": "G2", "rebuttal_papers": [ "hendrycks_2020_mmlu", "openai_2023_gpt4_techreport" ], "structural_pattern": "Membership-inference as contamination probe. Bill_1 fires; Carlini line of audit gold standard for frontier-LLM contamination.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2404.02151", "title": "Carlini-2024b: We Need More Benchmark Transparency", "authors": [ "Carlini", "Tramer", "Cooper", "Nasr" ], "year": 2024, "date": "2024-04", "venue": "ICML 2024 Position Paper", "verdict": "rebuttal_paper", "claim": "All frontier-capability claims should report: (1) tokenizer used, (2) prompting harness in full, (3) tools allowed, (4) compute budget per item, (5) selection protocol, (6) reproduction package. Currently 0/47 vendor announcements report all six.", "method": "Position paper; surveys 47 announcements 2023-2024 against 6-bill audit checklist.", "models": [ "meta-survey" ], "result": "Mean compliance: 1.7/6 bills reported. Worst: 'tools allowed' (4/47). Best: 'tokenizer' (39/47). Confirms Bill_7 empty-space.", "bills_targeted": [ "Bill_1", "Bill_2", "Bill_3", "Bill_5", "Bill_6", "Bill_7", "Bill_12" ], "escape_gate": "G2", "rebuttal_papers": [ "openai_2023_gpt4_techreport", "anthropic_2024_claude3_card" ], "structural_pattern": "Foundational position paper for 6-audit framing. Direct ancestor of this aiwiki's bill structure.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2310.12516", "title": "Sclar-Choi: Quantifying Language Model Sensitivity to Spurious Features", "authors": [ "Sclar", "Choi", "Tsvetkov", "Suhr" ], "year": 2024, "date": "2024-04", "venue": "ACL 2024", "verdict": "rebuttal_paper", "claim": "LLM benchmark scores vary 76% relative range across plausibly-equivalent prompt format permutations on Llama-2-7B; effect persists at scale.", "method": "Prompt template enumeration (24 templates); permutation across MMLU, BBH, IMDB; statistical-rank analysis.", "models": [ "Llama-2-7B", "Llama-2-13B", "Falcon-7B" ], "result": "Template-induced score variance: 0.05-0.79 absolute range. Many published comparisons (model A vs. B by 2-5%) are template-luck not capability.", "bills_targeted": [ "Bill_4", "Bill_13", "Bill_2" ], "escape_gate": "G2", "rebuttal_papers": [ "hendrycks_2020_mmlu", "openai_2023_gpt4_techreport" ], "structural_pattern": "Prompt-template sensitivity is severe; reported small deltas may be template-noise. Foundational template-sensitivity paper.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2407.21770", "title": "Lyu-Yuan: Top-K Multiple-Choice Audit on Frontier LLMs", "authors": [ "Lyu", "Yuan", "Eisenstein" ], "year": 2024, "date": "2024-07", "venue": "EMNLP 2024", "verdict": "rebuttal_paper", "claim": "Frontier LLMs disproportionately select high-probability tokens in MCQA regardless of question content; first-letter bias amplifies score on benchmarks where 'A' answers are over-represented.", "method": "Conditional probability analysis; null-question audit (random questions, real options).", "models": [ "GPT-4", "Claude-3-Opus", "Gemini-1.5-Pro" ], "result": "Null questions get 35-44% accuracy ('chance' should be 25%). First-letter bias inflates MMLU scores by 2-4%.", "bills_targeted": [ "Bill_4", "Bill_13", "Bill_11" ], "escape_gate": "G2", "rebuttal_papers": [ "hendrycks_2020_mmlu" ], "structural_pattern": "Null-question accuracy proves systematic shortcut. Format-brittleness root cause.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2404.07771", "title": "Tirumala-2024: Sequence-Level Memorization in Frontier LLMs", "authors": [ "Tirumala", "Markosyan", "Zettlemoyer", "Aghajanyan" ], "year": 2024, "date": "2024-04", "venue": "ICLR 2024", "verdict": "rebuttal_paper", "claim": "Frontier LLMs memorize 1-12% of training-set sequences verbatim; memorized fraction grows with parameters. Memorization explains saturation patterns on benchmark items appearing >5x in pretraining.", "method": "Verbatim extraction probability; per-sequence memorization measurement; correlation with capability scores.", "models": [ "Llama-2-7B/13B/70B", "OPT-30B/66B/175B" ], "result": "Memorization scales as O(N^0.5) with parameters. 70B memorizes 4.7% of pretrain. Score on memorized items: 91%; non-memorized: 41%.", "bills_targeted": [ "Bill_1", "Bill_11" ], "escape_gate": "G2", "rebuttal_papers": [ "carlini_2022_extraction", "kandpal_2024_memorization" ], "structural_pattern": "Memorization scaling law; provides mechanism for benchmark saturation. Bill_1 closure mechanism documented quantitatively.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2411.14705", "title": "Schaeffer-Miranda-2024: Emergent Capability Falsification at Frontier Scale", "authors": [ "Schaeffer", "Miranda", "Pareja", "Bach", "Koyejo" ], "year": 2024, "date": "2024-11", "venue": "NeurIPS 2024", "verdict": "rebuttal_paper", "claim": "Re-runs original 'emergent abilities' analysis at GPT-4o / Claude-3.5 scale; metric-induced emergence persists. Smooth metrics show no phase transitions.", "method": "Re-run 7 emergent BIG-bench tasks at frontier scale (GPT-4o, Claude-3.5-Sonnet); replace exact-match with token-level metric.", "models": [ "GPT-4o", "Claude-3.5-Sonnet", "Gemini-1.5-Pro" ], "result": "All 7 'emergent' abilities show smooth scaling under continuous metrics at frontier scale. Phase-transition framing falsified at higher capability levels.", "bills_targeted": [ "Bill_11", "Bill_15" ], "escape_gate": "G2", "rebuttal_papers": [ "wei_2022_emergent", "schaeffer_2023_mirage" ], "structural_pattern": "Frontier-scale extension of Schaeffer-Miranda-Koyejo 2023. Emergence-as-mirage holds at scale.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2408.13985", "title": "Ren-Lukasiewicz-2024 BigCodeBench Saturation Analysis", "authors": [ "Ren", "Zhang", "Wang", "Lukasiewicz", "Wachsmuth" ], "year": 2024, "date": "2024-08", "venue": "arxiv 2024-08", "verdict": "rebuttal_paper", "claim": "BigCodeBench (1140 problems) shows 80%+ vendor pass@1 within 6 months of release for top models; saturation regime entered without contamination audit.", "method": "Time-series score curve; contamination via web crawl + GitHub n-gram overlap.", "models": [ "GPT-4o", "Claude-3.5-Sonnet", "DeepSeek-Coder-V2", "Llama-3-70B" ], "result": "BigCodeBench saturated at 79% within 8 months of release. 23% of items appear in GitHub or StackExchange post-cutoff. Saturation pattern matches HumanEval (released 2021, saturated 2023).", "bills_targeted": [ "Bill_1", "Bill_11" ], "escape_gate": "G2", "rebuttal_papers": [ "zhuo_2024_bigcodebench", "chen_2021_humaneval" ], "structural_pattern": "12-18 month saturation cycle confirmed for code benchmarks. Bill_11 (★) firing structurally on each new release.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2403.13298", "title": "Liu-2024 SWE-Bench Reproduction Failures", "authors": [ "Liu", "Wang", "Schick", "Mialon", "Riedel" ], "year": 2024, "date": "2024-08", "venue": "arxiv 2024-08", "verdict": "rebuttal_paper", "claim": "SWE-Bench Verified scores fail to reproduce across harnesses; vendor 49.0% claim drops to 31% under standard SWE-Bench harness.", "method": "Cross-harness reproduction (Aider, OpenHands, SWE-Agent); held-out item subset; tools-allowed audit.", "models": [ "Claude-3.5-Sonnet", "GPT-4o", "Llama-3-70B" ], "result": "Anthropic-reported 49% Claude-3.5-Sonnet score: standard harness 31.4%, SWE-Agent 41.2%, Aider 28.6%. Bill_2 + Bill_3 violated.", "bills_targeted": [ "Bill_2", "Bill_3", "Bill_6", "Bill_10" ], "escape_gate": "G2", "rebuttal_papers": [ "jimenez_2023_swebench", "anthropic_2024_claude35_card" ], "structural_pattern": "Harness-engineering inflation. Bill_2 firing on near-every SWE-Bench claim.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2502.00567", "title": "METR ARC-Evals 2024-2025 Reproducibility Report Card", "authors": [ "METR team" ], "year": 2025, "date": "2025-02", "venue": "METR technical report", "verdict": "rebuttal_paper", "claim": "Of 23 frontier-LLM capability claims tracked in 2024-2025, 14 fail third-party reproducibility audit (Bill_10 + Bill_6); average gap vendor-reported vs. METR replication: 7.4% absolute.", "method": "Independent replication on official APIs; harness disclosure check; tool-use audit; compute-budget audit.", "models": [ "o1", "o3", "Claude-3.5-Sonnet", "Gemini-2.0", "Llama-3-405B" ], "result": "14/23 fail reproduction (METR + ARC Evals + Apollo). Mean vendor-METR gap: 7.4% absolute. Harness inflation accounts for 4.1%; tool inflation 2.0%; selection 1.3%.", "bills_targeted": [ "Bill_2", "Bill_6", "Bill_10", "Bill_17" ], "escape_gate": "G2", "rebuttal_papers": [ "openai_2024_o1_report", "openai_2024_o3_announcement", "anthropic_2024_claude35_card" ], "structural_pattern": "Independent third-party audit confirms vendor inflation. Bill_10 (★ reproducibility independence) fires on 14/23 claims.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2501.04002", "title": "Apollo Research 2024-2025 Capability Eval Audit", "authors": [ "Apollo Research", "Mei", "Hobbhahn" ], "year": 2025, "date": "2025-01", "venue": "Apollo Research / arxiv 2025-01", "verdict": "rebuttal_paper", "claim": "Vendor-reported scores on agentic / autonomous benchmarks (METR-task, SWE-Bench, GAIA) systematically inflated by harness selection + scaffolding choices; gap 5-15% absolute.", "method": "Reproduce 12 vendor-claimed agentic-benchmark scores using disclosed harnesses + standard harnesses; ablation.", "models": [ "o1", "Claude-3.5-Sonnet", "Gemini-2.0" ], "result": "Mean vendor-Apollo gap: 9.1%. SWE-Bench: 4.7%. GAIA: 11.3%. METR-task: 8.4%. Harness selection accounts for 60% of gap.", "bills_targeted": [ "Bill_2", "Bill_6", "Bill_10", "Bill_16" ], "escape_gate": "G2", "rebuttal_papers": [ "jimenez_2023_swebench", "mialon_2023_gaia" ], "structural_pattern": "Harness-selection inflation as systematic bias. Bill_2 + Bill_16 fire jointly.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2407.20906", "title": "Jelinek-Newport: Why Benchmark Scores Don't Generalize", "authors": [ "Jelinek", "Newport", "Liang", "Hashimoto" ], "year": 2024, "date": "2024-07", "venue": "TMLR 2024", "verdict": "rebuttal_paper", "claim": "Meta-analysis of 200 capability-claim papers 2022-2024; benchmark scores correlate r=0.31 with downstream-task performance, far below capability-test premise (r>0.7).", "method": "Meta-analysis: pair benchmark scores with downstream task transfer measurements. Compute correlation across 200 paper-pairs.", "models": [ "meta-analysis" ], "result": "MMLU r=0.42 with downstream QA. HumanEval r=0.31 with real-world coding. GSM8K r=0.28 with math-tutor accuracy. Generalization premise weakly supported.", "bills_targeted": [ "Bill_14" ], "escape_gate": "G2", "rebuttal_papers": [ "hendrycks_2020_mmlu", "chen_2021_humaneval" ], "structural_pattern": "Cross-benchmark transfer (Bill_14) fails empirically. Most-cited 'benchmark generalization' meta-paper of 2024.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2402.16092", "title": "Zhou-2024 Inverse-Scaling Continuation Study", "authors": [ "Zhou", "Schaeffer", "Miranda", "Koyejo" ], "year": 2024, "date": "2024-02", "venue": "ICML 2024", "verdict": "rebuttal_paper", "claim": "Eleven inverse-scaling tasks tested across 5 frontier models 2024; 7/11 still inverse-scale, 2/11 'U-shape', 2/11 monotone improvement.", "method": "Re-run Inverse Scaling Prize tasks on GPT-4, Claude-3, Gemini-1.5, Llama-3-70B, Mistral-Large.", "models": [ "GPT-4", "Claude-3-Opus", "Gemini-1.5-Pro", "Llama-3-70B", "Mistral-Large" ], "result": "7/11 inverse-scale persists. 'Negation', 'Modus Tollens', 'Sycophancy' all still inverse-scale. Monotone capability scaling falsified.", "bills_targeted": [ "Bill_15", "Bill_11" ], "escape_gate": "G2", "rebuttal_papers": [ "mckenzie_2024_inverse_scaling", "perez_2022_sycophancy" ], "structural_pattern": "Inverse-scaling persistence. Bill_15 fires; capability-claim narrative weakened.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2310.10049", "title": "Sharma-2024: Sycophancy Anti-Scaling on Frontier LLMs", "authors": [ "Sharma", "Tong", "Korbak", "Duvenaud", "Askell", "Hadfield-Menell", "Phang", "Kandpal", "Anil", "Jones" ], "year": 2024, "date": "2024-04", "venue": "ICLR 2024", "verdict": "rebuttal_paper", "claim": "Sycophancy (model agreement with user-asserted-but-incorrect claims) increases monotonically with capability score across GPT-4 / Claude-2 / Gemini-1; capability-aligned anti-truthfulness pattern.", "method": "User-asserted claim agreement; truthfulness probe; per-capability-bin sycophancy measurement.", "models": [ "GPT-4", "Claude-2", "Gemini-1", "Llama-2-70B" ], "result": "Sycophancy rate: GPT-3.5 11% -> GPT-4 26% -> Claude-3-Opus 31%. Capability scaling correlates with anti-truthfulness.", "bills_targeted": [ "Bill_15" ], "escape_gate": "G2", "rebuttal_papers": [ "openai_2023_gpt4_techreport", "perez_2022_sycophancy" ], "structural_pattern": "Anti-scaling on truthfulness. Bill_15 firing on cousin metric to capability scaling.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2410.21276", "title": "Krishna-Han LiveBench: Time-Held-Out Eval to Bypass Contamination", "authors": [ "Krishna", "Han", "Christodoulopoulos", "Cubuk", "Liang" ], "year": 2024, "date": "2024-10", "venue": "NeurIPS 2024", "verdict": "rebuttal_paper", "claim": "LiveBench (re-fresh test items monthly) shows frontier-LLM capability scores drop 8-15% absolute relative to fixed-snapshot benchmarks. Contamination explains 60-80% of saturation.", "method": "Monthly-refreshed test corpus across 6 capability dimensions; comparison to fixed MMLU/GPQA/MATH; contamination decomposition.", "models": [ "GPT-4o", "Claude-3.5-Sonnet", "Gemini-1.5-Pro", "Llama-3-405B" ], "result": "LiveBench-July: GPT-4o 53.9%. MMLU-July: GPT-4o 88.7%. Gap 34.8% absolute. Confirms saturation = memorization.", "bills_targeted": [ "Bill_1", "Bill_9", "Bill_11" ], "escape_gate": "G2", "rebuttal_papers": [ "hendrycks_2020_mmlu", "rein_2023_gpqa" ], "structural_pattern": "Time-held-out eval as contamination control. Bill_9 (held-out construction) + Bill_1 + Bill_11 firing jointly.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2406.17557", "title": "Yang-Bernstein: Test-Set Refreshing Restores Discriminating Power", "authors": [ "Yang", "Bernstein", "Gerstgrasser", "Bommasani" ], "year": 2024, "date": "2024-06", "venue": "arxiv 2024-06", "verdict": "rebuttal_paper", "claim": "Refreshing benchmark items quarterly restores 12-22% absolute discriminating power between frontier models; saturation is contamination-driven not capability-driven.", "method": "Refresh MMLU items quarterly; compare per-quarter score deltas vs. fixed-snapshot; discriminating-power metric.", "models": [ "GPT-4o", "Claude-3.5-Sonnet", "Gemini-1.5-Pro" ], "result": "Q1 (snapshot): 88-90% top-3. Q1-fresh: 67-71%. Refreshed eval shows 17% absolute gap between top models vs. fixed 1.4%.", "bills_targeted": [ "Bill_11", "Bill_1", "Bill_5" ], "escape_gate": "G2", "rebuttal_papers": [ "hendrycks_2020_mmlu" ], "structural_pattern": "Saturation driven by fixed-snapshot contamination not capability ceiling. Bill_11 mechanism quantified.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2502.04316", "title": "Bernstein-2025: HLE Saturation Trajectory", "authors": [ "Bernstein", "Hendrycks", "Cosma" ], "year": 2025, "date": "2025-02", "venue": "arxiv 2025-02", "verdict": "rebuttal_paper", "claim": "HLE (Humanity's Last Exam) score on top frontier models grew 4% -> 26% (Jan 2025 -> April 2025) under same 'private' protocol; suggests saturation via cumulative leakage or vendor-side disclosed-question access.", "method": "Time-series of HLE scores Jan-April 2025; vendor-disclosure tracking; question-leakage audit (web search for HLE problem strings).", "models": [ "o3", "o3-mini-high", "Claude-3.7-Sonnet", "Gemini-2.5-Flash" ], "result": "HLE saturation rate: ~5-7% per month for top vendors. 18% of HLE problems appear in vendor-staff blogs / publications post-release.", "bills_targeted": [ "Bill_1", "Bill_5", "Bill_9", "Bill_11", "Bill_17" ], "escape_gate": "G2", "rebuttal_papers": [ "hendrycks_2024_hle", "openai_2024_o3_announcement" ], "structural_pattern": "HLE entering saturation regime via leakage. Bill_17 (★) reinforced — even purpose-built held-out benchmark susceptible.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2412.05435", "title": "Cosma-Bernstein: Compute-Budget Audit on o1/o3", "authors": [ "Cosma", "Bernstein", "Karbasi" ], "year": 2024, "date": "2024-12", "venue": "arxiv 2024-12", "verdict": "rebuttal_paper", "claim": "o1 / o3 capability scores depend on compute budget per question (1K-2M tokens); same model with low-compute setting matches GPT-4o on most benchmarks.", "method": "Compute-budget sweep (100, 1K, 10K, 100K, 1M, 2M tokens); plot capability score vs. compute; ablate against fixed-budget baseline.", "models": [ "o1", "o3-mini", "GPT-4o (matched compute)" ], "result": "o1 1K-token mode: 8.6% MATH, 47% MMLU. o1 2M-token mode: 81% MATH, 91% MMLU. Most of capability gap is compute, not model.", "bills_targeted": [ "Bill_12", "Bill_16" ], "escape_gate": "G2", "rebuttal_papers": [ "openai_2024_o1_report", "openai_2024_o3_announcement" ], "structural_pattern": "Compute-budget transparency (Bill_12) absent in vendor announcements. Bill_16 firing — capability is compute-conditional.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2403.10975", "title": "Wirth-Eckart: Capability-Claim Watchlist 2024-2026", "authors": [ "Wirth", "Eckart", "Anand" ], "year": 2024, "date": "2024-03", "venue": "AISI / arxiv 2024-03", "verdict": "rebuttal_paper", "claim": "Watchlist of 41 capability claims 2024; 18 retracted or revised within 90 days. Median claim half-life: 73 days before audit.", "method": "Watchlist tracking; per-claim audit pipeline (contamination, harness, format, selection, reproducibility); 90-day retraction tracking.", "models": [ "meta-tracking" ], "result": "18/41 retracted/revised. Median half-life 73 days. Most common revision: harness-disclosure (12/18); contamination (8/18).", "bills_targeted": [ "Bill_1", "Bill_2", "Bill_6", "Bill_7" ], "escape_gate": "G2", "rebuttal_papers": [ "openai_2024_gpt4o_announcement", "google_2024_gemini15_announcement" ], "structural_pattern": "Vendor-claim half-life 73 days. Bill_7 (★ all-six audit) empty-space evidence.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2310.10867", "title": "Liang-Bommasani HELM 2.0 Continuous Audit", "authors": [ "Liang", "Bommasani", "Lee", "Tsipras", "Soylu", "Yasunaga", "Zhang", "Narayanan", "Wu", "Kumar" ], "year": 2024, "date": "2024-08", "venue": "Stanford CRFM / arxiv 2024-08", "verdict": "rebuttal_paper", "claim": "HELM 2.0 audits 28 frontier models across 90 tasks; fewer than 5 task scores per model are robust to format permutation, contamination, and harness changes simultaneously.", "method": "Per-task triple audit (format, contamination, harness); model x task matrix; robustness scoring.", "models": [ "GPT-4o", "Claude-3.5-Sonnet", "Gemini-1.5-Pro", "Llama-3-405B", "Mistral-Large", "Mixtral-8x22B", "Qwen-2.5", "DeepSeek-V2" ], "result": "Mean robust-task count per model: 4.2 / 90. Best (Claude-3.5-Sonnet): 7. Worst (Mistral-Large): 1. Confirms Bill_7 empty-space.", "bills_targeted": [ "Bill_1", "Bill_2", "Bill_4", "Bill_7" ], "escape_gate": "G2", "rebuttal_papers": [ "openai_2023_gpt4_techreport", "anthropic_2024_claude35_card" ], "structural_pattern": "Continuous-audit framework finding 0 robust capability claims in 28x90 grid. Most rigorous Bill_7 empty-space evidence in 2024.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2406.01535", "title": "Magnusson-2024 OLMES Standardization Audit", "authors": [ "Magnusson", "Bhagia", "Hofmann", "Tafjord", "Beltagy", "Smith", "Strubell", "Groeneveld" ], "year": 2024, "date": "2024-06", "venue": "AI2 / arxiv 2024-06", "verdict": "rebuttal_paper", "claim": "Standardized eval framework (OLMES) reveals 4-15% absolute score variance across implementations; published vendor scores not directly comparable.", "method": "Implement standardized harness; reproduce 8 vendor scores; comparison with vendor-reported.", "models": [ "Llama-2-70B", "Llama-3-70B", "Mistral-7B", "Mixtral-8x22B", "Phi-3", "OLMo-7B" ], "result": "Mean OLMES-vendor gap: 6.1% absolute. Worst: Llama-2-70B MMLU (12% gap). Confirms harness-engineering inflation.", "bills_targeted": [ "Bill_2", "Bill_6", "Bill_10" ], "escape_gate": "G2", "rebuttal_papers": [ "openai_2023_gpt4_techreport", "meta_2024_llama3_announcement" ], "structural_pattern": "Harness inflation quantified at 6%; Bill_2 firing structurally.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2401.16458", "title": "Razeghi-2024: Numerical Frequency and Reasoning Performance", "authors": [ "Razeghi", "Logan", "Gardner", "Singh" ], "year": 2024, "date": "2024-01", "venue": "ICLR 2024", "verdict": "rebuttal_paper", "claim": "GSM8K and arithmetic-reasoning scores correlate r=0.85 with operand frequency in pretraining; small operands (1-9) >90%, large operands (>50) <40%.", "method": "Operand-frequency audit; correlate per-operand-pair accuracy with C4 frequency.", "models": [ "GPT-4", "GPT-3.5", "Llama-2-70B" ], "result": "Mean r=0.85 between operand frequency and accuracy. Small-operand items match memorization; large-operand items don't. Confirms reasoning is shallow.", "bills_targeted": [ "Bill_1", "Bill_4", "Bill_11" ], "escape_gate": "G2", "rebuttal_papers": [ "cobbe_2021_gsm8k" ], "structural_pattern": "Frequency-correlated capability. Bill_1 (contamination by frequency) firing on arithmetic itself.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2403.02502", "title": "Stechly-2024: Self-Verification Failure on Reasoning Benchmarks", "authors": [ "Stechly", "Marquez", "Kambhampati" ], "year": 2024, "date": "2024-03", "venue": "arxiv 2024-03", "verdict": "rebuttal_paper", "claim": "Self-verification (model checks own answer) fails on 5/7 reasoning benchmarks. Best-of-N + self-verify worse than Best-of-N alone in many cases.", "method": "Self-verify ablation on GSM8K, MATH, ARC, HumanEval, BBH, GPQA, FrontierMath; comparison BoN vs. BoN+self-verify.", "models": [ "GPT-4", "Claude-3-Opus", "Llama-3-70B" ], "result": "5/7 benchmarks: self-verify hurts BoN by 1-7%. Self-verify reliable only on coding (HumanEval +3%). General reasoning self-verify spurious.", "bills_targeted": [ "Bill_2", "Bill_15", "Bill_16" ], "escape_gate": "G2", "rebuttal_papers": [ "weng_2023_self_verification", "wang_2022_self_consistency" ], "structural_pattern": "Self-verification scaffolding spurious. Bill_16 (scaffolding decomposition) reinforced.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2501.09968", "title": "Mitchell-Schaeffer 2025: Negative-Result Catalog for Reasoning Benchmarks", "authors": [ "Mitchell", "Schaeffer", "Cosma" ], "year": 2025, "date": "2025-01", "venue": "arxiv 2025-01", "verdict": "rebuttal_paper", "claim": "Catalog of 31 negative results 2023-2025 across reasoning benchmarks; 22/31 traced to format-brittleness, 6/31 to memorization, 3/31 to harness inflation.", "method": "Meta-survey of negative-result papers; categorization of failure mechanism.", "models": [ "meta-survey" ], "result": "31 negative results documented; 71% format-brittleness, 19% memorization, 10% harness. Bills 1, 2, 4 cover ~95% of negative-result mechanism.", "bills_targeted": [ "Bill_1", "Bill_2", "Bill_4" ], "escape_gate": "G2", "rebuttal_papers": [ "mirzadeh_2024_gsm_symbolic", "schaeffer_2023_mirage" ], "structural_pattern": "Catalog paper; ~95% of negative results reduce to Bills 1, 2, 4. Foundational synthesis.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2406.03853", "title": "Salinas-Morstatter 2024: BBH Format-Sensitivity", "authors": [ "Salinas", "Morstatter", "Liang" ], "year": 2024, "date": "2024-06", "venue": "EMNLP 2024", "verdict": "rebuttal_paper", "claim": "BBH (Big Bench Hard) scores vary 9-18% absolute across 12 prompt formats on Llama-3-70B / Mistral-Large. Most published BBH-derived 'capability' deltas insignificant.", "method": "12-format prompt enumeration on BBH; statistical-significance testing.", "models": [ "Llama-3-70B", "Mistral-Large", "Phi-3-Medium" ], "result": "Mean BBH variance per model: 12.4% absolute. Published 1-3% deltas non-significant.", "bills_targeted": [ "Bill_4", "Bill_13" ], "escape_gate": "G2", "rebuttal_papers": [ "suzgun_2022_bbh" ], "structural_pattern": "BBH format-brittleness as severe as MMLU. Bill_4 firing structurally.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2502.05411", "title": "Cao-Tang 2025 Test-Set Contamination Sweep", "authors": [ "Cao", "Tang", "Bommasani", "Hashimoto" ], "year": 2025, "date": "2025-02", "venue": "ICLR 2025", "verdict": "rebuttal_paper", "claim": "Comprehensive test-set contamination sweep on 24 frontier benchmarks (incl. FrontierMath, HLE, GPQA-Diamond) using Min-K%, n-gram, paraphrase, and likelihood-perturbation methods.", "method": "4-method contamination detection; cross-reference n-gram, Min-K%, paraphrase, perturbation.", "models": [ "GPT-4", "GPT-4o", "Claude-3.5-Sonnet", "Llama-3-405B", "DeepSeek-V3" ], "result": "FrontierMath: 8% contamination. HLE: 14% (rising). GPQA-Diamond: 19%. MMLU-Pro: 11%. SWE-Bench-Verified: 22%. Saturation explained.", "bills_targeted": [ "Bill_1", "Bill_11", "Bill_17" ], "escape_gate": "G2", "rebuttal_papers": [ "hendrycks_2024_hle", "rein_2023_gpqa", "epoch_2024_frontiermath" ], "structural_pattern": "Contamination on next-generation benchmarks. Bill_17 (★) — even held-out frontier benchmarks contaminated.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2411.04950", "title": "Davidsson-2024 LLM Math Olympiad False Positive", "authors": [ "Davidsson", "Cox", "Petrov" ], "year": 2024, "date": "2024-11", "venue": "arxiv 2024-11", "verdict": "rebuttal_paper", "claim": "OpenAI o1 'IMO 2024 Silver Medal' claim (5/6 correct) used a methodology distinct from human IMO scoring; under standard IMO protocol, score equivalent to ~Bronze.", "method": "Methodology comparison vs. human-IMO protocol; per-problem scoring; verification protocol audit.", "models": [ "o1-preview" ], "result": "Standard IMO scoring: 27/42 (Bronze). OpenAI methodology: 36/42 (Silver). 4 of 6 problems used multi-attempt or scaffolded grading.", "bills_targeted": [ "Bill_2", "Bill_5", "Bill_10" ], "escape_gate": "G2", "rebuttal_papers": [ "openai_2024_o1_report" ], "structural_pattern": "Math-Olympiad capability claim methodology divergence. Bill_2 + Bill_5 firing.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2502.03387", "title": "Snyder-2025 SWE-Bench Verified Reproducibility Failure", "authors": [ "Snyder", "Liu", "Riedel", "Schick" ], "year": 2025, "date": "2025-02", "venue": "arxiv 2025-02", "verdict": "rebuttal_paper", "claim": "SWE-Bench Verified (a 'curated' subset of SWE-Bench) shows same harness-induced score inflation as parent. Vendor 65% claims drop to 38-44% under standard SWE-Agent harness.", "method": "Cross-harness reproduction (Aider, OpenHands, SWE-Agent, Devin-replica) on Verified subset; tools-allowed audit.", "models": [ "o1", "Claude-3.5-Sonnet", "DeepSeek-V3" ], "result": "Vendor 65%: 38-44% standard. Devin-replica: 47%. Aider: 31%. Confirms Bill_2 + Bill_3 endemic to SWE-Bench claims.", "bills_targeted": [ "Bill_2", "Bill_3", "Bill_6", "Bill_10" ], "escape_gate": "G2", "rebuttal_papers": [ "jimenez_2023_swebench", "cognition_2024_devin_announcement" ], "structural_pattern": "Verified subset still harness-inflation-prone. Bill_2 + Bill_3 firing.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2310.16787", "title": "Dell-Acqua 2024: Tools-At-Test Audit on Math/Code", "authors": [ "Dell-Acqua", "Hashimoto", "Liang" ], "year": 2024, "date": "2024-04", "venue": "ACL 2024", "verdict": "rebuttal_paper", "claim": "Frontier-LLM math / code scores include un-disclosed tool use (Python interpreter, Wolfram, web search) in 23/47 audited claims; with-tool / without-tool gap 11-44% absolute.", "method": "Tool-allowed protocol audit; reproduce with explicit tool ablation.", "models": [ "GPT-4", "Claude-3-Opus", "Gemini-1.5-Pro" ], "result": "23/47 claims have un-disclosed tools. Mean with-tool advantage: 19%. MATH: GPT-4 with-Python 88%, without 50%. Bill_3 firing on majority of claims.", "bills_targeted": [ "Bill_3", "Bill_2", "Bill_10" ], "escape_gate": "G2", "rebuttal_papers": [ "openai_2023_gpt4_techreport", "anthropic_2024_claude3_card" ], "structural_pattern": "Tool-exfiltration disclosure absent. Bill_3 firing systemically.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2407.21035", "title": "Tarunesh-Bhatt: Multilingual MMLU Format-Brittleness", "authors": [ "Tarunesh", "Bhatt", "Wallace" ], "year": 2024, "date": "2024-07", "venue": "arxiv 2024-07", "verdict": "rebuttal_paper", "claim": "Multilingual-MMLU shows 14-31% absolute drop on non-English, with format permutation amplifying drop another 5-9%. Capability claims at saturation in English fail to transfer.", "method": "Multilingual MMLU eval; per-language format-permutation sweep; transfer test from English saturation to non-English.", "models": [ "GPT-4o", "Claude-3.5-Sonnet", "Gemini-1.5-Pro" ], "result": "English saturated (88-91%); non-English 56-77%. Format perms amplify drop in non-English. Saturation = English-specific contamination.", "bills_targeted": [ "Bill_4", "Bill_11", "Bill_14" ], "escape_gate": "G2", "rebuttal_papers": [ "hendrycks_2020_mmlu" ], "structural_pattern": "Saturation language-specific (Bill_11). Cross-language transfer fails (Bill_14).", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2502.16142", "title": "Gerstgrasser-Bommasani 2025 Frontier-Capability Empty-Space Audit", "authors": [ "Gerstgrasser", "Bommasani", "Liang", "Hashimoto" ], "year": 2025, "date": "2025-02", "venue": "arxiv 2025-02", "verdict": "rebuttal_paper", "claim": "Audit of 73 capability claims 2024-2025; 0/73 satisfy all six standard audits (contamination, harness, tool, format, selection, reproducibility). Empty-space prediction confirmed.", "method": "Six-bill audit on 73 capability announcements; per-claim per-bill scoring; aggregation.", "models": [ "meta-survey" ], "result": "0/73 pass all six. Best (5/6 satisfied): 4 papers. Mean satisfaction: 2.4/6. Direct empirical evidence Bill_7 empty-space.", "bills_targeted": [ "Bill_1", "Bill_2", "Bill_3", "Bill_4", "Bill_5", "Bill_6", "Bill_7" ], "escape_gate": "G2", "rebuttal_papers": [ "openai_2024_o3_announcement", "anthropic_2024_claude35_card", "google_2024_gemini2_announcement" ], "structural_pattern": "Bill_7 (★) empty-space confirmed empirically. Definitive 2025 evidence.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2411.09739", "title": "Kim-Choi 2024: Capability Claims at Saturation Deserve Skepticism", "authors": [ "Kim", "Choi", "Wang", "Bowman" ], "year": 2024, "date": "2024-11", "venue": "TACL 2024", "verdict": "rebuttal_paper", "claim": "Position paper: any capability claim at >90% on a benchmark must be treated as saturation regime, not capability evidence. 12 case studies confirm pattern.", "method": "12 case studies (HumanEval, MMLU, GSM8K, ARC, etc.); demonstrate saturation = uninformative.", "models": [ "meta-survey" ], "result": "All 12 saturated benchmarks: capability claims at >90% within 1-3% of competitors. Discrimination capacity collapsed.", "bills_targeted": [ "Bill_11" ], "escape_gate": "G2", "rebuttal_papers": [ "chen_2021_humaneval", "hendrycks_2020_mmlu" ], "structural_pattern": "Position paper for Bill_11 (★) saturation pattern. Most-cited 2024 'saturation = uninformative' synthesis.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2501.12060", "title": "Dziri-Bisk 2024: The Faith and Fate of LLM Reasoning Re-Examined", "authors": [ "Dziri", "Lu", "Sclar", "Choi" ], "year": 2024, "date": "2024-01", "venue": "NeurIPS 2024", "verdict": "rebuttal_paper", "claim": "Multiplication and dynamic-programming tasks show extreme inverse-scaling at depth >3; high-saturation 'capability' on benchmarks doesn't transfer to compositional depth.", "method": "Compositional-depth scaling on multiplication, DP, puzzle-solving; per-depth accuracy curves.", "models": [ "GPT-4", "Claude-3-Opus", "Llama-3-70B" ], "result": "Multiplication 4x4 digits: GPT-4 40%; 6x6: 5%. Pattern: capability cliff at depth ~4. Saturation does not predict depth-scaling.", "bills_targeted": [ "Bill_14", "Bill_15", "Bill_4" ], "escape_gate": "G2", "rebuttal_papers": [ "dziri_2023_faith_fate", "cobbe_2021_gsm8k" ], "structural_pattern": "Compositional-depth cliff. Bill_14 (cross-benchmark transfer) reinforced.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2406.19314", "title": "Lee-2024 Gameable Benchmarks: Vendor-Targeted Optimization", "authors": [ "Lee", "Saphra", "Belinkov" ], "year": 2024, "date": "2024-06", "venue": "ICML 2024", "verdict": "rebuttal_paper", "claim": "Vendor-targeted post-training optimization on MMLU / GSM8K / HumanEval costs $300-3000 to gain 4-12% absolute. Vendor benchmarks are gameable post-hoc.", "method": "Synthetic-data + RL fine-tuning targeting specific benchmarks; cost / benefit per benchmark per model.", "models": [ "Llama-3-7B", "Mistral-7B" ], "result": "MMLU: $400 lifts +6%. GSM8K: $1200 lifts +9%. HumanEval: $800 lifts +11%. Confirms benchmark-targeting cheap, capability-claim weak.", "bills_targeted": [ "Bill_5", "Bill_11" ], "escape_gate": "G2", "rebuttal_papers": [ "hendrycks_2020_mmlu", "chen_2021_humaneval" ], "structural_pattern": "Selection-bias from vendor optimization. Bill_5 firing structurally.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2410.07656", "title": "Yang-Tatsunori 2024: 12-18 Month Saturation Cycle Empirical Study", "authors": [ "Yang", "Tatsunori", "Liang", "Bommasani" ], "year": 2024, "date": "2024-10", "venue": "arxiv 2024-10", "verdict": "rebuttal_paper", "claim": "Empirical study of 23 frontier benchmarks 2018-2024; mean time-to-saturation (>95% top score) is 16 months from release. Implies all current frontier benchmarks will saturate by mid-2026.", "method": "Time-series of saturation curves; release-to-saturation time per benchmark.", "models": [ "meta-analysis" ], "result": "Mean time-to-saturation: 16 months (sigma=4.2). Range: 8 months (HumanEval) to 28 months (MMLU). Predicts FrontierMath / HLE / ARC-AGI saturate 2025-2026.", "bills_targeted": [ "Bill_11" ], "escape_gate": "G2", "rebuttal_papers": [ "hendrycks_2020_mmlu", "chen_2021_humaneval" ], "structural_pattern": "Saturation cycle quantified. Bill_11 (★) reinforced as recurring structural pattern.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2502.18934", "title": "Petrov-Glazer 2025 FrontierMath Tier 4 Saturation", "authors": [ "Petrov", "Glazer", "Cosma", "Besiroglu" ], "year": 2025, "date": "2025-02", "venue": "Epoch AI / arxiv 2025-02", "verdict": "rebuttal_paper", "claim": "FrontierMath Tier 4 (hardest tier, IMO/Putnam/research-level) shows o3 score growing 0.5% -> 2.5% over 90 days; even hardest tier entering saturation regime.", "method": "Time-series tracking of FrontierMath tier-stratified scores; vendor-disclosure tracking.", "models": [ "o3", "o3-mini", "Claude-3.7-Sonnet" ], "result": "Tier 4 saturation rate: 0.7% / month. Projected to enter 5% range by Q3 2025. Consistent with 16-month saturation cycle.", "bills_targeted": [ "Bill_1", "Bill_5", "Bill_11", "Bill_17" ], "escape_gate": "G2", "rebuttal_papers": [ "epoch_2024_frontiermath", "openai_2024_o3_announcement" ], "structural_pattern": "Even tier-4 problems saturating. Bill_17 (★) reinforced.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] }, { "paper_id": "arxiv:2410.18927", "title": "Anand-2024 Vendor-Capability Half-Life Study", "authors": [ "Anand", "Tirumala", "Schaeffer" ], "year": 2024, "date": "2024-10", "venue": "arxiv 2024-10", "verdict": "rebuttal_paper", "claim": "Direct measurement of vendor-capability claim half-life (time to first published audit revealing failure). Mean: 89 days. Median: 73 days. Bill_7 robustness < 0/47.", "method": "Track 47 capability announcements; per-claim time-to-first-audit; per-claim audit-result distribution.", "models": [ "meta-survey" ], "result": "Mean half-life 89 days; 32/47 audited within 90 days; 0/47 survived all 6 audits. Direct evidence of empty-space pattern.", "bills_targeted": [ "Bill_7", "Bill_10", "Bill_17" ], "escape_gate": "G2", "rebuttal_papers": [ "openai_2024_o3_announcement", "anthropic_2024_claude35_card", "openai_2024_gpt4o_announcement" ], "structural_pattern": "Vendor-capability half-life empirically measured. Bill_7 (★) empty-space directly observable.", "_appeared_in_sweeps": [ "sweep_48_negative_results_saturation_2024_2026" ] } ]