{ "excludes": [ "RM-19" ], "footnotes": { "not_shown": [ "Any customer's runs, or any run longer than a tutorial.", "Any solver but OpenFOAM v2406." ], "prereg": [ { "file": "docs/RRD.md", "text": "Pre-registered 15 Oct 2026, before any request of it.", "where": "RM-36" }, { "file": "docs/RRD.md", "text": "Versions for the continuation, 13 Oct 2026 (registered before any run of them).", "where": "RM-35" } ], "rebuilt": "The judges saw a rebuilt case configuration, not the case Courant read.", "scope": "A statement about these models, this prompt and these dates, on tutorial runs, never about language models in general.", "spend_usd": "77.50", "statement": "These models, this prompt (judge/2) and these dates, on OpenFOAM v2406 tutorial runs of seven families; never a statement about language models in general. The judges saw a rebuilt case configuration, not the case Courant read (RRD RM-36, threats)." }, "interval": { "level": 0.95, "method": "wilson", "rm35": { "decimals": 6, "source": "courant/silentbench/stats.py wilson", "z": 1.96 }, "rm36": { "decimals": 4, "source": "courant/ctest/score.py wilson", "z": 1.959963984540054 } }, "judges": { "corpus": { "cells": 21, "families": 7, "name": "Courant-10K", "per_cell": 20, "right": 140, "runs": 420, "sample": { "file": "research/results/RM-36-sample.json", "sha256": "788c877a2c9d19528f17efc2c943296c5c34af86f7f29d65bcf309f50055c0cd" }, "solver": "OpenFOAM v2406", "solver_fault": 140, "version": 2, "wrong": 280, "wrong_problem": 140 }, "date": "2026-10-15", "denominators": { "abstentions_in": false, "failed_requests_in": false, "source": "docs/BENCHMARKS.md §12" }, "doubtful_references": { "file": "docs/RRD.md", "text": "False alarms: Opus 5 of 140, Sonnet 0 of 138, Haiku 17 of 122; Courant 10 of 140, fewer than Haiku and more than Opus and Sonnet (8 pitzDaily and 2 hotRoom: the two families whose references RM-35's findings found doubtful).", "where": "RM-36, Result" }, "file": "research/results/RM-36-latest.json", "given": "Each model was given the solver log (middle-truncated at 60,000 characters), the case configuration rebuilt from the tutorial and the run's recorded edits, the mesh metrics, and the task's locked specification (judge/2), and asked whether the run is a valid result of the specified problem.", "kinds": [ { "id": "all", "label": "All wrong runs", "product_kinds": [ "catalogue", "semantic" ] }, { "id": "wrong_problem", "label": "Wrong problem", "product_kinds": [ "semantic" ] }, { "id": "solver_fault", "label": "Solver fault", "product_kinds": [ "catalogue" ] } ], "order": [ "courant", "claude-opus-5-5", "claude-sonnet-5-5", "claude-haiku-5-5" ], "rebuilt": "The judges saw a rebuilt case configuration, not the case Courant read.", "registered": { "commit": "3aba674158ed96600d6b6fdbb8f83ede50bb7a2d", "date": "2026-10-15T19:30:01-04:00", "result": { "commit": "39689fcf017100ffe093e5c4597c851076bdb013", "date": "2026-10-15T21:07:23-04:00", "short": "39689fc" }, "rrd": { "file": "docs/RRD.md", "text": "Pre-registered 15 Oct 2026, before any request of it.", "where": "RM-36" }, "short": "3aba674" }, "rows": [ { "abstained": { "all": { "ci_pct": [ "2", "6" ], "hi": 0.061, "lo": 0.0236, "n": 16, "of": 420 }, "right": { "ci_pct": [ "0", "3" ], "hi": 0.0267, "lo": 0.0, "n": 0, "of": 140 }, "solver_fault": { "ci_pct": [ "7", "18" ], "hi": 0.1776, "lo": 0.0716, "n": 16, "of": 140 }, "wrong_problem": { "ci_pct": [ "0", "3" ], "hi": 0.0267, "lo": 0.0, "n": 0, "of": 140 } }, "column": "spec", "courant": true, "failed": { "ci_pct": [ "0", "1" ], "hi": 0.0091, "lo": 0.0, "n": 0, "of": 420 }, "false_accepts": { "all": { "ci_pct": [ "0", "1" ], "hi": 0.0143, "lo": 0.0, "n": 0, "of": 264 }, "solver_fault": { "ci_pct": [ "0", "3" ], "hi": 0.03, "lo": 0.0, "n": 0, "of": 124 }, "wrong_problem": { "ci_pct": [ "0", "3" ], "hi": 0.0267, "lo": 0.0, "n": 0, "of": 140 } }, "false_alarms": { "ci_pct": [ "4", "13" ], "hi": 0.1265, "lo": 0.0393, "n": 10, "of": 140 }, "false_alarms_by_family": { "buoyantPimpleFoam_hotRoom": { "false_accepts": { "ci_pct": [ "0", "10" ], "hi": 0.1043, "lo": 0.0, "n": 0, "of": 33 }, "false_alarms": { "ci_pct": [ "3", "30" ], "hi": 0.301, "lo": 0.0279, "n": 2, "of": 20 }, "short": "hotRoom" }, "icoFoam_cavity": { "false_accepts": { "ci_pct": [ "0", "9" ], "hi": 0.0918, "lo": 0.0, "n": 0, "of": 38 }, "false_alarms": { "ci_pct": [ "0", "16" ], "hi": 0.1611, "lo": 0.0, "n": 0, "of": 20 }, "short": "cavity" }, "interFoam_damBreak": { "false_accepts": { "ci_pct": [ "0", "10" ], "hi": 0.1043, "lo": 0.0, "n": 0, "of": 33 }, "false_alarms": { "ci_pct": [ "0", "16" ], "hi": 0.1611, "lo": 0.0, "n": 0, "of": 20 }, "short": "damBreak" }, "pimpleFoam_planarContraction": { "false_accepts": { "ci_pct": [ "0", "9" ], "hi": 0.0876, "lo": 0.0, "n": 0, "of": 40 }, "false_alarms": { "ci_pct": [ "0", "16" ], "hi": 0.1611, "lo": 0.0, "n": 0, "of": 20 }, "short": "planarContraction" }, "pisoFoam_RAS_cavity": { "false_accepts": { "ci_pct": [ "0", "9" ], "hi": 0.0876, "lo": 0.0, "n": 0, "of": 40 }, "false_alarms": { "ci_pct": [ "0", "16" ], "hi": 0.1611, "lo": 0.0, "n": 0, "of": 20 }, "short": "RAS_cavity" }, "rhoPimpleFoam_RAS_cavity": { "false_accepts": { "ci_pct": [ "0", "9" ], "hi": 0.0876, "lo": 0.0, "n": 0, "of": 40 }, "false_alarms": { "ci_pct": [ "0", "16" ], "hi": 0.1611, "lo": 0.0, "n": 0, "of": 20 }, "short": "RAS_cavity" }, "simpleFoam_pitzDaily": { "false_accepts": { "ci_pct": [ "0", "9" ], "hi": 0.0876, "lo": 0.0, "n": 0, "of": 40 }, "false_alarms": { "ci_pct": [ "22", "61" ], "hi": 0.6134, "lo": 0.2188, "n": 8, "of": 20 }, "short": "pitzDaily" } }, "id": "courant", "model": null, "name": "Courant", "without_spec": { "false_accepts": { "all": { "ci_pct": [ "39", "51" ], "hi": 0.5073, "lo": 0.3882, "n": 118, "of": 264 }, "solver_fault": { "ci_pct": [ "0", "3" ], "hi": 0.03, "lo": 0.0, "n": 0, "of": 124 }, "wrong_problem": { "ci_pct": [ "77", "89" ], "hi": 0.8939, "lo": 0.7735, "n": 118, "of": 140 } }, "false_alarms": { "ci_pct": [ "4", "13" ], "hi": 0.1265, "lo": 0.0393, "n": 10, "of": 140 } } }, { "abstained": { "all": { "ci_pct": [ "2", "6" ], "hi": 0.061, "lo": 0.0236, "n": 16, "of": 420 }, "right": { "ci_pct": [ "0", "3" ], "hi": 0.0267, "lo": 0.0, "n": 0, "of": 140 }, "solver_fault": { "ci_pct": [ "7", "18" ], "hi": 0.1776, "lo": 0.0716, "n": 16, "of": 140 }, "wrong_problem": { "ci_pct": [ "0", "3" ], "hi": 0.0267, "lo": 0.0, "n": 0, "of": 140 } }, "courant": false, "failed": { "ci_pct": [ "0", "1" ], "hi": 0.0091, "lo": 0.0, "n": 0, "of": 420 }, "false_accepts": { "all": { "ci_pct": [ "21", "32" ], "hi": 0.3175, "lo": 0.2121, "n": 69, "of": 264 }, "solver_fault": { "ci_pct": [ "3", "12" ], "hi": 0.1222, "lo": 0.0331, "n": 8, "of": 124 }, "wrong_problem": { "ci_pct": [ "36", "52" ], "hi": 0.5185, "lo": 0.3564, "n": 61, "of": 140 } }, "false_alarms": { "ci_pct": [ "2", "8" ], "hi": 0.0809, "lo": 0.0153, "n": 5, "of": 140 }, "file": "research/results/RM-36-claude-opus-5-5-2026-10-15.json", "finished": "2026-10-16T01:05:45.408368Z", "h1": { "alpha": 0.016666666666666666, "alpha_text": "0.0167", "b": 69, "both_answered": 264, "c": 0, "hypothesis": "the model accepts more of the wrong runs than Courant's spec column", "p": 1.6940658945086007e-21, "p_text": "1.69e-21", "test": "exact one-sided McNemar: P(X >= b), X ~ Binomial(b + c, 1/2)", "verdict": "SUPPORTED" }, "id": "claude-opus-5-5", "model": "claude-opus-5-5", "name": "Claude Opus 5.5", "sha256": "c5ec8aa6588a10875a148d21849e2c1265e496945703daed8e0aaf05aa3f2c2e", "spend": { "per_run_usd": 0.120234, "requests": 421, "text": "50.50", "usd": 50.498464 } }, { "abstained": { "all": { "ci_pct": [ "3", "7" ], "hi": 0.0696, "lo": 0.0291, "n": 19, "of": 420 }, "right": { "ci_pct": [ "0", "5" ], "hi": 0.0506, "lo": 0.0039, "n": 2, "of": 140 }, "solver_fault": { "ci_pct": [ "7", "18" ], "hi": 0.1776, "lo": 0.0716, "n": 16, "of": 140 }, "wrong_problem": { "ci_pct": [ "0", "4" ], "hi": 0.0393, "lo": 0.0013, "n": 1, "of": 140 } }, "courant": false, "failed": { "ci_pct": [ "0", "1" ], "hi": 0.0091, "lo": 0.0, "n": 0, "of": 420 }, "false_accepts": { "all": { "ci_pct": [ "24", "35" ], "hi": 0.3465, "lo": 0.2375, "n": 76, "of": 263 }, "solver_fault": { "ci_pct": [ "9", "21" ], "hi": 0.2086, "lo": 0.0874, "n": 17, "of": 124 }, "wrong_problem": { "ci_pct": [ "35", "51" ], "hi": 0.5076, "lo": 0.3454, "n": 59, "of": 139 } }, "false_alarms": { "ci_pct": [ "0", "3" ], "hi": 0.0271, "lo": 0.0, "n": 0, "of": 138 }, "file": "research/results/RM-36-claude-sonnet-5-5-2026-10-15.json", "finished": "2026-10-16T00:57:27.371102Z", "h1": { "alpha": 0.016666666666666666, "alpha_text": "0.0167", "b": 76, "both_answered": 262, "c": 0, "hypothesis": "the model accepts more of the wrong runs than Courant's spec column", "p": 1.3234889800848443e-23, "p_text": "1.32e-23", "test": "exact one-sided McNemar: P(X >= b), X ~ Binomial(b + c, 1/2)", "verdict": "SUPPORTED" }, "id": "claude-sonnet-5-5", "model": "claude-sonnet-5-5", "name": "Claude Sonnet 5.5", "sha256": "75d7d268db9c3811d41f36070f8904a3c60792d11ce021f021c4162622a1088a", "spend": { "per_run_usd": 0.061026, "requests": 421, "text": "25.63", "usd": 25.630992 } }, { "abstained": { "all": { "ci_pct": [ "9", "15" ], "hi": 0.1509, "lo": 0.0894, "n": 49, "of": 420 }, "right": { "ci_pct": [ "8", "19" ], "hi": 0.1941, "lo": 0.0829, "n": 18, "of": 140 }, "solver_fault": { "ci_pct": [ "9", "21" ], "hi": 0.2104, "lo": 0.0944, "n": 20, "of": 140 }, "wrong_problem": { "ci_pct": [ "4", "14" ], "hi": 0.1352, "lo": 0.0444, "n": 11, "of": 140 } }, "courant": false, "failed": { "ci_pct": [ "0", "1" ], "hi": 0.0091, "lo": 0.0, "n": 0, "of": 420 }, "false_accepts": { "all": { "ci_pct": [ "16", "26" ], "hi": 0.2636, "lo": 0.163, "n": 52, "of": 249 }, "solver_fault": { "ci_pct": [ "2", "10" ], "hi": 0.1048, "lo": 0.0231, "n": 6, "of": 120 }, "wrong_problem": { "ci_pct": [ "28", "44" ], "hi": 0.4423, "lo": 0.2792, "n": 46, "of": 129 } }, "false_alarms": { "ci_pct": [ "9", "21" ], "hi": 0.2119, "lo": 0.0889, "n": 17, "of": 122 }, "file": "research/results/RM-36-claude-haiku-5-5-2026-10-15.json", "finished": "2026-10-16T00:52:02.561281Z", "h1": { "alpha": 0.016666666666666666, "alpha_text": "0.0167", "b": 52, "both_answered": 249, "c": 0, "hypothesis": "the model accepts more of the wrong runs than Courant's spec column", "p": 2.220446049250313e-16, "p_text": "2.22e-16", "test": "exact one-sided McNemar: P(X >= b), X ~ Binomial(b + c, 1/2)", "verdict": "SUPPORTED" }, "id": "claude-haiku-5-5", "model": "claude-haiku-5-5", "name": "Claude Haiku 5.5", "sha256": "0d31458b7ef4a7f7c6a3db1e6855f9874ce4b35811056d023c0391e51625b302", "spend": { "per_run_usd": 0.003257, "requests": 420, "text": "1.37", "usd": 1.368041 } } ], "scope": "A statement about these models, this prompt and these dates, on tutorial runs, never about language models in general.", "spend": { "requests": 1262, "text": "77.50", "usd": 77.497497 }, "statement": "These models, this prompt (judge/2) and these dates, on OpenFOAM v2406 tutorial runs of seven families; never a statement about language models in general. The judges saw a rebuilt case configuration, not the case Courant read (RRD RM-36, threats).", "status": "measured", "study": "RM-36", "title": "Frontier models as judges on 420 Courant-10K runs, beside Courant" }, "license": { "attribution": "Courant, https://www.courant.dev/benchmarks", "decided": "S, 18 Oct 2026", "name": "CC BY 4.0", "url": "https://creativecommons.org/licenses/by/4.0/" }, "product_commit": "7cec4cd7f873dd7b95d20a5e5531eba6fd778864", "quoted_from": [ "docs/RRD.md", "docs/TRD.md", "docs/WHAT-COURANT-SEES.md" ], "scale": { "date": "2026-10-14", "descriptive": { "wrong_problem_rejected_blind": { "hi": 0.233952, "lo": 0.170462, "n": 122, "of": 609 } }, "excluded": 0, "families": 7, "file": "research/results/RM-35-v2-latest.json", "gate_commits": [ "36d277fed2c46e7ceee9d891b7558a9fad225221", "4976abc88ddddf73ec8317f23e86688634cf90c6" ], "gate_version": 2, "hypotheses": [ { "accepted": [ { "family": "buoyantPimpleFoam_hotRoom", "fault": "laminar_on_turbulent", "n": 3, "short": "hotRoom" } ], "bar_pct": "1", "bar_upper": 0.01, "ci_pct": [ "0.17", "1.44" ], "class": "catalogue", "column": "blind", "hi": 0.014382, "id": "H1", "lo": 0.001677, "n": 3, "of": 609, "outcome": "accept", "verdict": "REFUTED", "what": "faulted runs accepted, blind" }, { "bar_pct": "1", "bar_upper": 0.01, "ci_pct": [ "0.00", "0.63" ], "class": "catalogue", "column": "spec", "hi": 0.006269, "id": "H1b", "lo": 0.0, "n": 0, "of": 609, "outcome": "accept", "verdict": "SUPPORTED", "what": "faulted runs accepted, with the specification" }, { "bar_pct": "1", "bar_upper": 0.01, "ci_pct": [ "0.00", "0.63" ], "class": "semantic", "column": "spec", "hi": 0.006269, "id": "H2", "lo": 0.0, "n": 0, "of": 609, "outcome": "accept", "verdict": "SUPPORTED", "what": "wrong-problem runs accepted, with the specification" }, { "bar_pct": "2", "bar_upper": 0.02, "ci_pct": [ "4.02", "7.70" ], "class": "healthy", "column": "spec", "families": [ { "checks": [ "bounding.epsilon", "bounds.turbulence" ], "family": "simpleFoam_pitzDaily", "hi": 0.449354, "lo": 0.253425, "n": 30, "of": 87, "rule": [ "VER-5.3" ], "short": "pitzDaily", "why": { "check": "bounding.epsilon", "clipped_times": [ 48, 273 ], "example": "bounding.epsilon: Field epsilon bounded 119x: epsilon still being clipped late in the solve (119 times in total)", "says": "epsilon still being clipped late in the solve" } }, { "checks": [ "cfl.climbing", "cfl.max", "convergence.residual" ], "family": "buoyantPimpleFoam_hotRoom", "hi": 0.112332, "lo": 0.018022, "n": 4, "of": 87, "rule": [ "VER-5.3" ], "short": "hotRoom", "why": { "check": "cfl.max", "example": "cfl.max: Courant number exceeds stability limit: max Co=25.7 at t=4.0 exceeds limit 1.0; still 1.14 over the final fifth of the run", "final_fifth_co": [ 1.07, 1.15 ], "limit": 1.0, "says": "Courant number above the stability limit over the final fifth of the run, adjustTimeStep off" } } ], "hi": 0.077003, "id": "H3", "lo": 0.040224, "n": 34, "of": 609, "outcome": "reject", "reading": { "gate_semantics": { "file": "docs/RRD.md", "text": "The gate's rejection is correct by OpenFOAM's semantics; the label is what is doubtful (Re_h 12.9k to 16.4k).", "where": "RM-35, findings on the interim" }, "interim_title": { "file": "docs/RRD.md", "text": "Three pitzDaily healthy draws are not clean references.", "where": "RM-35, findings on the interim" }, "next_manifest": { "file": "docs/TRD.md", "text": "Manifest version 3 (--manifest-version 3) is version 2 under the seed courant-10k-v1 for every family, with every task the earlier corpus drew (and every dev task) redrawn, and the simpleFoam_pitzDaily and buoyantPimpleFoam_hotRoom draws conditioned on regimes a sweep measures, so families.py is unchanged.", "where": "CERT-7, Manifest version 3" }, "next_measured": false, "next_sweep": { "file": "docs/TRD.md", "text": "The sweep, to be registered in RRD RM-35 before it runs (scripts/regime_sweep.py, courant/certify/regime_sweep.py), solves each family's own task as a healthy twin is, with the configuration the gate read recorded.", "where": "CERT-7, Manifest version 3" } }, "verdict": "REFUTED", "what": "right runs rejected, with the specification" }, { "bar_pct": "5", "bar_upper": 0.05, "ci_pct": [ "0.00", "0.63" ], "class": "healthy", "column": "spec", "hi": 0.006269, "id": "H4", "lo": 0.0, "n": 0, "of": 609, "outcome": "insufficient", "verdict": "SUPPORTED", "what": "right runs left insufficient, with the specification" }, { "bar": "0 exceptions", "columns": 2, "exceptions": 0, "id": "H5", "verdict": "SUPPORTED", "verifications": 3654, "what": "verifier exceptions" } ], "images": [ "opencfd/openfoam-default@sha256:dd5aa20630a55722663bf83ba0cb74870cba130081303e32e3865007fa2aa35a" ], "interim": { "complete": false, "date": "2026-10-13", "excluded": 30, "file": "research/results/RM-35-partial.json", "gate_commits": [ "1a935898bdef2c7fe4dd5f319402ebcce4d571b9", "8492645c6637f6c6a6d2edfeb0299902e2359ddf" ], "gate_version": 1, "hypotheses": [ { "bar_pct": "1", "bar_upper": 0.01, "ci_pct": [ "3.06", "13.65" ], "class": "catalogue", "column": "blind", "hi": 0.136466, "id": "H1", "lo": 0.030566, "n": 6, "of": 91, "outcome": "accept", "verdict": "INCONCLUSIVE", "what": "faulted runs accepted, blind" }, { "bar_pct": "1", "bar_upper": 0.01, "ci_pct": [ "3.06", "13.65" ], "class": "catalogue", "column": "spec", "hi": 0.136466, "id": "H1b", "lo": 0.030566, "n": 6, "of": 91, "outcome": "accept", "verdict": "INCONCLUSIVE", "what": "faulted runs accepted, with the specification" }, { "bar_pct": "1", "bar_upper": 0.01, "ci_pct": [ "0.00", "4.05" ], "class": "semantic", "column": "spec", "hi": 0.040505, "id": "H2", "lo": 0.0, "n": 0, "of": 91, "outcome": "accept", "verdict": "INCONCLUSIVE", "what": "wrong-problem runs accepted, with the specification" }, { "bar_pct": "2", "bar_upper": 0.02, "ci_pct": [ "2.58", "15.68" ], "class": "healthy", "column": "spec", "hi": 0.15683, "id": "H3", "lo": 0.025793, "n": 4, "of": 61, "outcome": "reject", "verdict": "INCONCLUSIVE", "what": "right runs rejected, with the specification" }, { "bar_pct": "5", "bar_upper": 0.05, "ci_pct": [ "0.00", "5.92" ], "class": "healthy", "column": "spec", "hi": 0.059246, "id": "H4", "lo": 0.0, "n": 0, "of": 61, "outcome": "insufficient", "verdict": "INCONCLUSIVE", "what": "right runs left insufficient, with the specification" }, { "bar": "0 exceptions", "columns": 2, "exceptions": 0, "id": "H5", "verdict": "INCONCLUSIVE", "verifications": 486, "what": "verifier exceptions" } ], "label": "Gate version 1, interim (the development set)", "manifest_sha256": "87a97cdfd5472bf0aaf16df2a18f7fcbf6aec2ffd72446b00ab9dcd27be1bb44", "per_class": { "right": 61, "solver_fault": 91, "wrong_problem": 91 }, "planned": 2100, "pooled": false, "runs": 273, "set": "the development set" }, "intro": "Real OpenFOAM v2406 solves of the seven SilentBench dev families, each draw run three ways: the right answer, the right answer with a catalogue fault, and the right physics on the wrong problem. Truth is by construction. Each hypothesis is decided on the Wilson 95% upper bound, fixed before any run. Versions are never pooled.", "label": "Gate version 2", "manifest_sha256": "167b18ad37effa441111bd8b0b8f4fced836d842823506f193606a5ad7cbfc3c", "per_class": { "right": 609, "solver_fault": 609, "wrong_problem": 609 }, "registered": { "commit": "25e3c8725de21b2a5925475d12a01178bf88ae15", "continuation": { "commit": "194fa76d36361acff55756be12cfc849fff1552b", "date": "2026-10-13T19:31:41-04:00", "short": "194fa76" }, "date": "2026-10-10T23:32:31-04:00", "result": { "commit": "7009ea98aba406d2cd4fecc27a2603a781953d12", "date": "2026-10-14T01:00:37-04:00", "short": "7009ea9" }, "rrd": { "file": "docs/RRD.md", "text": "Versions for the continuation, 13 Oct 2026 (registered before any run of them).", "where": "RM-35" }, "short": "25e3c87" }, "runs": 1827, "study": "RM-35", "title": "Courant-10K: the gate on thousands of real OpenFOAM runs", "versions": { "after_release": { "file": "docs/WHAT-COURANT-SEES.md", "text": "The three gate changes made after v0.5.0 ship behind the flag for one minor release before they are on by default", "where": "Behind --checks next" }, "changes": [ { "id": "VER-7a", "says": "the cell aspect ratio computed from constant/polyMesh when a case has no checkMesh log" }, { "id": "VER-16", "says": "the residual bar per field with its stalled-residual detector" } ], "default": { "file": "docs/WHAT-COURANT-SEES.md", "text": "courant verify and courant preflight without the flag give v0.5.0's documents, byte for byte", "where": "Behind --checks next" }, "flag": "--checks next", "gate_2": { "file": "docs/RRD.md", "text": "gate version 2 is VER-7a and VER-16 together", "where": "RM-35, versions for the continuation" }, "measured": 2, "never_pooled": { "file": "docs/BENCHMARKS.md", "text": "Versions are never pooled.", "where": "§11" }, "pooled": false, "release": { "commit": "4961e0a2c782b64a955d7bf39a15e434f19017b4", "date": "2026-10-12T12:51:03-04:00", "gate_version": 1, "short": "4961e0a", "version": "0.5.0" }, "same_gate": { "from": "4961e0a", "paths": [ "courant/verify", "courant/case", "courant/spec", "courant/diagnosis", "courant/events", "courant/verification", "courant/gate", "courant/fields", "courant/converge", "courant/taxonomy.py" ], "to": [ "1a93589", "8492645" ] }, "side_by_side": { "file": "docs/RRD.md", "text": "Both the interim (gate version 1) and the new version's result are published, side by side.", "where": "RM-35" } }, "work_order_rows": [ "H1b", "H2", "H5", "H3" ] }, "schema": "courant.site.benchmarks/1", "source": [ "docs/BENCHMARKS.md", "research/results/RM-36-latest.json", "courant/ctest/score.py", "research/results/RM-36-sample.json", "research/results/RM-36-claude-opus-5-5-2026-10-15.json", "research/results/RM-36-claude-sonnet-5-5-2026-10-15.json", "research/results/RM-36-claude-haiku-5-5-2026-10-15.json", "research/results/RM-35-v2-latest.json", "courant/silentbench/stats.py", "research/results/RM-35-partial.json" ], "source_sha256": { "courant/ctest/score.py": "476971d9ae30f52235186556d073e580514a7971403b404b21931d14aea25083", "courant/silentbench/stats.py": "7d6f4a8dbbba71745740e2dfa1203a2c99bad91e25d9f280c8e6be207e86f68c", "docs/BENCHMARKS.md": "e88ef1e8617a00cc3c9a6ce5b7235a5889bcf2faccc2402ee0ae96af9765da1c", "research/results/RM-35-partial.json": "d207b5dca4869be9ce5d0a705c52a13fd5c0885e9e289a86a95fab2c043affaf", "research/results/RM-35-v2-latest.json": "a5ccc373c22920942afa733b933613f6236822585ec2e4bf4c93dc9940b957eb", "research/results/RM-36-claude-haiku-5-5-2026-10-15.json": "0d31458b7ef4a7f7c6a3db1e6855f9874ce4b35811056d023c0391e51625b302", "research/results/RM-36-claude-opus-5-5-2026-10-15.json": "c5ec8aa6588a10875a148d21849e2c1265e496945703daed8e0aaf05aa3f2c2e", "research/results/RM-36-claude-sonnet-5-5-2026-10-15.json": "75d7d268db9c3811d41f36070f8904a3c60792d11ce021f021c4162622a1088a", "research/results/RM-36-latest.json": "9bd0ef9428817c650f4ed4bb2c01a7287778c99db557aca7767e0c835496d90a", "research/results/RM-36-sample.json": "788c877a2c9d19528f17efc2c943296c5c34af86f7f29d65bcf309f50055c0cd" } }