{"benchmark":{"excluded_question_count":9,"exclusion_categories":{"malformed":1,"multiple_defensible":7,"wrong_key":1},"name":"GPQA-Diamond-Clean","original_question_count":198,"retained_question_count":189,"version":"1.2.2"},"canonical_evidence":{"contract_version":"gpqa-results-identity-v1","one_row_per_configuration":true,"score_blind":true,"selection_timestamp_policy":"All hash-pinned accepted inputs are assigned the frozen release timestamp only for deterministic canonical selection. This administrative timestamp is not evidence creation, run, first-seen, or real admission time."},"current_release_facts":{"all_accepted_change_summary":{"configuration_count":243,"decrease_count":8,"definition":"Every canonical accepted direct, compatibility, derived-uniform, targeted-recovery, and independent-native configuration in the combined ranking.","first_quartile_delta":{"denominator":2772,"numerator":29,"percentage_points":"+1.05 pp"},"increase_count":235,"label":"All accepted configurations","median_delta":{"denominator":4158,"numerator":101,"percentage_points":"+2.43 pp"},"third_quartile_delta":{"denominator":2079,"numerator":62,"percentage_points":"+2.98 pp"},"thresholds":[{"clean_count":45,"crossed_down_count":0,"crossed_up_count":20,"paired_original_count":25,"threshold_percent":90},{"clean_count":14,"crossed_down_count":0,"crossed_up_count":14,"paired_original_count":0,"threshold_percent":95}],"unchanged_count":0},"direct_complete_epoch_change_summary":{"configuration_count":224,"decrease_count":8,"definition":"Only the 224 strict or compatibility-complete direct Epoch public artifacts; excludes derived, targeted-recovery, and independent-native configurations.","first_quartile_delta":{"denominator":88704,"numerator":785,"percentage_points":"+0.88 pp"},"increase_count":216,"label":"Direct complete Epoch artifacts","median_delta":{"denominator":8316,"numerator":193,"percentage_points":"+2.32 pp"},"third_quartile_delta":{"denominator":33264,"numerator":977,"percentage_points":"+2.94 pp"},"thresholds":[{"clean_count":31,"crossed_down_count":0,"crossed_up_count":18,"paired_original_count":13,"threshold_percent":90},{"clean_count":6,"crossed_down_count":0,"crossed_up_count":6,"paired_original_count":0,"threshold_percent":95}],"unchanged_count":0},"ranked_configuration_count":243,"top_50_coverage":{"covered_count":38,"gap_count":12,"source_top_n":50}},"disclosure":{"minimum_configurations":5,"minimum_questions":5,"prohibited":"Question text, options, keys, completions, reasoning, raw evaluation artifacts, private evidence locations, and per-configuration-by-question matrices.","question_output":"aggregate_only"},"eligibility":{"material_source_conflicts_ranked_as_pure_source_results":false,"provider_transport_failures":"Evidence states unless a predeclared terminal-no-answer policy explicitly scores the outcome incorrect.","required_clean_question_coverage":189,"required_original_question_coverage":198,"required_uniform_attempts":true},"evidence_profile_counts":{"derived_uniform":6,"direct_compatibility":116,"direct_strict":108,"project_native":9,"targeted_recovery":4},"independence":"Independent research project; inclusion is not a provider submission, certification, or endorsement.","paired_score_contract":{"clean":"Arithmetic mean accuracy over all 189 retained questions.","delta":"Clean minus paired Original; a published source aggregate is never mixed into this subtraction.","original":"Original 198 is reconstructed from the same accepted attempt universe as Clean 189."},"ranking":{"aggregation":"combined_mean_at_n","compute_normalized":false,"metric":"Clean accuracy over 189 retained questions","ordering":"descending exact rational Clean score","protocol_interpretation":"mean@N is the arithmetic mean across N attempts; it is not best-of-N, pass@N, or majority voting.","tie_policy":"competition exact-rational ties"},"release_id":"gpqa-clean-results-v0.1.0","schema_version":"gpqa-clean-results-methodology-v2","title":"GPQA Diamond Clean results methodology","uncertainty":{"confidence_level":"95%","interpretation":"Observed rank is descriptive; overlapping rank intervals show frontier order fragility.","method":"Synchronized Record-ID cluster bootstrap shared across configurations.","replicates":20000,"seed":20260817}}