{"$defs":{"benchmark":{"additionalProperties":false,"properties":{"excluded_question_count":{"const":9},"exclusion_categories":{"additionalProperties":false,"properties":{"malformed":{"const":1},"multiple_defensible":{"const":7},"wrong_key":{"const":1}},"required":["multiple_defensible","malformed","wrong_key"],"type":"object"},"name":{"const":"GPQA-Diamond-Clean"},"original_question_count":{"const":198},"retained_question_count":{"const":189},"version":{"minLength":1,"type":"string"}},"required":["name","version","original_question_count","retained_question_count","excluded_question_count","exclusion_categories"],"type":"object"},"canonical_evidence":{"additionalProperties":false,"properties":{"contract_version":{"const":"gpqa-results-identity-v1"},"one_row_per_configuration":{"const":true},"score_blind":{"const":true},"selection_timestamp_policy":{"minLength":1,"type":"string"}},"required":["score_blind","one_row_per_configuration","contract_version","selection_timestamp_policy"],"type":"object"},"cohort_summary":{"additionalProperties":false,"properties":{"configuration_count":{"minimum":1,"type":"integer"},"decrease_count":{"minimum":0,"type":"integer"},"definition":{"minLength":1,"type":"string"},"first_quartile_delta":{"$ref":"#/$defs/delta"},"increase_count":{"minimum":0,"type":"integer"},"label":{"minLength":1,"type":"string"},"median_delta":{"$ref":"#/$defs/delta"},"third_quartile_delta":{"$ref":"#/$defs/delta"},"thresholds":{"items":{"$ref":"#/$defs/threshold"},"minItems":1,"type":"array"},"unchanged_count":{"minimum":0,"type":"integer"}},"required":["label","definition","configuration_count","increase_count","decrease_count","unchanged_count","median_delta","first_quartile_delta","third_quartile_delta","thresholds"],"type":"object"},"current_release_facts":{"additionalProperties":false,"properties":{"all_accepted_change_summary":{"$ref":"#/$defs/cohort_summary"},"direct_complete_epoch_change_summary":{"$ref":"#/$defs/cohort_summary"},"ranked_configuration_count":{"minimum":1,"type":"integer"},"top_50_coverage":{"$ref":"#/$defs/top_50_coverage"}},"required":["ranked_configuration_count","all_accepted_change_summary","direct_complete_epoch_change_summary","top_50_coverage"],"type":"object"},"delta":{"additionalProperties":false,"properties":{"denominator":{"minimum":1,"type":"integer"},"numerator":{"type":"integer"},"percentage_points":{"$ref":"#/$defs/points"}},"required":["numerator","denominator","percentage_points"],"type":"object"},"disclosure":{"additionalProperties":false,"properties":{"minimum_configurations":{"minimum":5,"type":"integer"},"minimum_questions":{"minimum":5,"type":"integer"},"prohibited":{"minLength":1,"type":"string"},"question_output":{"const":"aggregate_only"}},"required":["question_output","minimum_questions","minimum_configurations","prohibited"],"type":"object"},"eligibility":{"additionalProperties":false,"properties":{"material_source_conflicts_ranked_as_pure_source_results":{"const":false},"provider_transport_failures":{"minLength":1,"type":"string"},"required_clean_question_coverage":{"const":189},"required_original_question_coverage":{"const":198},"required_uniform_attempts":{"const":true}},"required":["required_original_question_coverage","required_clean_question_coverage","required_uniform_attempts","material_source_conflicts_ranked_as_pure_source_results","provider_transport_failures"],"type":"object"},"evidence_profile_counts":{"additionalProperties":false,"properties":{"derived_uniform":{"minimum":0,"type":"integer"},"direct_compatibility":{"minimum":0,"type":"integer"},"direct_strict":{"minimum":0,"type":"integer"},"project_native":{"minimum":0,"type":"integer"},"targeted_recovery":{"minimum":0,"type":"integer"}},"required":["project_native","direct_strict","direct_compatibility","targeted_recovery","derived_uniform"],"type":"object"},"paired_score_contract":{"additionalProperties":false,"properties":{"clean":{"minLength":1,"type":"string"},"delta":{"minLength":1,"type":"string"},"original":{"minLength":1,"type":"string"}},"required":["original","clean","delta"],"type":"object"},"points":{"pattern":"^(?:\\+|-)?(?:0|[1-9][0-9]?|100)\\.[0-9]{2} pp$","type":"string"},"ranking":{"additionalProperties":false,"properties":{"aggregation":{"const":"combined_mean_at_n"},"compute_normalized":{"const":false},"metric":{"minLength":1,"type":"string"},"ordering":{"minLength":1,"type":"string"},"protocol_interpretation":{"minLength":1,"type":"string"},"tie_policy":{"const":"competition exact-rational ties"}},"required":["aggregation","metric","ordering","tie_policy","protocol_interpretation","compute_normalized"],"type":"object"},"release_id":{"pattern":"^gpqa-clean-results-v[0-9]+\\.[0-9]+\\.[0-9]+$","type":"string"},"threshold":{"additionalProperties":false,"properties":{"clean_count":{"minimum":0,"type":"integer"},"crossed_down_count":{"minimum":0,"type":"integer"},"crossed_up_count":{"minimum":0,"type":"integer"},"paired_original_count":{"minimum":0,"type":"integer"},"threshold_percent":{"maximum":100,"minimum":0,"type":"integer"}},"required":["threshold_percent","paired_original_count","clean_count","crossed_up_count","crossed_down_count"],"type":"object"},"top_50_coverage":{"additionalProperties":false,"properties":{"covered_count":{"maximum":50,"minimum":0,"type":"integer"},"gap_count":{"maximum":50,"minimum":0,"type":"integer"},"source_top_n":{"const":50}},"required":["source_top_n","covered_count","gap_count"],"type":"object"},"uncertainty":{"additionalProperties":false,"properties":{"confidence_level":{"const":"95%"},"interpretation":{"minLength":1,"type":"string"},"method":{"minLength":1,"type":"string"},"replicates":{"minimum":20000,"type":"integer"},"seed":{"minimum":0,"type":"integer"}},"required":["method","replicates","seed","confidence_level","interpretation"],"type":"object"}},"$id":"https://gpqa.ai/schemas/gpqa-clean-results-methodology-v2.json","$schema":"https://json-schema.org/draft/2020-12/schema","additionalProperties":false,"properties":{"benchmark":{"$ref":"#/$defs/benchmark"},"canonical_evidence":{"$ref":"#/$defs/canonical_evidence"},"current_release_facts":{"$ref":"#/$defs/current_release_facts"},"disclosure":{"$ref":"#/$defs/disclosure"},"eligibility":{"$ref":"#/$defs/eligibility"},"evidence_profile_counts":{"$ref":"#/$defs/evidence_profile_counts"},"independence":{"minLength":1,"type":"string"},"paired_score_contract":{"$ref":"#/$defs/paired_score_contract"},"ranking":{"$ref":"#/$defs/ranking"},"release_id":{"$ref":"#/$defs/release_id"},"schema_version":{"const":"gpqa-clean-results-methodology-v2"},"title":{"minLength":1,"type":"string"},"uncertainty":{"$ref":"#/$defs/uncertainty"}},"required":["schema_version","release_id","title","benchmark","ranking","paired_score_contract","eligibility","canonical_evidence","evidence_profile_counts","uncertainty","disclosure","current_release_facts","independence"],"title":"GPQA Diamond Clean results methodology","type":"object"}