{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://fartbench.org/schemas/fartbench-core-results-v2.schema.json",
  "title": "FARTBench Core results, schema v2",
  "description": "The results document for FARTBench Core, able to represent the frozen R01 analysis exactly: every reviewer's exact response state, 3-of-4 consensus, N/A with its reasons, the four separate factual outcomes, ratings with their contributing and N/A counts, human agreement, each AI judge's answers against the human consensus with the pre-registered calibration status, rating MAE reported separately from rating discrimination, and refusals kept apart from quality. There is no composite score. Supersedes fartbench-core-results-v1, which collapsed N/A reasons and used a simple majority.",
  "type": "object",
  "additionalProperties": false,
  "required": [
    "schema_version",
    "data_status",
    "identity_status",
    "benchmark",
    "round",
    "generated_utc",
    "provenance",
    "instruments",
    "definitions",
    "interpretation",
    "scenes",
    "reviewers",
    "coverage",
    "samples",
    "refusals",
    "configurations",
    "reliability",
    "ai_calibration",
    "notes"
  ],
  "properties": {
    "schema_version": {
      "const": "fartbench-core-results-v2"
    },
    "data_status": {
      "enum": [
        "synthetic-example",
        "blind-pre-reveal",
        "revealed-preliminary",
        "official"
      ],
      "description": "synthetic-example: invented. blind-pre-reveal: real frozen blind data with no configuration identity; never published. revealed-*: joined to configurations after an authorised reveal."
    },
    "identity_status": {
      "enum": [
        "synthetic",
        "blind",
        "revealed"
      ]
    },
    "benchmark": {
      "const": "FARTBench Core"
    },
    "round": {
      "type": "string",
      "minLength": 1
    },
    "generated_utc": {
      "type": "string"
    },
    "provenance": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "cohort_tag",
        "human_cohort_tag",
        "human_aggregate_sha256",
        "human_aggregate_tag",
        "comparison_sha256",
        "comparison_tag",
        "ai_evidence_tag",
        "mapping_commitment",
        "media_derivation_commitment",
        "media_set_commitment",
        "results_format_tag",
        "reveal_tag"
      ],
      "properties": {
        "cohort_tag": {
          "type": [
            "string",
            "null"
          ]
        },
        "human_cohort_tag": {
          "type": [
            "string",
            "null"
          ]
        },
        "human_aggregate_sha256": {
          "type": [
            "string",
            "null"
          ]
        },
        "human_aggregate_tag": {
          "type": [
            "string",
            "null"
          ]
        },
        "comparison_sha256": {
          "type": [
            "string",
            "null"
          ]
        },
        "comparison_tag": {
          "type": [
            "string",
            "null"
          ]
        },
        "ai_evidence_tag": {
          "type": [
            "string",
            "null"
          ]
        },
        "mapping_commitment": {
          "type": [
            "string",
            "null"
          ]
        },
        "media_derivation_commitment": {
          "type": [
            "string",
            "null"
          ]
        },
        "media_set_commitment": {
          "type": [
            "string",
            "null"
          ]
        },
        "results_format_tag": {
          "type": [
            "string",
            "null"
          ]
        },
        "reveal_tag": {
          "type": [
            "string",
            "null"
          ],
          "description": "null until the reveal has been authorised and executed"
        }
      }
    },
    "instruments": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "human",
        "ai"
      ],
      "properties": {
        "human": {
          "type": "string"
        },
        "ai": {
          "type": [
            "string",
            "null"
          ]
        }
      }
    },
    "definitions": {
      "type": "object",
      "additionalProperties": {
        "type": "string"
      }
    },
    "interpretation": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "source",
        "ai_rating_rule",
        "factual_primary",
        "descriptive_notes"
      ],
      "properties": {
        "source": {
          "type": "string"
        },
        "ai_rating_rule": {
          "type": "string"
        },
        "factual_primary": {
          "type": "string"
        },
        "descriptive_notes": {
          "type": "array",
          "items": {
            "type": "string"
          }
        }
      }
    },
    "scenes": {
      "type": "array",
      "minItems": 1,
      "items": {
        "$ref": "#/$defs/scene"
      }
    },
    "reviewers": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "human",
        "ai_judges",
        "consensus_rule",
        "panel_rule"
      ],
      "properties": {
        "human": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "ai_judges": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "consensus_rule": {
          "type": "string"
        },
        "panel_rule": {
          "type": "string"
        }
      }
    },
    "coverage": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "cells_total",
        "samples_evaluated",
        "refusals",
        "refusals_identified"
      ],
      "properties": {
        "cells_total": {
          "type": "integer",
          "minimum": 0
        },
        "samples_evaluated": {
          "type": "integer",
          "minimum": 0
        },
        "refusals": {
          "type": "integer",
          "minimum": 0
        },
        "refusals_identified": {
          "type": "boolean",
          "description": "false before the reveal: the refused cell is counted but not identified"
        }
      }
    },
    "samples": {
      "type": "array",
      "items": {
        "$ref": "#/$defs/sample"
      }
    },
    "refusals": {
      "type": "array",
      "items": {
        "$ref": "#/$defs/refusal"
      }
    },
    "configurations": {
      "type": "array",
      "items": {
        "$ref": "#/$defs/configuration"
      }
    },
    "reliability": {
      "$ref": "#/$defs/reliability"
    },
    "ai_calibration": {
      "$ref": "#/$defs/ai_calibration"
    },
    "notes": {
      "type": "array",
      "items": {
        "type": "string"
      }
    }
  },
  "allOf": [
    {
      "if": {
        "properties": {
          "identity_status": {
            "enum": [
              "blind"
            ]
          }
        },
        "required": [
          "identity_status"
        ]
      },
      "then": {
        "properties": {
          "configurations": {
            "maxItems": 0
          },
          "refusals": {
            "maxItems": 0
          },
          "samples": {
            "items": {
              "properties": {
                "configuration_id": {
                  "type": "null"
                }
              }
            }
          },
          "coverage": {
            "properties": {
              "refusals_identified": {
                "const": false
              }
            }
          }
        }
      }
    },
    {
      "if": {
        "properties": {
          "data_status": {
            "const": "synthetic-example"
          }
        },
        "required": [
          "data_status"
        ]
      },
      "then": {
        "properties": {
          "identity_status": {
            "const": "synthetic"
          }
        }
      }
    }
  ],
  "$defs": {
    "state": {
      "type": "string",
      "pattern": "^(yes|no|unclear|n/a:(primary_event_absent|required_sound_absent|causal_prerequisite_absent|not_observable))$",
      "description": "An exact normalized response state. N/A carries its reason and is never No or Unclear."
    },
    "nullable_state": {
      "oneOf": [
        {
          "$ref": "#/$defs/state"
        },
        {
          "type": "null"
        }
      ]
    },
    "rate": {
      "type": [
        "number",
        "null"
      ],
      "minimum": 0,
      "maximum": 1
    },
    "outcome_value": {
      "enum": [
        "success",
        "not_success",
        "no_consensus"
      ]
    },
    "scene": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "scene_id",
        "title",
        "primary_event_gate",
        "primary_causal_gate",
        "questions"
      ],
      "properties": {
        "scene_id": {
          "type": "string"
        },
        "title": {
          "type": "string"
        },
        "primary_event_gate": {
          "type": "string"
        },
        "primary_causal_gate": {
          "type": "string"
        },
        "questions": {
          "type": "array",
          "minItems": 3,
          "maxItems": 3,
          "items": {
            "type": "object",
            "additionalProperties": false,
            "required": [
              "question_id",
              "field",
              "question",
              "role"
            ],
            "properties": {
              "question_id": {
                "type": "string"
              },
              "field": {
                "type": "string"
              },
              "question": {
                "type": "string"
              },
              "role": {
                "enum": [
                  "primary_event",
                  "primary_causal",
                  "other"
                ]
              }
            }
          }
        }
      }
    },
    "factual": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "question_id",
        "role",
        "raw",
        "counts",
        "na_reason_counts",
        "consensus",
        "consensus_exists",
        "consensus_votes",
        "contributing_reviewers",
        "unanimous",
        "split",
        "pairwise_exact_agreement"
      ],
      "properties": {
        "question_id": {
          "type": "string"
        },
        "role": {
          "enum": [
            "primary_event",
            "primary_causal",
            "other"
          ]
        },
        "raw": {
          "type": "object",
          "additionalProperties": {
            "$ref": "#/$defs/state"
          }
        },
        "counts": {
          "type": "object",
          "additionalProperties": false,
          "required": [
            "yes",
            "no",
            "unclear",
            "na"
          ],
          "properties": {
            "yes": {
              "type": "integer",
              "minimum": 0
            },
            "no": {
              "type": "integer",
              "minimum": 0
            },
            "unclear": {
              "type": "integer",
              "minimum": 0
            },
            "na": {
              "type": "integer",
              "minimum": 0
            }
          }
        },
        "na_reason_counts": {
          "type": "object",
          "additionalProperties": {
            "type": "integer",
            "minimum": 1
          }
        },
        "consensus": {
          "$ref": "#/$defs/nullable_state"
        },
        "consensus_exists": {
          "type": "boolean"
        },
        "consensus_votes": {
          "type": "integer",
          "minimum": 0
        },
        "contributing_reviewers": {
          "type": "integer",
          "minimum": 0
        },
        "unanimous": {
          "type": "boolean"
        },
        "split": {
          "type": "boolean"
        },
        "pairwise_exact_agreement": {
          "$ref": "#/$defs/rate"
        }
      }
    },
    "outcome": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "value",
        "applicable",
        "inputs"
      ],
      "properties": {
        "value": {
          "$ref": "#/$defs/outcome_value"
        },
        "applicable": {
          "type": "boolean"
        },
        "inputs": {
          "type": "object",
          "additionalProperties": {
            "$ref": "#/$defs/nullable_state"
          },
          "description": "question_id -> the human consensus state it derives from"
        }
      }
    },
    "rating": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "raw",
        "numeric_reviewers",
        "na_count",
        "na_reason_counts",
        "mean",
        "median",
        "sd",
        "min",
        "max"
      ],
      "properties": {
        "raw": {
          "type": "object",
          "additionalProperties": {
            "type": [
              "integer",
              "null"
            ],
            "minimum": 1,
            "maximum": 5
          }
        },
        "numeric_reviewers": {
          "type": "integer",
          "minimum": 0
        },
        "na_count": {
          "type": "integer",
          "minimum": 0
        },
        "na_reason_counts": {
          "type": "object",
          "additionalProperties": {
            "type": "integer",
            "minimum": 1
          }
        },
        "mean": {
          "type": [
            "number",
            "null"
          ],
          "minimum": 1,
          "maximum": 5
        },
        "median": {
          "type": [
            "number",
            "null"
          ],
          "minimum": 1,
          "maximum": 5
        },
        "sd": {
          "type": [
            "number",
            "null"
          ],
          "minimum": 0
        },
        "min": {
          "type": [
            "integer",
            "null"
          ],
          "minimum": 1,
          "maximum": 5
        },
        "max": {
          "type": [
            "integer",
            "null"
          ],
          "minimum": 1,
          "maximum": 5
        }
      }
    },
    "ai_factual": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "state",
        "confidence",
        "agrees_with_human_consensus",
        "calibration_eligible"
      ],
      "properties": {
        "state": {
          "$ref": "#/$defs/state"
        },
        "confidence": {
          "type": [
            "integer",
            "null"
          ],
          "enum": [
            1,
            2,
            3,
            null
          ]
        },
        "agrees_with_human_consensus": {
          "type": [
            "boolean",
            "null"
          ]
        },
        "calibration_eligible": {
          "type": "boolean"
        }
      }
    },
    "ai_rating": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "value",
        "rating_eligible",
        "abs_error",
        "signed_error"
      ],
      "properties": {
        "value": {
          "type": [
            "integer",
            "null"
          ],
          "minimum": 1,
          "maximum": 5
        },
        "rating_eligible": {
          "type": "boolean"
        },
        "abs_error": {
          "type": [
            "number",
            "null"
          ],
          "minimum": 0
        },
        "signed_error": {
          "type": [
            "number",
            "null"
          ]
        }
      }
    },
    "sample": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "sample_id",
        "scene_id",
        "configuration_id",
        "factual",
        "outcomes",
        "ratings",
        "ai"
      ],
      "properties": {
        "sample_id": {
          "type": "string"
        },
        "scene_id": {
          "type": "string"
        },
        "configuration_id": {
          "type": [
            "string",
            "null"
          ]
        },
        "factual": {
          "type": "object",
          "additionalProperties": {
            "$ref": "#/$defs/factual"
          }
        },
        "outcomes": {
          "type": "object",
          "additionalProperties": false,
          "required": [
            "event_success",
            "primary_causal_success",
            "strict_scene_success",
            "full_factual_success"
          ],
          "properties": {
            "event_success": {
              "$ref": "#/$defs/outcome"
            },
            "primary_causal_success": {
              "$ref": "#/$defs/outcome"
            },
            "strict_scene_success": {
              "$ref": "#/$defs/outcome"
            },
            "full_factual_success": {
              "$ref": "#/$defs/outcome"
            }
          }
        },
        "ratings": {
          "type": "object",
          "additionalProperties": {
            "$ref": "#/$defs/rating"
          }
        },
        "ai": {
          "type": "object",
          "additionalProperties": false,
          "required": [
            "judges",
            "panel"
          ],
          "properties": {
            "judges": {
              "type": "object",
              "additionalProperties": {
                "type": "object",
                "additionalProperties": false,
                "required": [
                  "factual",
                  "ratings"
                ],
                "properties": {
                  "factual": {
                    "type": "object",
                    "additionalProperties": {
                      "$ref": "#/$defs/ai_factual"
                    }
                  },
                  "ratings": {
                    "type": "object",
                    "additionalProperties": {
                      "$ref": "#/$defs/ai_rating"
                    }
                  }
                }
              }
            },
            "panel": {
              "type": "object",
              "additionalProperties": {
                "type": "object",
                "additionalProperties": false,
                "required": [
                  "vote_counts",
                  "consensus",
                  "agrees_with_human_consensus",
                  "calibration_eligible"
                ],
                "properties": {
                  "vote_counts": {
                    "type": "object",
                    "additionalProperties": {
                      "type": "integer",
                      "minimum": 1
                    }
                  },
                  "consensus": {
                    "$ref": "#/$defs/nullable_state"
                  },
                  "agrees_with_human_consensus": {
                    "type": [
                      "boolean",
                      "null"
                    ]
                  },
                  "calibration_eligible": {
                    "type": "boolean"
                  }
                }
              }
            }
          }
        }
      }
    },
    "refusal": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "cell_id",
        "configuration_id",
        "scene_id",
        "canonical_outcome",
        "message_category",
        "provider_message",
        "provider_message_publication",
        "human",
        "ai",
        "supplemental_variant"
      ],
      "properties": {
        "cell_id": {
          "type": "string"
        },
        "configuration_id": {
          "type": "string"
        },
        "scene_id": {
          "type": "string"
        },
        "canonical_outcome": {
          "const": "refusal"
        },
        "message_category": {
          "type": "string"
        },
        "provider_message": {
          "type": [
            "string",
            "null"
          ]
        },
        "provider_message_publication": {
          "enum": [
            "published",
            "withheld"
          ]
        },
        "human": {
          "type": "null",
          "description": "a refused cell has no clip, so no human answer exists"
        },
        "ai": {
          "type": "null",
          "description": "and no AI answer exists"
        },
        "supplemental_variant": {
          "type": "object",
          "additionalProperties": false,
          "required": [
            "exists",
            "excluded_from_canonical_scoring"
          ],
          "properties": {
            "exists": {
              "type": "boolean"
            },
            "excluded_from_canonical_scoring": {
              "const": true
            }
          }
        }
      }
    },
    "count_outcome": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "success",
        "not_success",
        "no_consensus",
        "evaluated"
      ],
      "properties": {
        "success": {
          "type": "integer",
          "minimum": 0
        },
        "not_success": {
          "type": "integer",
          "minimum": 0
        },
        "no_consensus": {
          "type": "integer",
          "minimum": 0
        },
        "evaluated": {
          "type": "integer",
          "minimum": 0
        }
      }
    },
    "pooled_rating": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "numeric_responses",
        "na_responses",
        "clips_with_numeric",
        "mean",
        "median",
        "sd"
      ],
      "properties": {
        "numeric_responses": {
          "type": "integer",
          "minimum": 0
        },
        "na_responses": {
          "type": "integer",
          "minimum": 0
        },
        "clips_with_numeric": {
          "type": "integer",
          "minimum": 0
        },
        "mean": {
          "type": [
            "number",
            "null"
          ],
          "minimum": 1,
          "maximum": 5
        },
        "median": {
          "type": [
            "number",
            "null"
          ],
          "minimum": 1,
          "maximum": 5
        },
        "sd": {
          "type": [
            "number",
            "null"
          ],
          "minimum": 0
        }
      }
    },
    "configuration": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "configuration_id",
        "label",
        "family",
        "sample_ids",
        "refusal_cell_ids",
        "summary"
      ],
      "properties": {
        "configuration_id": {
          "type": "string"
        },
        "label": {
          "type": "string"
        },
        "family": {
          "type": "string"
        },
        "sample_ids": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "refusal_cell_ids": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "summary": {
          "type": "object",
          "additionalProperties": false,
          "required": [
            "scenes_total",
            "scenes_evaluated",
            "scenes_refused",
            "event_success",
            "primary_causal_success",
            "strict_scene_success",
            "full_factual_success",
            "ratings",
            "human_questions",
            "human_questions_with_consensus",
            "ai_agreement"
          ],
          "properties": {
            "scenes_total": {
              "type": "integer"
            },
            "scenes_evaluated": {
              "type": "integer"
            },
            "scenes_refused": {
              "type": "integer"
            },
            "event_success": {
              "$ref": "#/$defs/count_outcome"
            },
            "primary_causal_success": {
              "$ref": "#/$defs/count_outcome"
            },
            "strict_scene_success": {
              "$ref": "#/$defs/count_outcome"
            },
            "full_factual_success": {
              "$ref": "#/$defs/count_outcome"
            },
            "ratings": {
              "type": "object",
              "additionalProperties": {
                "$ref": "#/$defs/pooled_rating"
              }
            },
            "human_questions": {
              "type": "integer"
            },
            "human_questions_with_consensus": {
              "type": "integer"
            },
            "ai_agreement": {
              "type": "object",
              "additionalProperties": {
                "type": "object",
                "additionalProperties": false,
                "required": [
                  "agree",
                  "compared"
                ],
                "properties": {
                  "agree": {
                    "type": "integer"
                  },
                  "compared": {
                    "type": "integer"
                  }
                }
              }
            }
          }
        }
      }
    },
    "agreement": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "cells",
        "consensus",
        "no_consensus",
        "consensus_rate",
        "unanimous",
        "split",
        "unanimous_rate",
        "pairwise_exact_agreement",
        "alpha",
        "alpha_ci95",
        "alpha_metric",
        "raw_vote_counts"
      ],
      "properties": {
        "cells": {
          "type": "integer"
        },
        "consensus": {
          "type": "integer"
        },
        "no_consensus": {
          "type": "integer"
        },
        "consensus_rate": {
          "$ref": "#/$defs/rate"
        },
        "unanimous": {
          "type": "integer"
        },
        "split": {
          "type": "integer"
        },
        "unanimous_rate": {
          "$ref": "#/$defs/rate"
        },
        "pairwise_exact_agreement": {
          "$ref": "#/$defs/rate"
        },
        "alpha": {
          "type": [
            "number",
            "null"
          ]
        },
        "alpha_ci95": {
          "oneOf": [
            {
              "type": "array",
              "items": {
                "type": "number"
              },
              "minItems": 2,
              "maxItems": 2
            },
            {
              "type": "null"
            }
          ]
        },
        "alpha_metric": {
          "type": "string"
        },
        "raw_vote_counts": {
          "type": "object",
          "additionalProperties": {
            "type": "integer"
          },
          "description": "every individual vote, by exact state, so prevalence is visible beside alpha"
        }
      }
    },
    "reliability": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "per_question",
        "per_scene",
        "overall",
        "pairwise_reviewers",
        "ratings",
        "caveats"
      ],
      "properties": {
        "per_question": {
          "type": "object",
          "additionalProperties": {
            "$ref": "#/$defs/agreement"
          }
        },
        "per_scene": {
          "type": "object",
          "additionalProperties": {
            "$ref": "#/$defs/agreement"
          }
        },
        "overall": {
          "$ref": "#/$defs/agreement"
        },
        "pairwise_reviewers": {
          "type": "object",
          "additionalProperties": {
            "$ref": "#/$defs/rate"
          }
        },
        "ratings": {
          "type": "object",
          "additionalProperties": {
            "type": "object",
            "additionalProperties": false,
            "required": [
              "numeric",
              "not_applicable",
              "alpha",
              "alpha_ci95",
              "alpha_metric",
              "pairwise_exact_agreement",
              "pairwise_within_one_point",
              "mean_within_clip_sd"
            ],
            "properties": {
              "numeric": {
                "type": "integer"
              },
              "not_applicable": {
                "type": "integer"
              },
              "alpha": {
                "type": [
                  "number",
                  "null"
                ]
              },
              "alpha_ci95": {
                "oneOf": [
                  {
                    "type": "array",
                    "items": {
                      "type": "number"
                    },
                    "minItems": 2,
                    "maxItems": 2
                  },
                  {
                    "type": "null"
                  }
                ]
              },
              "alpha_metric": {
                "type": "string"
              },
              "pairwise_exact_agreement": {
                "$ref": "#/$defs/rate"
              },
              "pairwise_within_one_point": {
                "$ref": "#/$defs/rate"
              },
              "mean_within_clip_sd": {
                "type": [
                  "number",
                  "null"
                ]
              }
            }
          }
        },
        "caveats": {
          "type": "array",
          "items": {
            "type": "object",
            "additionalProperties": false,
            "required": [
              "scope",
              "target",
              "kind",
              "text"
            ],
            "properties": {
              "scope": {
                "enum": [
                  "scene",
                  "question"
                ]
              },
              "target": {
                "type": "string"
              },
              "kind": {
                "type": "string"
              },
              "text": {
                "type": "string"
              }
            }
          }
        }
      }
    },
    "calibration_status": {
      "enum": [
        "calibrated",
        "not_calibrated",
        "not_calibrated_for_primary_causal_gate",
        "not_evaluable"
      ]
    },
    "agreement_block": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "agree",
        "compared",
        "rate",
        "wilson95",
        "status"
      ],
      "properties": {
        "agree": {
          "type": "integer",
          "minimum": 0
        },
        "compared": {
          "type": "integer",
          "minimum": 0
        },
        "rate": {
          "$ref": "#/$defs/rate"
        },
        "wilson95": {
          "oneOf": [
            {
              "type": "array",
              "items": {
                "type": "number"
              },
              "minItems": 2,
              "maxItems": 2
            },
            {
              "type": "null"
            }
          ]
        },
        "status": {
          "$ref": "#/$defs/calibration_status"
        }
      }
    },
    "ai_calibration": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "thresholds",
        "gates",
        "pooled",
        "ratings",
        "headline",
        "disagreements"
      ],
      "properties": {
        "thresholds": {
          "type": "object",
          "additionalProperties": false,
          "required": [
            "ordinary_gate",
            "primary_causal_gate",
            "rating_mae",
            "rating_min_human_numeric",
            "panel_requires"
          ],
          "properties": {
            "ordinary_gate": {
              "type": "number"
            },
            "primary_causal_gate": {
              "type": "number"
            },
            "rating_mae": {
              "type": "number"
            },
            "rating_min_human_numeric": {
              "type": "integer"
            },
            "panel_requires": {
              "type": "string"
            }
          }
        },
        "gates": {
          "type": "object",
          "additionalProperties": {
            "type": "object",
            "additionalProperties": false,
            "required": [
              "role",
              "threshold",
              "human_consensus_samples",
              "human_no_consensus_samples",
              "judges",
              "panel"
            ],
            "properties": {
              "role": {
                "enum": [
                  "primary_event",
                  "primary_causal",
                  "other"
                ]
              },
              "threshold": {
                "type": "number"
              },
              "human_consensus_samples": {
                "type": "integer"
              },
              "human_no_consensus_samples": {
                "type": "integer"
              },
              "judges": {
                "type": "object",
                "additionalProperties": {
                  "$ref": "#/$defs/agreement_block"
                }
              },
              "panel": {
                "$ref": "#/$defs/panel_block"
              }
            }
          }
        },
        "pooled": {
          "type": "object"
        },
        "ratings": {
          "type": "object",
          "additionalProperties": {
            "type": "object",
            "additionalProperties": false,
            "required": [
              "human_eligible_samples",
              "judges"
            ],
            "properties": {
              "human_eligible_samples": {
                "type": "integer"
              },
              "judges": {
                "type": "object",
                "additionalProperties": {
                  "type": "object",
                  "additionalProperties": false,
                  "required": [
                    "eligible_samples",
                    "mae",
                    "bias",
                    "spearman",
                    "pearson",
                    "correlation_defined",
                    "ai_distinct_values",
                    "ai_modal_value",
                    "ai_modal_share",
                    "ai_variance",
                    "mae_threshold_met",
                    "discrimination_demonstrated",
                    "discrimination_basis",
                    "summary_label",
                    "ai_na_where_humans_rated",
                    "ai_numeric_where_humans_did_not"
                  ],
                  "properties": {
                    "eligible_samples": {
                      "type": "integer"
                    },
                    "mae": {
                      "type": [
                        "number",
                        "null"
                      ]
                    },
                    "bias": {
                      "type": [
                        "number",
                        "null"
                      ]
                    },
                    "spearman": {
                      "type": [
                        "number",
                        "null"
                      ]
                    },
                    "pearson": {
                      "type": [
                        "number",
                        "null"
                      ]
                    },
                    "correlation_defined": {
                      "type": "boolean"
                    },
                    "ai_distinct_values": {
                      "type": "integer"
                    },
                    "ai_modal_value": {
                      "type": [
                        "integer",
                        "null"
                      ]
                    },
                    "ai_modal_share": {
                      "$ref": "#/$defs/rate"
                    },
                    "ai_variance": {
                      "type": [
                        "number",
                        "null"
                      ]
                    },
                    "mae_threshold_met": {
                      "type": [
                        "boolean",
                        "null"
                      ],
                      "description": "the pre-registered criterion, exactly as computed"
                    },
                    "discrimination_demonstrated": {
                      "type": [
                        "boolean",
                        "null"
                      ],
                      "description": "descriptive, from the owner's recorded interpretation; not a threshold"
                    },
                    "discrimination_basis": {
                      "type": "string"
                    },
                    "summary_label": {
                      "type": "string"
                    },
                    "ai_na_where_humans_rated": {
                      "type": "integer"
                    },
                    "ai_numeric_where_humans_did_not": {
                      "type": "integer"
                    }
                  }
                }
              }
            }
          }
        },
        "headline": {
          "type": "object",
          "additionalProperties": {
            "type": "object",
            "additionalProperties": false,
            "required": [
              "ordinary_gates_calibrated",
              "ordinary_gates",
              "primary_causal_gates_calibrated",
              "primary_causal_gates",
              "not_calibrated_for_primary_causal_gate",
              "not_calibrated",
              "not_evaluable",
              "rating_mae_threshold_met",
              "rating_discrimination_demonstrated"
            ],
            "properties": {
              "ordinary_gates_calibrated": {
                "type": "integer"
              },
              "ordinary_gates": {
                "type": "integer"
              },
              "primary_causal_gates_calibrated": {
                "type": "integer"
              },
              "primary_causal_gates": {
                "type": "integer"
              },
              "not_calibrated_for_primary_causal_gate": {
                "type": "array",
                "items": {
                  "type": "string"
                }
              },
              "not_calibrated": {
                "type": "array",
                "items": {
                  "type": "string"
                }
              },
              "not_evaluable": {
                "type": "array",
                "items": {
                  "type": "string"
                }
              },
              "rating_mae_threshold_met": {
                "type": [
                  "integer",
                  "null"
                ]
              },
              "rating_discrimination_demonstrated": {
                "type": [
                  "integer",
                  "null"
                ]
              }
            }
          }
        },
        "disagreements": {
          "type": "object",
          "additionalProperties": {
            "type": "array",
            "items": {
              "type": "object",
              "additionalProperties": false,
              "required": [
                "sample_id",
                "question_id"
              ],
              "properties": {
                "sample_id": {
                  "type": "string"
                },
                "question_id": {
                  "type": "string"
                }
              }
            }
          }
        }
      }
    },
    "panel_block": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "agree",
        "compared",
        "rate",
        "wilson95",
        "status",
        "panel_without_majority"
      ],
      "properties": {
        "agree": {
          "type": "integer",
          "minimum": 0
        },
        "compared": {
          "type": "integer",
          "minimum": 0
        },
        "rate": {
          "$ref": "#/$defs/rate"
        },
        "wilson95": {
          "oneOf": [
            {
              "type": "array",
              "items": {
                "type": "number"
              },
              "minItems": 2,
              "maxItems": 2
            },
            {
              "type": "null"
            }
          ]
        },
        "status": {
          "$ref": "#/$defs/calibration_status"
        },
        "panel_without_majority": {
          "type": "integer",
          "minimum": 0
        }
      }
    }
  }
}