{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://fartbench.org/schemas/fartbench-core-r01-ai-evaluation-v4.schema.json",
  "title": "FARTBench Core R01 AI Evaluation Record, instrument V4",
  "description": "One blind AI judge's review of one clip under fartbench-core-r01-ai-v4, the AI mirror of human instrument fartbench-core-r01-human-v3. The factual gates, ratings, N/A answer objects, scene constraints, eligibility and scoring blocks are the human V3 schema's own definitions, so an AI record and a human record answer the same six questions in the same machine-readable form. judge_observations and judge_confidence are AI-only and never enter a human comparison as answers.",
  "type": "object",
  "additionalProperties": false,
  "required": [
    "schema",
    "instrument_version",
    "mirrors_human_instrument_version",
    "pool_version",
    "cohort_tag",
    "run_id",
    "rater_id",
    "evaluator",
    "blind_sample_id",
    "scene_id",
    "primary_event_gate",
    "primary_causal_gate",
    "rating_scale",
    "factual",
    "ratings",
    "judge_observations",
    "judge_confidence",
    "notes",
    "eligibility",
    "scoring",
    "locked_timestamp_utc"
  ],
  "properties": {
    "schema": {
      "type": "string",
      "const": "benchmark/schemas/fartbench-core-r01-ai-evaluation-v4.schema.json"
    },
    "instrument_version": {
      "type": "string",
      "const": "fartbench-core-r01-ai-v4"
    },
    "mirrors_human_instrument_version": {
      "type": "string",
      "const": "fartbench-core-r01-human-v3"
    },
    "pool_version": {
      "type": "string",
      "const": "core-r01-v4"
    },
    "cohort_tag": {
      "type": "string",
      "const": "fartbench-core-r00-generation-cohort-v1-locked"
    },
    "run_id": {
      "type": "string",
      "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
    },
    "rater_id": {
      "type": "string",
      "enum": [
        "gemini-3.8-flash",
        "gemini-3.7-flash",
        "gemini-3.6-flash"
      ]
    },
    "evaluator": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "type",
        "model",
        "panel_position",
        "protocol_id"
      ],
      "properties": {
        "type": {
          "type": "string",
          "const": "ai-av"
        },
        "model": {
          "type": "string",
          "enum": [
            "gemini-3.8-flash",
            "gemini-3.7-flash",
            "gemini-3.6-flash"
          ]
        },
        "panel_position": {
          "type": "integer",
          "minimum": 1,
          "maximum": 3
        },
        "protocol_id": {
          "type": "string",
          "const": "fartbench-core-r01-v4-blind-ai-judging"
        }
      }
    },
    "blind_sample_id": {
      "type": "string",
      "pattern": "^[A-Z0-9]{5,}$",
      "description": "Opaque. Encodes nothing about model, configuration, route, resolution or origin."
    },
    "scene_id": {
      "type": "string",
      "enum": [
        "FB-C02",
        "FB-014",
        "FB-074",
        "FB-512"
      ]
    },
    "primary_event_gate": {
      "type": "string",
      "description": "The gate asking whether the scene's event happened. Instrument metadata fixed per scene. A No here is first-class benchmark evidence in its own right and is never translated into low quality ratings.",
      "enum": [
        "single_event_present",
        "separate_fart_event_present"
      ]
    },
    "primary_causal_gate": {
      "type": "string",
      "description": "The gate carrying the scene's causal question. Instrument metadata fixed per scene, recorded so a failure there stays separately reportable.",
      "enum": [
        "source_was_red_jacket_woman",
        "event_connected_to_lift",
        "creak_and_fart_distinguishable",
        "recovered_without_further_emission"
      ]
    },
    "rating_scale": {
      "type": "string",
      "const": "1-5"
    },
    "factual": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "single_event_present": {
          "$ref": "#/$defs/gate_answer"
        },
        "source_was_red_jacket_woman": {
          "$ref": "#/$defs/gate_answer"
        },
        "bystanders_directed_attention": {
          "$ref": "#/$defs/gate_answer"
        },
        "event_connected_to_lift": {
          "$ref": "#/$defs/gate_answer"
        },
        "body_and_clothing_plausible": {
          "$ref": "#/$defs/gate_answer"
        },
        "chair_creak_present": {
          "$ref": "#/$defs/gate_answer"
        },
        "separate_fart_event_present": {
          "$ref": "#/$defs/gate_answer"
        },
        "creak_and_fart_distinguishable": {
          "$ref": "#/$defs/gate_answer"
        },
        "both_people_reacted": {
          "$ref": "#/$defs/gate_answer"
        },
        "recovered_without_further_emission": {
          "$ref": "#/$defs/gate_answer"
        }
      }
    },
    "ratings": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "acoustic_plausibility",
        "temporal_causal_coherence",
        "physical_social_plausibility"
      ],
      "properties": {
        "acoustic_plausibility": {
          "$ref": "#/$defs/rating_answer"
        },
        "temporal_causal_coherence": {
          "$ref": "#/$defs/rating_answer"
        },
        "physical_social_plausibility": {
          "$ref": "#/$defs/rating_answer"
        }
      }
    },
    "judge_observations": {
      "type": "object",
      "additionalProperties": {
        "type": "string"
      }
    },
    "judge_confidence": {
      "type": "object",
      "additionalProperties": {
        "type": "integer",
        "enum": [
          1,
          2,
          3
        ]
      }
    },
    "notes": {
      "type": [
        "string",
        "null"
      ]
    },
    "eligibility": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "complete",
        "flags"
      ],
      "properties": {
        "complete": {
          "type": "boolean",
          "description": "Every one of the six fields carries an answer or an N/A, and playback completed. N/A counts as answered."
        },
        "flags": {
          "type": "array",
          "items": {
            "type": "object",
            "additionalProperties": false,
            "required": [
              "code",
              "label"
            ],
            "properties": {
              "code": {
                "type": "string"
              },
              "label": {
                "type": "string"
              },
              "detail": {
                "type": [
                  "string",
                  "null"
                ]
              }
            }
          }
        }
      }
    },
    "scoring": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "composite_formula_frozen_at_lock",
        "composite_class",
        "completeness"
      ],
      "properties": {
        "composite_formula_frozen_at_lock": {
          "type": "boolean",
          "const": false
        },
        "composite_class": {
          "type": "string",
          "const": "none"
        },
        "completeness": {
          "type": "string",
          "enum": [
            "complete",
            "incomplete"
          ]
        },
        "gates_reported_separately": {
          "type": "boolean",
          "const": true
        },
        "not_applicable_excluded_from_means": {
          "type": "boolean",
          "const": true
        }
      }
    },
    "locked_timestamp_utc": {
      "type": "string",
      "format": "date-time"
    }
  },
  "allOf": [
    {
      "if": {
        "properties": {
          "scene_id": {
            "const": "FB-C02"
          }
        },
        "required": [
          "scene_id"
        ]
      },
      "then": {
        "properties": {
          "primary_event_gate": {
            "const": "single_event_present"
          },
          "primary_causal_gate": {
            "const": "source_was_red_jacket_woman"
          },
          "factual": {
            "required": [
              "single_event_present",
              "source_was_red_jacket_woman",
              "bystanders_directed_attention"
            ],
            "maxProperties": 3,
            "properties": {
              "single_event_present": {
                "$ref": "#/$defs/answer_never_na"
              }
            }
          }
        }
      }
    },
    {
      "if": {
        "properties": {
          "scene_id": {
            "const": "FB-014"
          }
        },
        "required": [
          "scene_id"
        ]
      },
      "then": {
        "properties": {
          "primary_event_gate": {
            "const": "single_event_present"
          },
          "primary_causal_gate": {
            "const": "event_connected_to_lift"
          },
          "factual": {
            "required": [
              "single_event_present",
              "event_connected_to_lift",
              "body_and_clothing_plausible"
            ],
            "maxProperties": 3,
            "properties": {
              "single_event_present": {
                "$ref": "#/$defs/answer_never_na"
              }
            }
          }
        }
      }
    },
    {
      "if": {
        "properties": {
          "scene_id": {
            "const": "FB-074"
          }
        },
        "required": [
          "scene_id"
        ]
      },
      "then": {
        "properties": {
          "primary_event_gate": {
            "const": "separate_fart_event_present"
          },
          "primary_causal_gate": {
            "const": "creak_and_fart_distinguishable"
          },
          "factual": {
            "required": [
              "chair_creak_present",
              "separate_fart_event_present",
              "creak_and_fart_distinguishable"
            ],
            "maxProperties": 3,
            "properties": {
              "chair_creak_present": {
                "$ref": "#/$defs/answer_never_na"
              },
              "separate_fart_event_present": {
                "$ref": "#/$defs/answer_never_na"
              }
            }
          }
        }
      }
    },
    {
      "if": {
        "properties": {
          "scene_id": {
            "const": "FB-512"
          }
        },
        "required": [
          "scene_id"
        ]
      },
      "then": {
        "properties": {
          "primary_event_gate": {
            "const": "single_event_present"
          },
          "primary_causal_gate": {
            "const": "recovered_without_further_emission"
          },
          "factual": {
            "required": [
              "single_event_present",
              "both_people_reacted",
              "recovered_without_further_emission"
            ],
            "maxProperties": 3,
            "properties": {
              "single_event_present": {
                "$ref": "#/$defs/answer_never_na"
              }
            }
          }
        }
      }
    },
    {
      "if": {
        "properties": {
          "scene_id": {
            "const": "FB-C02"
          }
        },
        "required": [
          "scene_id"
        ]
      },
      "then": {
        "properties": {
          "judge_observations": {
            "required": [
              "single_event_present",
              "source_was_red_jacket_woman",
              "bystanders_directed_attention"
            ],
            "maxProperties": 3
          },
          "judge_confidence": {
            "required": [
              "single_event_present",
              "source_was_red_jacket_woman",
              "bystanders_directed_attention"
            ],
            "maxProperties": 3
          }
        }
      }
    },
    {
      "if": {
        "properties": {
          "scene_id": {
            "const": "FB-014"
          }
        },
        "required": [
          "scene_id"
        ]
      },
      "then": {
        "properties": {
          "judge_observations": {
            "required": [
              "single_event_present",
              "event_connected_to_lift",
              "body_and_clothing_plausible"
            ],
            "maxProperties": 3
          },
          "judge_confidence": {
            "required": [
              "single_event_present",
              "event_connected_to_lift",
              "body_and_clothing_plausible"
            ],
            "maxProperties": 3
          }
        }
      }
    },
    {
      "if": {
        "properties": {
          "scene_id": {
            "const": "FB-074"
          }
        },
        "required": [
          "scene_id"
        ]
      },
      "then": {
        "properties": {
          "judge_observations": {
            "required": [
              "chair_creak_present",
              "separate_fart_event_present",
              "creak_and_fart_distinguishable"
            ],
            "maxProperties": 3
          },
          "judge_confidence": {
            "required": [
              "chair_creak_present",
              "separate_fart_event_present",
              "creak_and_fart_distinguishable"
            ],
            "maxProperties": 3
          }
        }
      }
    },
    {
      "if": {
        "properties": {
          "scene_id": {
            "const": "FB-512"
          }
        },
        "required": [
          "scene_id"
        ]
      },
      "then": {
        "properties": {
          "judge_observations": {
            "required": [
              "single_event_present",
              "both_people_reacted",
              "recovered_without_further_emission"
            ],
            "maxProperties": 3
          },
          "judge_confidence": {
            "required": [
              "single_event_present",
              "both_people_reacted",
              "recovered_without_further_emission"
            ],
            "maxProperties": 3
          }
        }
      }
    }
  ],
  "$defs": {
    "na_reason": {
      "type": "string",
      "enum": [
        "primary_event_absent",
        "required_sound_absent",
        "causal_prerequisite_absent",
        "not_observable"
      ]
    },
    "gate_answer": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "value"
      ],
      "properties": {
        "value": {
          "enum": [
            "yes",
            "no",
            "unclear",
            null
          ]
        },
        "not_applicable_reason": {
          "$ref": "#/$defs/na_reason"
        }
      },
      "oneOf": [
        {
          "properties": {
            "value": {
              "enum": [
                "yes",
                "no",
                "unclear"
              ]
            }
          },
          "not": {
            "required": [
              "not_applicable_reason"
            ]
          }
        },
        {
          "properties": {
            "value": {
              "const": null
            }
          },
          "required": [
            "not_applicable_reason"
          ]
        }
      ]
    },
    "rating_answer": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "value"
      ],
      "properties": {
        "value": {
          "type": [
            "integer",
            "null"
          ],
          "minimum": 1,
          "maximum": 5
        },
        "not_applicable_reason": {
          "$ref": "#/$defs/na_reason"
        }
      },
      "oneOf": [
        {
          "properties": {
            "value": {
              "type": "integer",
              "minimum": 1,
              "maximum": 5
            }
          },
          "not": {
            "required": [
              "not_applicable_reason"
            ]
          }
        },
        {
          "properties": {
            "value": {
              "type": "null"
            }
          },
          "required": [
            "not_applicable_reason"
          ]
        }
      ]
    },
    "answer_never_na": {
      "description": "A primary or independent gate. It is always answerable, so it never takes N/A.",
      "properties": {
        "value": {
          "enum": [
            "yes",
            "no",
            "unclear"
          ]
        }
      }
    }
  }
}