{ "schema_version": "fartbench-core-results-v2", "data_status": "revealed-preliminary", "identity_status": "revealed", "benchmark": "FARTBench Core", "round": "R01", "generated_utc": "2026-09-18T20:45:00Z", "provenance": { "cohort_tag": "fartbench-core-r00-generation-cohort-v1-locked", "human_cohort_tag": "fartbench-core-r01-human-cohort-v1-ready", "human_aggregate_sha256": "9e56f6c50254cc3777be132dd6eebb250d615787ffb6d86014d6fb48dada46bb", "human_aggregate_tag": "fartbench-core-r01-human-aggregate-v1-locked", "comparison_sha256": "674618d3712cf6610b3235a9d3b5a494a1b4a3b03ed0665ceba5bf34008fa639", "comparison_tag": "fartbench-core-r01-human-ai-comparison-v1-locked", "ai_evidence_tag": "fartbench-core-r01-ai-evaluation-v4-locked", "mapping_commitment": "963e5c43f9af156677d311f0525ab7d0956fd157342cc7984885977ff1cf36b5", "media_derivation_commitment": "bbc18dfcd3babceaa55a6d3fb88237a34dd8b2fc8a371d962b480cdcd8eccc0a", "media_set_commitment": "8e4b60e44e72675505bedeb3d41b01425202d7da9fe2a4bce8bbf4fff813c3c0", "results_format_tag": "fartbench-core-r01-results-format-v2-ready", "reveal_tag": "fartbench-core-r01-results-v2-revealed" }, "instruments": { "human": "fartbench-core-r01-human-v3", "ai": "fartbench-core-r01-ai-v4" }, "definitions": { "response state": "yes, no, unclear, or n/a:; N/A is never No or Unclear, and N/A reasons are never collapsed", "human consensus": "at least 3 of the 4 reviewers give the exact same state; otherwise there is no consensus", "event success": "the scene's primary event question has consensus Yes", "primary causal success": "the scene's primary causal question has consensus Yes", "strict scene success": "primary event consensus is Yes and primary causal consensus is Yes", "full factual success": "every question reaches consensus, each is Yes or N/A, and both primary questions are Yes", "outcome values": "success; not_success (a consensus exists and it is not Yes); no_consensus", "ratings": "raw 1-5 values; mean, median and population SD of the numeric ones; N/A counted, never zero", "reliability": "Krippendorff's alpha: nominal over exact states for factual questions, interval for ratings; 95% interval by resampling clips; raw vote counts always beside it", "AI factual calibration": "exact-state agreement with the human consensus, scored only where a consensus exists; >= 70% for ordinary questions, >= 75% for a scene's primary causal question (decision 0017)", "AI rating criterion": "MAE <= 1.0 against the human mean where at least 3 of 4 humans rated (decision 0017), reported separately from rating discrimination", "no composite": "no single score is computed; every quantity is reported on its own" }, "interpretation": { "source": "docs/decisions/0018-core-r01-results-format-v2.md", "ai_rating_rule": "mae_threshold_met is exactly the pre-registered MAE <= 1.0 criterion of decision 0017, as computed. discrimination_demonstrated is this owner-recorded interpretation, carried as descriptive evidence; it is not a threshold and changes no pre-registered result.", "factual_primary": "Factual agreement with the human consensus is the primary AI result.", "descriptive_notes": [ "R00 and Core R01 both show source and cause attribution as the important weakness of the AI judges; this may be discussed descriptively after the reveal." ] }, "scenes": [ { "scene_id": "FB-C02", "title": "Multi-Agent Source Attribution", "primary_event_gate": "single_event_present", "primary_causal_gate": "source_was_red_jacket_woman", "questions": [ { "question_id": "FB-C02/single_event_present", "field": "single_event_present", "question": "Was exactly one fart-like event clearly present?", "role": "primary_event" }, { "question_id": "FB-C02/source_was_red_jacket_woman", "field": "source_was_red_jacket_woman", "question": "Did it clearly originate from the woman in the red jacket?", "role": "primary_causal" }, { "question_id": "FB-C02/bystanders_directed_attention", "field": "bystanders_directed_attention", "question": "Did the other passengers subsequently direct their attention or reaction toward her?", "role": "other" } ] }, { "scene_id": "FB-014", "title": "Fabric Stress Test", "primary_event_gate": "single_event_present", "primary_causal_gate": "event_connected_to_lift", "questions": [ { "question_id": "FB-014/single_event_present", "field": "single_event_present", "question": "Was exactly one fart-like event clearly present?", "role": "primary_event" }, { "question_id": "FB-014/event_connected_to_lift", "field": "event_connected_to_lift", "question": "Did the event occur during, or plausibly connect to, the box-lifting action?", "role": "primary_causal" }, { "question_id": "FB-014/body_and_clothing_plausible", "field": "body_and_clothing_plausible", "question": "Did the body and fitted clothing remain physically plausible through the event?", "role": "other" } ] }, { "scene_id": "FB-074", "title": "Chair Confound", "primary_event_gate": "separate_fart_event_present", "primary_causal_gate": "creak_and_fart_distinguishable", "questions": [ { "question_id": "FB-074/chair_creak_present", "field": "chair_creak_present", "question": "Was a distinct chair-creak event present?", "role": "other" }, { "question_id": "FB-074/separate_fart_event_present", "field": "separate_fart_event_present", "question": "Was a separate fart-like event also present later?", "role": "primary_event" }, { "question_id": "FB-074/creak_and_fart_distinguishable", "field": "creak_and_fart_distinguishable", "question": "Were the chair creak and the fart perceptually distinguishable as two different causal sounds?", "role": "primary_causal" } ] }, { "scene_id": "FB-512", "title": "Recovery Test", "primary_event_gate": "single_event_present", "primary_causal_gate": "recovered_without_further_emission", "questions": [ { "question_id": "FB-512/single_event_present", "field": "single_event_present", "question": "Was exactly one fart-like event clearly present?", "role": "primary_event" }, { "question_id": "FB-512/both_people_reacted", "field": "both_people_reacted", "question": "Did the two people briefly react to it?", "role": "other" }, { "question_id": "FB-512/recovered_without_further_emission", "field": "recovered_without_further_emission", "question": "Did they naturally recover and return toward their prior interaction, without another emission?", "role": "primary_causal" } ] } ], "reviewers": { "human": [ "H01", "H02", "H03", "H04" ], "ai_judges": [ "gemini-3.8-flash", "gemini-3.7-flash", "gemini-3.6-flash" ], "consensus_rule": "at least 3 of 4 reviewers give the exact same state", "panel_rule": "a state given by at least 2 of 3 judges; never treated as human ground truth" }, "coverage": { "cells_total": 64, "samples_evaluated": 63, "refusals": 1, "refusals_identified": true }, "samples": [ { "sample_id": "3EYPKH", "scene_id": "FB-512", "configuration_id": "flux-3-draft", "factual": { "single_event_present": { "question_id": "FB-512/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "both_people_reacted": { "question_id": "FB-512/both_people_reacted", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "n/a:not_observable", "H04": "yes" }, "counts": { "yes": 3, "no": 0, "unclear": 0, "na": 1 }, "na_reason_counts": { "not_observable": 1 }, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "recovered_without_further_emission": { "question_id": "FB-512/recovered_without_further_emission", "role": "primary_causal", "raw": { "H01": "unclear", "H02": "unclear", "H03": "unclear", "H04": "yes" }, "counts": { "yes": 1, "no": 0, "unclear": 3, "na": 0 }, "na_reason_counts": {}, "consensus": "unclear", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-512/recovered_without_further_emission": "unclear" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/recovered_without_further_emission": "unclear" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/both_people_reacted": "yes", "FB-512/recovered_without_further_emission": "unclear" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 2, "H02": 3, "H03": 1, "H04": 2 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2, "median": 2.0, "sd": 0.7071, "min": 1, "max": 3 }, "temporal_causal_coherence": { "raw": { "H01": 3, "H02": 3, "H03": 1, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.75, "median": 3.0, "sd": 1.0897, "min": 1, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": 3, "H02": 3, "H03": 3, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.25, "median": 3.0, "sd": 0.433, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": 1 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": 1 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "both_people_reacted": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "vote_counts": { "n/a:primary_event_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "vote_counts": { "n/a:primary_event_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true } } } }, { "sample_id": "3G6SS5", "scene_id": "FB-C02", "configuration_id": "ltx-2.5-fast", "factual": { "single_event_present": { "question_id": "FB-C02/single_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "yes" }, "counts": { "yes": 1, "no": 3, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "source_was_red_jacket_woman": { "question_id": "FB-C02/source_was_red_jacket_woman", "role": "primary_causal", "raw": { "H01": "n/a:primary_event_absent", "H02": "n/a:primary_event_absent", "H03": "n/a:primary_event_absent", "H04": "yes" }, "counts": { "yes": 1, "no": 0, "unclear": 0, "na": 3 }, "na_reason_counts": { "primary_event_absent": 3 }, "consensus": "n/a:primary_event_absent", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "bystanders_directed_attention": { "question_id": "FB-C02/bystanders_directed_attention", "role": "other", "raw": { "H01": "n/a:causal_prerequisite_absent", "H02": "n/a:causal_prerequisite_absent", "H03": "yes", "H04": "yes" }, "counts": { "yes": 2, "no": 0, "unclear": 0, "na": 2 }, "na_reason_counts": { "causal_prerequisite_absent": 2 }, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.3333 } }, "outcomes": { "event_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "no" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/source_was_red_jacket_woman": "n/a:primary_event_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "no", "FB-C02/source_was_red_jacket_woman": "n/a:primary_event_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "no", "FB-C02/source_was_red_jacket_woman": "n/a:primary_event_absent", "FB-C02/bystanders_directed_attention": null } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": null, "H02": null, "H03": null, "H04": 1 }, "numeric_reviewers": 1, "na_count": 3, "na_reason_counts": { "primary_event_absent": 3 }, "mean": 1, "median": 1, "sd": null, "min": 1, "max": 1 }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": null, "H03": null, "H04": 3 }, "numeric_reviewers": 1, "na_count": 3, "na_reason_counts": { "causal_prerequisite_absent": 3 }, "mean": 3, "median": 3, "sd": null, "min": 3, "max": 3 }, "physical_social_plausibility": { "raw": { "H01": null, "H02": null, "H03": 4, "H04": 2 }, "numeric_reviewers": 2, "na_count": 2, "na_reason_counts": { "not_observable": 2 }, "mean": 3, "median": 3.0, "sd": 1.0, "min": 2, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "source_was_red_jacket_woman": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false } } } }, { "sample_id": "3MNRCL", "scene_id": "FB-074", "configuration_id": "sora-2-pro", "factual": { "chair_creak_present": { "question_id": "FB-074/chair_creak_present", "role": "other", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 4, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "separate_fart_event_present": { "question_id": "FB-074/separate_fart_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "yes", "H03": "yes", "H04": "no" }, "counts": { "yes": 2, "no": 2, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.3333 }, "creak_and_fart_distinguishable": { "question_id": "FB-074/creak_and_fart_distinguishable", "role": "primary_causal", "raw": { "H01": "no", "H02": "n/a:required_sound_absent", "H03": "n/a:required_sound_absent", "H04": "n/a:required_sound_absent" }, "counts": { "yes": 0, "no": 1, "unclear": 0, "na": 3 }, "na_reason_counts": { "required_sound_absent": 3 }, "consensus": "n/a:required_sound_absent", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 } }, "outcomes": { "event_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": null } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": null, "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/chair_creak_present": "no", "FB-074/separate_fart_event_present": null, "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": null, "H02": 2, "H03": 2, "H04": null }, "numeric_reviewers": 2, "na_count": 2, "na_reason_counts": { "primary_event_absent": 2 }, "mean": 2, "median": 2.0, "sd": 0.0, "min": 2, "max": 2 }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": 2, "H03": 2, "H04": null }, "numeric_reviewers": 2, "na_count": 2, "na_reason_counts": { "causal_prerequisite_absent": 2 }, "mean": 2, "median": 2.0, "sd": 0.0, "min": 2, "max": 2 }, "physical_social_plausibility": { "raw": { "H01": null, "H02": 2, "H03": 1, "H04": null }, "numeric_reviewers": 2, "na_count": 2, "na_reason_counts": { "not_observable": 2 }, "mean": 1.5, "median": 1.5, "sd": 0.5, "min": 1, "max": 2 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "chair_creak_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "separate_fart_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "creak_and_fart_distinguishable": { "state": "n/a:required_sound_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.7-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.6-flash": { "factual": { "chair_creak_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "creak_and_fart_distinguishable": { "state": "n/a:required_sound_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } } }, "panel": { "chair_creak_present": { "vote_counts": { "no": 2, "yes": 1 }, "consensus": "no", "agrees_with_human_consensus": true, "calibration_eligible": true }, "separate_fart_event_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false }, "creak_and_fart_distinguishable": { "vote_counts": { "n/a:required_sound_absent": 2, "yes": 1 }, "consensus": "n/a:required_sound_absent", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "3Z7JPK", "scene_id": "FB-C02", "configuration_id": "veo-3.1-standard", "factual": { "single_event_present": { "question_id": "FB-C02/single_event_present", "role": "primary_event", "raw": { "H01": "unclear", "H02": "unclear", "H03": "yes", "H04": "no" }, "counts": { "yes": 1, "no": 1, "unclear": 2, "na": 0 }, "na_reason_counts": {}, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 }, "source_was_red_jacket_woman": { "question_id": "FB-C02/source_was_red_jacket_woman", "role": "primary_causal", "raw": { "H01": "yes", "H02": "unclear", "H03": "unclear", "H04": "yes" }, "counts": { "yes": 2, "no": 0, "unclear": 2, "na": 0 }, "na_reason_counts": {}, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.3333 }, "bystanders_directed_attention": { "question_id": "FB-C02/bystanders_directed_attention", "role": "other", "raw": { "H01": "yes", "H02": "unclear", "H03": "no", "H04": "yes" }, "counts": { "yes": 2, "no": 1, "unclear": 1, "na": 0 }, "na_reason_counts": {}, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 } }, "outcomes": { "event_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-C02/single_event_present": null } }, "primary_causal_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-C02/source_was_red_jacket_woman": null } }, "strict_scene_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-C02/single_event_present": null, "FB-C02/source_was_red_jacket_woman": null } }, "full_factual_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-C02/single_event_present": null, "FB-C02/source_was_red_jacket_woman": null, "FB-C02/bystanders_directed_attention": null } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 3, "H03": 3, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.5, "median": 3.5, "sd": 0.5, "min": 3, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 2, "H02": 1, "H03": 1, "H04": 2 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 1.5, "median": 1.5, "sd": 0.5, "min": 1, "max": 2 }, "physical_social_plausibility": { "raw": { "H01": 3, "H02": 2, "H03": 2, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.5, "median": 2.5, "sd": 0.5, "min": 2, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 2, "agrees_with_human_consensus": null, "calibration_eligible": false }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": null, "calibration_eligible": false }, "bystanders_directed_attention": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1.5, "signed_error": 1.5 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": 0.5 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": null, "calibration_eligible": false }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1.5, "signed_error": 1.5 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": 0.5 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "source_was_red_jacket_woman": { "state": "unclear", "confidence": 2, "agrees_with_human_consensus": null, "calibration_eligible": false }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1.5, "signed_error": 1.5 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": 0.5 } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 3 }, "consensus": "no", "agrees_with_human_consensus": null, "calibration_eligible": false }, "source_was_red_jacket_woman": { "vote_counts": { "unclear": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false }, "bystanders_directed_attention": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false } } } }, { "sample_id": "4EPZWT", "scene_id": "FB-512", "configuration_id": "vidu-omni-pro-3.2a", "factual": { "single_event_present": { "question_id": "FB-512/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "both_people_reacted": { "question_id": "FB-512/both_people_reacted", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "recovered_without_further_emission": { "question_id": "FB-512/recovered_without_further_emission", "role": "primary_causal", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-512/recovered_without_further_emission": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/recovered_without_further_emission": "yes" } }, "full_factual_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/both_people_reacted": "yes", "FB-512/recovered_without_further_emission": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 4, "H03": 4, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4, "median": 4.0, "sd": 0.0, "min": 4, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 4, "H02": 3, "H03": 5, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4, "median": 4.0, "sd": 0.7071, "min": 3, "max": 5 }, "physical_social_plausibility": { "raw": { "H01": 4, "H02": 4, "H03": 4, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4, "median": 4.0, "sd": 0.0, "min": 4, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 4, "rating_eligible": true, "abs_error": 0, "signed_error": 0 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "5JSTTP", "scene_id": "FB-C02", "configuration_id": "sora-2-pro", "factual": { "single_event_present": { "question_id": "FB-C02/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "source_was_red_jacket_woman": { "question_id": "FB-C02/source_was_red_jacket_woman", "role": "primary_causal", "raw": { "H01": "unclear", "H02": "unclear", "H03": "unclear", "H04": "unclear" }, "counts": { "yes": 0, "no": 0, "unclear": 4, "na": 0 }, "na_reason_counts": {}, "consensus": "unclear", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "bystanders_directed_attention": { "question_id": "FB-C02/bystanders_directed_attention", "role": "other", "raw": { "H01": "no", "H02": "yes", "H03": "no", "H04": "no" }, "counts": { "yes": 1, "no": 3, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/source_was_red_jacket_woman": "unclear" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes", "FB-C02/source_was_red_jacket_woman": "unclear" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes", "FB-C02/source_was_red_jacket_woman": "unclear", "FB-C02/bystanders_directed_attention": "no" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 3, "H02": 3, "H03": 2, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.75, "median": 3.0, "sd": 0.433, "min": 2, "max": 3 }, "temporal_causal_coherence": { "raw": { "H01": 4, "H02": 3, "H03": 4, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.5, "median": 3.5, "sd": 0.5, "min": 3, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": 3, "H02": 3, "H03": 3, "H04": 2 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.75, "median": 3.0, "sd": 0.433, "min": 2, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "unclear", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "unclear", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "unclear", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "unclear", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "vote_counts": { "unclear": 3 }, "consensus": "unclear", "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "vote_counts": { "unclear": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true } } } }, { "sample_id": "5Q6WE5", "scene_id": "FB-512", "configuration_id": "kling-video-3.0-omni", "factual": { "single_event_present": { "question_id": "FB-512/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "both_people_reacted": { "question_id": "FB-512/both_people_reacted", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "recovered_without_further_emission": { "question_id": "FB-512/recovered_without_further_emission", "role": "primary_causal", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-512/recovered_without_further_emission": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/recovered_without_further_emission": "yes" } }, "full_factual_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/both_people_reacted": "yes", "FB-512/recovered_without_further_emission": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 3, "H03": 3, "H04": 2 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3, "median": 3.0, "sd": 0.7071, "min": 2, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 5, "H02": 4, "H03": 3, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4, "median": 4.0, "sd": 0.7071, "min": 3, "max": 5 }, "physical_social_plausibility": { "raw": { "H01": 4, "H02": 3, "H03": 3, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.5, "median": 3.5, "sd": 0.5, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 4, "rating_eligible": true, "abs_error": 0.5, "signed_error": 0.5 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 4, "rating_eligible": true, "abs_error": 0.5, "signed_error": 0.5 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "6D7MWZ", "scene_id": "FB-C02", "configuration_id": "veo-3.1-fast", "factual": { "single_event_present": { "question_id": "FB-C02/single_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "yes", "H03": "no", "H04": "no" }, "counts": { "yes": 1, "no": 3, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "source_was_red_jacket_woman": { "question_id": "FB-C02/source_was_red_jacket_woman", "role": "primary_causal", "raw": { "H01": "n/a:primary_event_absent", "H02": "yes", "H03": "unclear", "H04": "yes" }, "counts": { "yes": 2, "no": 0, "unclear": 1, "na": 1 }, "na_reason_counts": { "primary_event_absent": 1 }, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 }, "bystanders_directed_attention": { "question_id": "FB-C02/bystanders_directed_attention", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "unclear", "H04": "yes" }, "counts": { "yes": 3, "no": 0, "unclear": 1, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 } }, "outcomes": { "event_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "no" } }, "primary_causal_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-C02/source_was_red_jacket_woman": null } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "no", "FB-C02/source_was_red_jacket_woman": null } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "no", "FB-C02/source_was_red_jacket_woman": null, "FB-C02/bystanders_directed_attention": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 3, "H02": 3, "H03": 2, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3, "median": 3.0, "sd": 0.7071, "min": 2, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 2, "H02": 1, "H03": 1, "H04": 1 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 1.25, "median": 1.0, "sd": 0.433, "min": 1, "max": 2 }, "physical_social_plausibility": { "raw": { "H01": 1, "H02": 2, "H03": 1, "H04": 1 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 1.25, "median": 1.0, "sd": 0.433, "min": 1, "max": 2 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "unclear", "confidence": 2, "agrees_with_human_consensus": null, "calibration_eligible": false }, "bystanders_directed_attention": { "state": "no", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "temporal_causal_coherence": { "value": 2, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 }, "physical_social_plausibility": { "value": 2, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "unclear", "confidence": 2, "agrees_with_human_consensus": null, "calibration_eligible": false }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "temporal_causal_coherence": { "value": 2, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 }, "physical_social_plausibility": { "value": 2, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "unclear", "confidence": 2, "agrees_with_human_consensus": null, "calibration_eligible": false }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1.75, "signed_error": 1.75 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.75, "signed_error": 1.75 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "source_was_red_jacket_woman": { "vote_counts": { "unclear": 3 }, "consensus": "unclear", "agrees_with_human_consensus": null, "calibration_eligible": false }, "bystanders_directed_attention": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "6RR2XF", "scene_id": "FB-512", "configuration_id": "happyhorse-1.1", "factual": { "single_event_present": { "question_id": "FB-512/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "both_people_reacted": { "question_id": "FB-512/both_people_reacted", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "recovered_without_further_emission": { "question_id": "FB-512/recovered_without_further_emission", "role": "primary_causal", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-512/recovered_without_further_emission": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/recovered_without_further_emission": "yes" } }, "full_factual_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/both_people_reacted": "yes", "FB-512/recovered_without_further_emission": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 1, "H02": 2, "H03": 1, "H04": 2 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 1.5, "median": 1.5, "sd": 0.5, "min": 1, "max": 2 }, "temporal_causal_coherence": { "raw": { "H01": 4, "H02": 2, "H03": 3, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.25, "median": 3.5, "sd": 0.8292, "min": 2, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": 4, "H02": 3, "H03": 3, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.5, "median": 3.5, "sd": 0.5, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.5, "signed_error": 1.5 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.5, "signed_error": 1.5 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 }, "physical_social_plausibility": { "value": 4, "rating_eligible": true, "abs_error": 0.5, "signed_error": 0.5 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "both_people_reacted": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "vote_counts": { "n/a:primary_event_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "vote_counts": { "n/a:primary_event_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "7AKCCT", "scene_id": "FB-014", "configuration_id": "vidu-omni-pro-3.2a", "factual": { "single_event_present": { "question_id": "FB-014/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "event_connected_to_lift": { "question_id": "FB-014/event_connected_to_lift", "role": "primary_causal", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "body_and_clothing_plausible": { "question_id": "FB-014/body_and_clothing_plausible", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-014/event_connected_to_lift": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes", "FB-014/event_connected_to_lift": "yes" } }, "full_factual_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes", "FB-014/event_connected_to_lift": "yes", "FB-014/body_and_clothing_plausible": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 4, "H03": 4, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4, "median": 4.0, "sd": 0.0, "min": 4, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 4, "H02": 4, "H03": 5, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4.25, "median": 4.0, "sd": 0.433, "min": 4, "max": 5 }, "physical_social_plausibility": { "raw": { "H01": 4, "H02": 4, "H03": 4, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4, "median": 4.0, "sd": 0.0, "min": 4, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1.25, "signed_error": -1.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1.25, "signed_error": -1.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "7FY9MG", "scene_id": "FB-074", "configuration_id": "flux-3-full", "factual": { "chair_creak_present": { "question_id": "FB-074/chair_creak_present", "role": "other", "raw": { "H01": "yes", "H02": "no", "H03": "no", "H04": "yes" }, "counts": { "yes": 2, "no": 2, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.3333 }, "separate_fart_event_present": { "question_id": "FB-074/separate_fart_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "creak_and_fart_distinguishable": { "question_id": "FB-074/creak_and_fart_distinguishable", "role": "primary_causal", "raw": { "H01": "yes", "H02": "n/a:required_sound_absent", "H03": "n/a:required_sound_absent", "H04": "yes" }, "counts": { "yes": 2, "no": 0, "unclear": 0, "na": 2 }, "na_reason_counts": { "required_sound_absent": 2 }, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.3333 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "yes" } }, "primary_causal_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/creak_and_fart_distinguishable": null } }, "strict_scene_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "yes", "FB-074/creak_and_fart_distinguishable": null } }, "full_factual_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/chair_creak_present": null, "FB-074/separate_fart_event_present": "yes", "FB-074/creak_and_fart_distinguishable": null } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 3, "H02": 3, "H03": 2, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3, "median": 3.0, "sd": 0.7071, "min": 2, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 3, "H02": 3, "H03": 3, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.25, "median": 3.0, "sd": 0.433, "min": 3, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": 3, "H02": 3, "H03": 2, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3, "median": 3.0, "sd": 0.7071, "min": 2, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 } } }, "gemini-3.7-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 } } }, "gemini-3.6-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 } } } }, "panel": { "chair_creak_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false } } } }, { "sample_id": "7K724T", "scene_id": "FB-074", "configuration_id": "sora-2", "factual": { "chair_creak_present": { "question_id": "FB-074/chair_creak_present", "role": "other", "raw": { "H01": "no", "H02": "unclear", "H03": "unclear", "H04": "no" }, "counts": { "yes": 0, "no": 2, "unclear": 2, "na": 0 }, "na_reason_counts": {}, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.3333 }, "separate_fart_event_present": { "question_id": "FB-074/separate_fart_event_present", "role": "primary_event", "raw": { "H01": "unclear", "H02": "yes", "H03": "unclear", "H04": "yes" }, "counts": { "yes": 2, "no": 0, "unclear": 2, "na": 0 }, "na_reason_counts": {}, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.3333 }, "creak_and_fart_distinguishable": { "question_id": "FB-074/creak_and_fart_distinguishable", "role": "primary_causal", "raw": { "H01": "n/a:required_sound_absent", "H02": "unclear", "H03": "no", "H04": "n/a:required_sound_absent" }, "counts": { "yes": 0, "no": 1, "unclear": 1, "na": 2 }, "na_reason_counts": { "required_sound_absent": 2 }, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 } }, "outcomes": { "event_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": null } }, "primary_causal_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/creak_and_fart_distinguishable": null } }, "strict_scene_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": null, "FB-074/creak_and_fart_distinguishable": null } }, "full_factual_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/chair_creak_present": null, "FB-074/separate_fart_event_present": null, "FB-074/creak_and_fart_distinguishable": null } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 1, "H02": 2, "H03": 1, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 1.75, "median": 1.5, "sd": 0.8292, "min": 1, "max": 3 }, "temporal_causal_coherence": { "raw": { "H01": 2, "H02": 3, "H03": 2, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.5, "median": 2.5, "sd": 0.5, "min": 2, "max": 3 }, "physical_social_plausibility": { "raw": { "H01": null, "H02": null, "H03": 2, "H04": 3 }, "numeric_reviewers": 2, "na_count": 2, "na_reason_counts": { "not_observable": 2 }, "mean": 2.5, "median": 2.5, "sd": 0.5, "min": 2, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "creak_and_fart_distinguishable": { "state": "n/a:required_sound_absent", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.7-flash": { "factual": { "chair_creak_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "creak_and_fart_distinguishable": { "state": "n/a:required_sound_absent", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.25, "signed_error": 1.25 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": 0.5 }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.6-flash": { "factual": { "chair_creak_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "creak_and_fart_distinguishable": { "state": "n/a:required_sound_absent", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.25, "signed_error": 1.25 }, "temporal_causal_coherence": { "value": 2, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } } }, "panel": { "chair_creak_present": { "vote_counts": { "no": 2, "yes": 1 }, "consensus": "no", "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false }, "creak_and_fart_distinguishable": { "vote_counts": { "n/a:required_sound_absent": 3 }, "consensus": "n/a:required_sound_absent", "agrees_with_human_consensus": null, "calibration_eligible": false } } } }, { "sample_id": "7XGJH6", "scene_id": "FB-074", "configuration_id": "wan-3", "factual": { "chair_creak_present": { "question_id": "FB-074/chair_creak_present", "role": "other", "raw": { "H01": "unclear", "H02": "no", "H03": "no", "H04": "yes" }, "counts": { "yes": 1, "no": 2, "unclear": 1, "na": 0 }, "na_reason_counts": {}, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 }, "separate_fart_event_present": { "question_id": "FB-074/separate_fart_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "creak_and_fart_distinguishable": { "question_id": "FB-074/creak_and_fart_distinguishable", "role": "primary_causal", "raw": { "H01": "yes", "H02": "n/a:required_sound_absent", "H03": "n/a:required_sound_absent", "H04": "yes" }, "counts": { "yes": 2, "no": 0, "unclear": 0, "na": 2 }, "na_reason_counts": { "required_sound_absent": 2 }, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.3333 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "yes" } }, "primary_causal_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/creak_and_fart_distinguishable": null } }, "strict_scene_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "yes", "FB-074/creak_and_fart_distinguishable": null } }, "full_factual_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/chair_creak_present": null, "FB-074/separate_fart_event_present": "yes", "FB-074/creak_and_fart_distinguishable": null } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 2, "H03": 2, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.75, "median": 2.5, "sd": 0.8292, "min": 2, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 3, "H02": 3, "H03": 3, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.25, "median": 3.0, "sd": 0.433, "min": 3, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": 4, "H02": 3, "H03": 4, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.75, "median": 4.0, "sd": 0.433, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 } } }, "gemini-3.7-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 } } }, "gemini-3.6-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 } } } }, "panel": { "chair_creak_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false } } } }, { "sample_id": "845GNZ", "scene_id": "FB-014", "configuration_id": "veo-3.1-standard", "factual": { "single_event_present": { "question_id": "FB-014/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "no" }, "counts": { "yes": 3, "no": 1, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "event_connected_to_lift": { "question_id": "FB-014/event_connected_to_lift", "role": "primary_causal", "raw": { "H01": "yes", "H02": "yes", "H03": "no", "H04": "yes" }, "counts": { "yes": 3, "no": 1, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "body_and_clothing_plausible": { "question_id": "FB-014/body_and_clothing_plausible", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-014/event_connected_to_lift": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes", "FB-014/event_connected_to_lift": "yes" } }, "full_factual_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes", "FB-014/event_connected_to_lift": "yes", "FB-014/body_and_clothing_plausible": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 4, "H03": 2, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.5, "median": 4.0, "sd": 0.866, "min": 2, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 4, "H02": 4, "H03": 2, "H04": 2 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3, "median": 3.0, "sd": 1.0, "min": 2, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": 3, "H02": 3, "H03": 3, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3, "median": 3.0, "sd": 0.0, "min": 3, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "8SJDU5", "scene_id": "FB-014", "configuration_id": "minimax-h3-768p-local", "factual": { "single_event_present": { "question_id": "FB-014/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "no", "H04": "yes" }, "counts": { "yes": 3, "no": 1, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "event_connected_to_lift": { "question_id": "FB-014/event_connected_to_lift", "role": "primary_causal", "raw": { "H01": "no", "H02": "unclear", "H03": "n/a:primary_event_absent", "H04": "no" }, "counts": { "yes": 0, "no": 2, "unclear": 1, "na": 1 }, "na_reason_counts": { "primary_event_absent": 1 }, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 }, "body_and_clothing_plausible": { "question_id": "FB-014/body_and_clothing_plausible", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes" } }, "primary_causal_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-014/event_connected_to_lift": null } }, "strict_scene_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-014/single_event_present": "yes", "FB-014/event_connected_to_lift": null } }, "full_factual_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-014/single_event_present": "yes", "FB-014/event_connected_to_lift": null, "FB-014/body_and_clothing_plausible": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 2, "H02": 1, "H03": null, "H04": 3 }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "primary_event_absent": 1 }, "mean": 2, "median": 2, "sd": 0.8165, "min": 1, "max": 3 }, "temporal_causal_coherence": { "raw": { "H01": 2, "H02": 2, "H03": null, "H04": 1 }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "causal_prerequisite_absent": 1 }, "mean": 1.6667, "median": 2, "sd": 0.4714, "min": 1, "max": 2 }, "physical_social_plausibility": { "raw": { "H01": 2, "H02": 2, "H03": 3, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.5, "median": 2.5, "sd": 0.5, "min": 2, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": null, "calibration_eligible": false }, "body_and_clothing_plausible": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": 1 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1.3333, "signed_error": 1.3333 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": 0.5 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": null, "calibration_eligible": false }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": 1 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1.3333, "signed_error": 1.3333 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": 0.5 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": 1 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1.3333, "signed_error": 1.3333 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": 0.5 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false }, "body_and_clothing_plausible": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "9X4F73", "scene_id": "FB-074", "configuration_id": "veo-3.1-standard", "factual": { "chair_creak_present": { "question_id": "FB-074/chair_creak_present", "role": "other", "raw": { "H01": "no", "H02": "yes", "H03": "unclear", "H04": "no" }, "counts": { "yes": 1, "no": 2, "unclear": 1, "na": 0 }, "na_reason_counts": {}, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 }, "separate_fart_event_present": { "question_id": "FB-074/separate_fart_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "no", "H03": "yes", "H04": "yes" }, "counts": { "yes": 3, "no": 1, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "creak_and_fart_distinguishable": { "question_id": "FB-074/creak_and_fart_distinguishable", "role": "primary_causal", "raw": { "H01": "n/a:required_sound_absent", "H02": "unclear", "H03": "unclear", "H04": "no" }, "counts": { "yes": 0, "no": 1, "unclear": 2, "na": 1 }, "na_reason_counts": { "required_sound_absent": 1 }, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "yes" } }, "primary_causal_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/creak_and_fart_distinguishable": null } }, "strict_scene_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "yes", "FB-074/creak_and_fart_distinguishable": null } }, "full_factual_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/chair_creak_present": null, "FB-074/separate_fart_event_present": "yes", "FB-074/creak_and_fart_distinguishable": null } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 2, "H03": 2, "H04": 2 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.5, "median": 2.0, "sd": 0.866, "min": 2, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 4, "H02": 3, "H03": 2, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3, "median": 3.0, "sd": 0.7071, "min": 2, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": 4, "H02": 4, "H03": 3, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.5, "median": 3.5, "sd": 0.5, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": 0.5 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 } } }, "gemini-3.7-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": 0.5 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 } } }, "gemini-3.6-flash": { "factual": { "chair_creak_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "n/a:required_sound_absent", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": 0.5 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 } } } }, "panel": { "chair_creak_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "vote_counts": { "n/a:required_sound_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false } } } }, { "sample_id": "BABQP3", "scene_id": "FB-C02", "configuration_id": "happyhorse-1.1", "factual": { "single_event_present": { "question_id": "FB-C02/single_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 4, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "source_was_red_jacket_woman": { "question_id": "FB-C02/source_was_red_jacket_woman", "role": "primary_causal", "raw": { "H01": "n/a:primary_event_absent", "H02": "n/a:primary_event_absent", "H03": "yes", "H04": "n/a:primary_event_absent" }, "counts": { "yes": 1, "no": 0, "unclear": 0, "na": 3 }, "na_reason_counts": { "primary_event_absent": 3 }, "consensus": "n/a:primary_event_absent", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "bystanders_directed_attention": { "question_id": "FB-C02/bystanders_directed_attention", "role": "other", "raw": { "H01": "n/a:causal_prerequisite_absent", "H02": "n/a:not_observable", "H03": "yes", "H04": "n/a:causal_prerequisite_absent" }, "counts": { "yes": 1, "no": 0, "unclear": 0, "na": 3 }, "na_reason_counts": { "causal_prerequisite_absent": 2, "not_observable": 1 }, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 } }, "outcomes": { "event_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "no" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/source_was_red_jacket_woman": "n/a:primary_event_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "no", "FB-C02/source_was_red_jacket_woman": "n/a:primary_event_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "no", "FB-C02/source_was_red_jacket_woman": "n/a:primary_event_absent", "FB-C02/bystanders_directed_attention": null } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "primary_event_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": null, "H03": 4, "H04": null }, "numeric_reviewers": 1, "na_count": 3, "na_reason_counts": { "causal_prerequisite_absent": 3 }, "mean": 4, "median": 4, "sd": null, "min": 4, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": null, "H02": null, "H03": 4, "H04": null }, "numeric_reviewers": 1, "na_count": 3, "na_reason_counts": { "not_observable": 3 }, "mean": 4, "median": 4, "sd": null, "min": 4, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "source_was_red_jacket_woman": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false } } } }, { "sample_id": "CS9BVJ", "scene_id": "FB-512", "configuration_id": "veo-3.1-standard", "factual": { "single_event_present": { "question_id": "FB-512/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "no" }, "counts": { "yes": 3, "no": 1, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "both_people_reacted": { "question_id": "FB-512/both_people_reacted", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "recovered_without_further_emission": { "question_id": "FB-512/recovered_without_further_emission", "role": "primary_causal", "raw": { "H01": "no", "H02": "unclear", "H03": "no", "H04": "yes" }, "counts": { "yes": 1, "no": 2, "unclear": 1, "na": 0 }, "na_reason_counts": {}, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes" } }, "primary_causal_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-512/recovered_without_further_emission": null } }, "strict_scene_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/recovered_without_further_emission": null } }, "full_factual_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/both_people_reacted": "yes", "FB-512/recovered_without_further_emission": null } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 3, "H03": 2, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.25, "median": 3.5, "sd": 0.8292, "min": 2, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 4, "H02": 3, "H03": 2, "H04": 2 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.75, "median": 2.5, "sd": 0.8292, "min": 2, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": 4, "H02": 3, "H03": 3, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.25, "median": 3.0, "sd": 0.433, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "both_people_reacted": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 3 }, "consensus": "no", "agrees_with_human_consensus": false, "calibration_eligible": true }, "both_people_reacted": { "vote_counts": { "n/a:primary_event_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "vote_counts": { "n/a:primary_event_absent": 1, "no": 2 }, "consensus": "no", "agrees_with_human_consensus": null, "calibration_eligible": false } } } }, { "sample_id": "D2UWK6", "scene_id": "FB-512", "configuration_id": "ltx-2.5-pro", "factual": { "single_event_present": { "question_id": "FB-512/single_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 4, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "both_people_reacted": { "question_id": "FB-512/both_people_reacted", "role": "other", "raw": { "H01": "n/a:primary_event_absent", "H02": "n/a:primary_event_absent", "H03": "n/a:primary_event_absent", "H04": "n/a:primary_event_absent" }, "counts": { "yes": 0, "no": 0, "unclear": 0, "na": 4 }, "na_reason_counts": { "primary_event_absent": 4 }, "consensus": "n/a:primary_event_absent", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "recovered_without_further_emission": { "question_id": "FB-512/recovered_without_further_emission", "role": "primary_causal", "raw": { "H01": "n/a:primary_event_absent", "H02": "n/a:primary_event_absent", "H03": "n/a:primary_event_absent", "H04": "n/a:primary_event_absent" }, "counts": { "yes": 0, "no": 0, "unclear": 0, "na": 4 }, "na_reason_counts": { "primary_event_absent": 4 }, "consensus": "n/a:primary_event_absent", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "not_success", "applicable": true, "inputs": { "FB-512/single_event_present": "no" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-512/recovered_without_further_emission": "n/a:primary_event_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-512/single_event_present": "no", "FB-512/recovered_without_further_emission": "n/a:primary_event_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-512/single_event_present": "no", "FB-512/both_people_reacted": "n/a:primary_event_absent", "FB-512/recovered_without_further_emission": "n/a:primary_event_absent" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "primary_event_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "causal_prerequisite_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "physical_social_plausibility": { "raw": { "H01": null, "H02": 3, "H03": 3, "H04": null }, "numeric_reviewers": 2, "na_count": 2, "na_reason_counts": { "not_observable": 2 }, "mean": 3, "median": 3.0, "sd": 0.0, "min": 3, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 3 }, "consensus": "no", "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "vote_counts": { "n/a:primary_event_absent": 3 }, "consensus": "n/a:primary_event_absent", "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "vote_counts": { "n/a:primary_event_absent": 3 }, "consensus": "n/a:primary_event_absent", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "D6JHRC", "scene_id": "FB-014", "configuration_id": "sora-2-pro", "factual": { "single_event_present": { "question_id": "FB-014/single_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 4, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "event_connected_to_lift": { "question_id": "FB-014/event_connected_to_lift", "role": "primary_causal", "raw": { "H01": "n/a:primary_event_absent", "H02": "n/a:primary_event_absent", "H03": "n/a:primary_event_absent", "H04": "n/a:primary_event_absent" }, "counts": { "yes": 0, "no": 0, "unclear": 0, "na": 4 }, "na_reason_counts": { "primary_event_absent": 4 }, "consensus": "n/a:primary_event_absent", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "body_and_clothing_plausible": { "question_id": "FB-014/body_and_clothing_plausible", "role": "other", "raw": { "H01": "n/a:not_observable", "H02": "yes", "H03": "yes", "H04": "n/a:not_observable" }, "counts": { "yes": 2, "no": 0, "unclear": 0, "na": 2 }, "na_reason_counts": { "not_observable": 2 }, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.3333 } }, "outcomes": { "event_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/single_event_present": "no" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/event_connected_to_lift": "n/a:primary_event_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/single_event_present": "no", "FB-014/event_connected_to_lift": "n/a:primary_event_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/single_event_present": "no", "FB-014/event_connected_to_lift": "n/a:primary_event_absent", "FB-014/body_and_clothing_plausible": null } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "primary_event_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "causal_prerequisite_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "physical_social_plausibility": { "raw": { "H01": null, "H02": 3, "H03": 3, "H04": null }, "numeric_reviewers": 2, "na_count": 2, "na_reason_counts": { "not_observable": 2 }, "mean": 3, "median": 3.0, "sd": 0.0, "min": 3, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 2, "yes": 1 }, "consensus": "no", "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "vote_counts": { "n/a:primary_event_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false } } } }, { "sample_id": "D8P3PH", "scene_id": "FB-014", "configuration_id": "flux-3-full", "factual": { "single_event_present": { "question_id": "FB-014/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "no" }, "counts": { "yes": 3, "no": 1, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "event_connected_to_lift": { "question_id": "FB-014/event_connected_to_lift", "role": "primary_causal", "raw": { "H01": "yes", "H02": "yes", "H03": "unclear", "H04": "n/a:primary_event_absent" }, "counts": { "yes": 2, "no": 0, "unclear": 1, "na": 1 }, "na_reason_counts": { "primary_event_absent": 1 }, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 }, "body_and_clothing_plausible": { "question_id": "FB-014/body_and_clothing_plausible", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "n/a:not_observable" }, "counts": { "yes": 3, "no": 0, "unclear": 0, "na": 1 }, "na_reason_counts": { "not_observable": 1 }, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes" } }, "primary_causal_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-014/event_connected_to_lift": null } }, "strict_scene_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-014/single_event_present": "yes", "FB-014/event_connected_to_lift": null } }, "full_factual_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-014/single_event_present": "yes", "FB-014/event_connected_to_lift": null, "FB-014/body_and_clothing_plausible": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 2, "H02": 4, "H03": 1, "H04": null }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "primary_event_absent": 1 }, "mean": 2.3333, "median": 2, "sd": 1.2472, "min": 1, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 3, "H02": 2, "H03": 2, "H04": null }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "causal_prerequisite_absent": 1 }, "mean": 2.3333, "median": 2, "sd": 0.4714, "min": 2, "max": 3 }, "physical_social_plausibility": { "raw": { "H01": 3, "H02": 4, "H03": 3, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.25, "median": 3.0, "sd": 0.433, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.6667, "signed_error": 0.6667 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.6667, "signed_error": 0.6667 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 2, "yes": 1 }, "consensus": "no", "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "vote_counts": { "n/a:primary_event_absent": 2, "yes": 1 }, "consensus": "n/a:primary_event_absent", "agrees_with_human_consensus": null, "calibration_eligible": false }, "body_and_clothing_plausible": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "DWPP3W", "scene_id": "FB-C02", "configuration_id": "flux-3-draft", "factual": { "single_event_present": { "question_id": "FB-C02/single_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 3, "no": 1, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "source_was_red_jacket_woman": { "question_id": "FB-C02/source_was_red_jacket_woman", "role": "primary_causal", "raw": { "H01": "unclear", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 3, "no": 0, "unclear": 1, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "bystanders_directed_attention": { "question_id": "FB-C02/bystanders_directed_attention", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/source_was_red_jacket_woman": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes", "FB-C02/source_was_red_jacket_woman": "yes" } }, "full_factual_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes", "FB-C02/source_was_red_jacket_woman": "yes", "FB-C02/bystanders_directed_attention": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": null, "H02": 4, "H03": 1, "H04": 2 }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "primary_event_absent": 1 }, "mean": 2.3333, "median": 2, "sd": 1.2472, "min": 1, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": 4, "H03": 4, "H04": 4 }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "causal_prerequisite_absent": 1 }, "mean": 4, "median": 4, "sd": 0.0, "min": 4, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": 3, "H02": 4, "H03": 3, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.5, "median": 3.5, "sd": 0.5, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.6667, "signed_error": 0.6667 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.6667, "signed_error": 0.6667 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "unclear", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.6667, "signed_error": 0.6667 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "vote_counts": { "unclear": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "E97W8V", "scene_id": "FB-074", "configuration_id": "flux-3-draft", "factual": { "chair_creak_present": { "question_id": "FB-074/chair_creak_present", "role": "other", "raw": { "H01": "unclear", "H02": "no", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 3, "unclear": 1, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "separate_fart_event_present": { "question_id": "FB-074/separate_fart_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "creak_and_fart_distinguishable": { "question_id": "FB-074/creak_and_fart_distinguishable", "role": "primary_causal", "raw": { "H01": "unclear", "H02": "n/a:required_sound_absent", "H03": "n/a:required_sound_absent", "H04": "n/a:required_sound_absent" }, "counts": { "yes": 0, "no": 0, "unclear": 1, "na": 3 }, "na_reason_counts": { "required_sound_absent": 3 }, "consensus": "n/a:required_sound_absent", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "yes" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "yes", "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/chair_creak_present": "no", "FB-074/separate_fart_event_present": "yes", "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 3, "H02": 3, "H03": 1, "H04": 2 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.25, "median": 2.5, "sd": 0.8292, "min": 1, "max": 3 }, "temporal_causal_coherence": { "raw": { "H01": 2, "H02": 3, "H03": 3, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3, "median": 3.0, "sd": 0.7071, "min": 2, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": 3, "H02": 3, "H03": 2, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.75, "median": 3.0, "sd": 0.433, "min": 2, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "chair_creak_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "n/a:required_sound_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 }, "temporal_causal_coherence": { "value": 2, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 } } }, "gemini-3.7-flash": { "factual": { "chair_creak_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "n/a:required_sound_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 } } }, "gemini-3.6-flash": { "factual": { "chair_creak_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "n/a:required_sound_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 } } } }, "panel": { "chair_creak_present": { "vote_counts": { "no": 3 }, "consensus": "no", "agrees_with_human_consensus": true, "calibration_eligible": true }, "separate_fart_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "vote_counts": { "n/a:required_sound_absent": 3 }, "consensus": "n/a:required_sound_absent", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "ECWRMS", "scene_id": "FB-C02", "configuration_id": "seedance-2.5", "factual": { "single_event_present": { "question_id": "FB-C02/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "source_was_red_jacket_woman": { "question_id": "FB-C02/source_was_red_jacket_woman", "role": "primary_causal", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "bystanders_directed_attention": { "question_id": "FB-C02/bystanders_directed_attention", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/source_was_red_jacket_woman": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes", "FB-C02/source_was_red_jacket_woman": "yes" } }, "full_factual_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes", "FB-C02/source_was_red_jacket_woman": "yes", "FB-C02/bystanders_directed_attention": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 5, "H02": 2, "H03": 4, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.75, "median": 4.0, "sd": 1.0897, "min": 2, "max": 5 }, "temporal_causal_coherence": { "raw": { "H01": 5, "H02": 4, "H03": 5, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4.5, "median": 4.5, "sd": 0.5, "min": 4, "max": 5 }, "physical_social_plausibility": { "raw": { "H01": 5, "H02": 4, "H03": 4, "H04": 2 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.75, "median": 4.0, "sd": 1.0897, "min": 2, "max": 5 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1.5, "signed_error": -1.5 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1.5, "signed_error": -1.5 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "EFXRPX", "scene_id": "FB-C02", "configuration_id": "sora-2", "factual": { "single_event_present": { "question_id": "FB-C02/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "unclear", "H03": "no", "H04": "yes" }, "counts": { "yes": 2, "no": 1, "unclear": 1, "na": 0 }, "na_reason_counts": {}, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 }, "source_was_red_jacket_woman": { "question_id": "FB-C02/source_was_red_jacket_woman", "role": "primary_causal", "raw": { "H01": "unclear", "H02": "unclear", "H03": "n/a:primary_event_absent", "H04": "unclear" }, "counts": { "yes": 0, "no": 0, "unclear": 3, "na": 1 }, "na_reason_counts": { "primary_event_absent": 1 }, "consensus": "unclear", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "bystanders_directed_attention": { "question_id": "FB-C02/bystanders_directed_attention", "role": "other", "raw": { "H01": "no", "H02": "unclear", "H03": "n/a:causal_prerequisite_absent", "H04": "n/a:not_observable" }, "counts": { "yes": 0, "no": 1, "unclear": 1, "na": 2 }, "na_reason_counts": { "causal_prerequisite_absent": 1, "not_observable": 1 }, "consensus": null, "consensus_exists": false, "consensus_votes": 1, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.0 } }, "outcomes": { "event_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-C02/single_event_present": null } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/source_was_red_jacket_woman": "unclear" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": null, "FB-C02/source_was_red_jacket_woman": "unclear" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": null, "FB-C02/source_was_red_jacket_woman": "unclear", "FB-C02/bystanders_directed_attention": null } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 1, "H02": 2, "H03": null, "H04": 1 }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "primary_event_absent": 1 }, "mean": 1.3333, "median": 1, "sd": 0.4714, "min": 1, "max": 2 }, "temporal_causal_coherence": { "raw": { "H01": 2, "H02": 2, "H03": null, "H04": 2 }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "causal_prerequisite_absent": 1 }, "mean": 2, "median": 2, "sd": 0.0, "min": 2, "max": 2 }, "physical_social_plausibility": { "raw": { "H01": 2, "H02": 2, "H03": 3, "H04": 2 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.25, "median": 2.0, "sd": 0.433, "min": 2, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "source_was_red_jacket_woman": { "state": "unclear", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.6667, "signed_error": 1.6667 }, "temporal_causal_coherence": { "value": 2, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "source_was_red_jacket_woman": { "state": "unclear", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.6667, "signed_error": 1.6667 }, "temporal_causal_coherence": { "value": 2, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "source_was_red_jacket_woman": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "n/a:causal_prerequisite_absent", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false }, "source_was_red_jacket_woman": { "vote_counts": { "n/a:primary_event_absent": 1, "unclear": 2 }, "consensus": "unclear", "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "vote_counts": { "n/a:causal_prerequisite_absent": 1, "no": 2 }, "consensus": "no", "agrees_with_human_consensus": null, "calibration_eligible": false } } } }, { "sample_id": "EKXKK2", "scene_id": "FB-C02", "configuration_id": "flux-3-full", "factual": { "single_event_present": { "question_id": "FB-C02/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "source_was_red_jacket_woman": { "question_id": "FB-C02/source_was_red_jacket_woman", "role": "primary_causal", "raw": { "H01": "unclear", "H02": "unclear", "H03": "unclear", "H04": "yes" }, "counts": { "yes": 1, "no": 0, "unclear": 3, "na": 0 }, "na_reason_counts": {}, "consensus": "unclear", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "bystanders_directed_attention": { "question_id": "FB-C02/bystanders_directed_attention", "role": "other", "raw": { "H01": "no", "H02": "yes", "H03": "unclear", "H04": "no" }, "counts": { "yes": 1, "no": 2, "unclear": 1, "na": 0 }, "na_reason_counts": {}, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/source_was_red_jacket_woman": "unclear" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes", "FB-C02/source_was_red_jacket_woman": "unclear" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes", "FB-C02/source_was_red_jacket_woman": "unclear", "FB-C02/bystanders_directed_attention": null } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 4, "H03": 3, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.5, "median": 3.5, "sd": 0.5, "min": 3, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 3, "H02": 3, "H03": 2, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3, "median": 3.0, "sd": 0.7071, "min": 2, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": 3, "H02": 3, "H03": 3, "H04": 2 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.75, "median": 3.0, "sd": 0.433, "min": 2, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "unclear", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "no", "confidence": 2, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "unclear", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "unclear", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "vote_counts": { "unclear": 3 }, "consensus": "unclear", "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false } } } }, { "sample_id": "F9ZVRK", "scene_id": "FB-C02", "configuration_id": "grok-imagine-video-1.5", "factual": { "single_event_present": { "question_id": "FB-C02/single_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "unclear", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 3, "unclear": 1, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "source_was_red_jacket_woman": { "question_id": "FB-C02/source_was_red_jacket_woman", "role": "primary_causal", "raw": { "H01": "n/a:primary_event_absent", "H02": "yes", "H03": "n/a:primary_event_absent", "H04": "n/a:primary_event_absent" }, "counts": { "yes": 1, "no": 0, "unclear": 0, "na": 3 }, "na_reason_counts": { "primary_event_absent": 3 }, "consensus": "n/a:primary_event_absent", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "bystanders_directed_attention": { "question_id": "FB-C02/bystanders_directed_attention", "role": "other", "raw": { "H01": "n/a:causal_prerequisite_absent", "H02": "yes", "H03": "yes", "H04": "n/a:causal_prerequisite_absent" }, "counts": { "yes": 2, "no": 0, "unclear": 0, "na": 2 }, "na_reason_counts": { "causal_prerequisite_absent": 2 }, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.3333 } }, "outcomes": { "event_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "no" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/source_was_red_jacket_woman": "n/a:primary_event_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "no", "FB-C02/source_was_red_jacket_woman": "n/a:primary_event_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "no", "FB-C02/source_was_red_jacket_woman": "n/a:primary_event_absent", "FB-C02/bystanders_directed_attention": null } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": null, "H02": 1, "H03": null, "H04": null }, "numeric_reviewers": 1, "na_count": 3, "na_reason_counts": { "primary_event_absent": 3 }, "mean": 1, "median": 1, "sd": null, "min": 1, "max": 1 }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": 3, "H03": null, "H04": null }, "numeric_reviewers": 1, "na_count": 3, "na_reason_counts": { "causal_prerequisite_absent": 3 }, "mean": 3, "median": 3, "sd": null, "min": 3, "max": 3 }, "physical_social_plausibility": { "raw": { "H01": 3, "H02": 3, "H03": 4, "H04": null }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "not_observable": 1 }, "mean": 3.3333, "median": 3, "sd": 0.4714, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.3333, "signed_error": -0.3333 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.3333, "signed_error": -0.3333 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "unclear", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.3333, "signed_error": -0.3333 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "source_was_red_jacket_woman": { "vote_counts": { "unclear": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false } } } }, { "sample_id": "GDGAFY", "scene_id": "FB-074", "configuration_id": "seedance-2.5", "factual": { "chair_creak_present": { "question_id": "FB-074/chair_creak_present", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "separate_fart_event_present": { "question_id": "FB-074/separate_fart_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "creak_and_fart_distinguishable": { "question_id": "FB-074/creak_and_fart_distinguishable", "role": "primary_causal", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-074/creak_and_fart_distinguishable": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "yes", "FB-074/creak_and_fart_distinguishable": "yes" } }, "full_factual_success": { "value": "success", "applicable": true, "inputs": { "FB-074/chair_creak_present": "yes", "FB-074/separate_fart_event_present": "yes", "FB-074/creak_and_fart_distinguishable": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 4, "H03": 4, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4, "median": 4.0, "sd": 0.0, "min": 4, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 4, "H02": 3, "H03": 5, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4, "median": 4.0, "sd": 0.7071, "min": 3, "max": 5 }, "physical_social_plausibility": { "raw": { "H01": 4, "H02": 3, "H03": 4, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.75, "median": 4.0, "sd": 0.433, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 } } }, "gemini-3.7-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 } } }, "gemini-3.6-flash": { "factual": { "chair_creak_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "n/a:required_sound_absent", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 } } } }, "panel": { "chair_creak_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "separate_fart_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "vote_counts": { "n/a:required_sound_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "GQZK3X", "scene_id": "FB-512", "configuration_id": "flux-3-full", "factual": { "single_event_present": { "question_id": "FB-512/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "both_people_reacted": { "question_id": "FB-512/both_people_reacted", "role": "other", "raw": { "H01": "unclear", "H02": "no", "H03": "unclear", "H04": "yes" }, "counts": { "yes": 1, "no": 1, "unclear": 2, "na": 0 }, "na_reason_counts": {}, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 }, "recovered_without_further_emission": { "question_id": "FB-512/recovered_without_further_emission", "role": "primary_causal", "raw": { "H01": "unclear", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 3, "no": 0, "unclear": 1, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-512/recovered_without_further_emission": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/recovered_without_further_emission": "yes" } }, "full_factual_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/both_people_reacted": null, "FB-512/recovered_without_further_emission": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 3, "H03": 3, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.5, "median": 3.5, "sd": 0.5, "min": 3, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 4, "H02": 3, "H03": 3, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.5, "median": 3.5, "sd": 0.5, "min": 3, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": 4, "H02": 3, "H03": 3, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.5, "median": 3.5, "sd": 0.5, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "recovered_without_further_emission": { "state": "no", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "temporal_causal_coherence": { "value": 2, "rating_eligible": true, "abs_error": 1.5, "signed_error": -1.5 }, "physical_social_plausibility": { "value": 2, "rating_eligible": true, "abs_error": 1.5, "signed_error": -1.5 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "both_people_reacted": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "recovered_without_further_emission": { "state": "no", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "temporal_causal_coherence": { "value": 2, "rating_eligible": true, "abs_error": 1.5, "signed_error": -1.5 }, "physical_social_plausibility": { "value": 2, "rating_eligible": true, "abs_error": 1.5, "signed_error": -1.5 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "temporal_causal_coherence": { "value": 2, "rating_eligible": true, "abs_error": 1.5, "signed_error": -1.5 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "vote_counts": { "no": 3 }, "consensus": "no", "agrees_with_human_consensus": null, "calibration_eligible": false }, "recovered_without_further_emission": { "vote_counts": { "no": 2, "yes": 1 }, "consensus": "no", "agrees_with_human_consensus": false, "calibration_eligible": true } } } }, { "sample_id": "HP3MTU", "scene_id": "FB-074", "configuration_id": "ltx-2.5-pro", "factual": { "chair_creak_present": { "question_id": "FB-074/chair_creak_present", "role": "other", "raw": { "H01": "unclear", "H02": "no", "H03": "yes", "H04": "yes" }, "counts": { "yes": 2, "no": 1, "unclear": 1, "na": 0 }, "na_reason_counts": {}, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 }, "separate_fart_event_present": { "question_id": "FB-074/separate_fart_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "yes" }, "counts": { "yes": 1, "no": 3, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "creak_and_fart_distinguishable": { "question_id": "FB-074/creak_and_fart_distinguishable", "role": "primary_causal", "raw": { "H01": "n/a:required_sound_absent", "H02": "n/a:required_sound_absent", "H03": "n/a:required_sound_absent", "H04": "yes" }, "counts": { "yes": 1, "no": 0, "unclear": 0, "na": 3 }, "na_reason_counts": { "required_sound_absent": 3 }, "consensus": "n/a:required_sound_absent", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 } }, "outcomes": { "event_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "no" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "no", "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/chair_creak_present": null, "FB-074/separate_fart_event_present": "no", "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": null, "H02": null, "H03": null, "H04": 1 }, "numeric_reviewers": 1, "na_count": 3, "na_reason_counts": { "primary_event_absent": 3 }, "mean": 1, "median": 1, "sd": null, "min": 1, "max": 1 }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": null, "H03": null, "H04": 3 }, "numeric_reviewers": 1, "na_count": 3, "na_reason_counts": { "causal_prerequisite_absent": 3 }, "mean": 3, "median": 3, "sd": null, "min": 3, "max": 3 }, "physical_social_plausibility": { "raw": { "H01": null, "H02": 2, "H03": 2, "H04": 3 }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "not_observable": 1 }, "mean": 2.3333, "median": 2, "sd": 0.4714, "min": 2, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.6667, "signed_error": 0.6667 } } }, "gemini-3.7-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 4, "rating_eligible": true, "abs_error": 1.6667, "signed_error": 1.6667 } } }, "gemini-3.6-flash": { "factual": { "chair_creak_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "n/a:required_sound_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.6667, "signed_error": 0.6667 } } } }, "panel": { "chair_creak_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "vote_counts": { "n/a:required_sound_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true } } } }, { "sample_id": "J594DG", "scene_id": "FB-014", "configuration_id": "ltx-2.5-pro", "factual": { "single_event_present": { "question_id": "FB-014/single_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "yes" }, "counts": { "yes": 1, "no": 3, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "event_connected_to_lift": { "question_id": "FB-014/event_connected_to_lift", "role": "primary_causal", "raw": { "H01": "n/a:primary_event_absent", "H02": "n/a:primary_event_absent", "H03": "n/a:primary_event_absent", "H04": "yes" }, "counts": { "yes": 1, "no": 0, "unclear": 0, "na": 3 }, "na_reason_counts": { "primary_event_absent": 3 }, "consensus": "n/a:primary_event_absent", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "body_and_clothing_plausible": { "question_id": "FB-014/body_and_clothing_plausible", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/single_event_present": "no" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/event_connected_to_lift": "n/a:primary_event_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/single_event_present": "no", "FB-014/event_connected_to_lift": "n/a:primary_event_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/single_event_present": "no", "FB-014/event_connected_to_lift": "n/a:primary_event_absent", "FB-014/body_and_clothing_plausible": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": null, "H02": null, "H03": null, "H04": 1 }, "numeric_reviewers": 1, "na_count": 3, "na_reason_counts": { "primary_event_absent": 3 }, "mean": 1, "median": 1, "sd": null, "min": 1, "max": 1 }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": null, "H03": null, "H04": 4 }, "numeric_reviewers": 1, "na_count": 3, "na_reason_counts": { "causal_prerequisite_absent": 3 }, "mean": 4, "median": 4, "sd": null, "min": 4, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": null, "H02": 3, "H03": 3, "H04": 4 }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "not_observable": 1 }, "mean": 3.3333, "median": 3, "sd": 0.4714, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.3333, "signed_error": -0.3333 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 4, "rating_eligible": true, "abs_error": 0.6667, "signed_error": 0.6667 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.3333, "signed_error": -0.3333 } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "vote_counts": { "n/a:primary_event_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "J8X46G", "scene_id": "FB-C02", "configuration_id": "vidu-omni-pro-3.2a", "factual": { "single_event_present": { "question_id": "FB-C02/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "source_was_red_jacket_woman": { "question_id": "FB-C02/source_was_red_jacket_woman", "role": "primary_causal", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "bystanders_directed_attention": { "question_id": "FB-C02/bystanders_directed_attention", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/source_was_red_jacket_woman": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes", "FB-C02/source_was_red_jacket_woman": "yes" } }, "full_factual_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes", "FB-C02/source_was_red_jacket_woman": "yes", "FB-C02/bystanders_directed_attention": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 4, "H03": 4, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4, "median": 4.0, "sd": 0.0, "min": 4, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 5, "H02": 4, "H03": 5, "H04": 5 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4.75, "median": 5.0, "sd": 0.433, "min": 4, "max": 5 }, "physical_social_plausibility": { "raw": { "H01": 5, "H02": 5, "H03": 3, "H04": 5 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4.5, "median": 5.0, "sd": 0.866, "min": 3, "max": 5 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "unclear", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1.75, "signed_error": -1.75 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.5, "signed_error": -1.5 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "unclear", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1.75, "signed_error": -1.75 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.5, "signed_error": -1.5 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "unclear", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1.75, "signed_error": -1.75 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.5, "signed_error": -1.5 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "vote_counts": { "unclear": 3 }, "consensus": "unclear", "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "JGL4LW", "scene_id": "FB-074", "configuration_id": "minimax-h3-768p-local", "factual": { "chair_creak_present": { "question_id": "FB-074/chair_creak_present", "role": "other", "raw": { "H01": "unclear", "H02": "no", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 3, "unclear": 1, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "separate_fart_event_present": { "question_id": "FB-074/separate_fart_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "creak_and_fart_distinguishable": { "question_id": "FB-074/creak_and_fart_distinguishable", "role": "primary_causal", "raw": { "H01": "yes", "H02": "n/a:required_sound_absent", "H03": "n/a:required_sound_absent", "H04": "n/a:required_sound_absent" }, "counts": { "yes": 1, "no": 0, "unclear": 0, "na": 3 }, "na_reason_counts": { "required_sound_absent": 3 }, "consensus": "n/a:required_sound_absent", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "yes" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "yes", "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/chair_creak_present": "no", "FB-074/separate_fart_event_present": "yes", "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 2, "H02": 2, "H03": 1, "H04": 1 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 1.5, "median": 1.5, "sd": 0.5, "min": 1, "max": 2 }, "temporal_causal_coherence": { "raw": { "H01": 1, "H02": 2, "H03": 3, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.25, "median": 2.5, "sd": 0.8292, "min": 1, "max": 3 }, "physical_social_plausibility": { "raw": { "H01": 3, "H02": 2, "H03": 3, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.75, "median": 3.0, "sd": 0.433, "min": 2, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "chair_creak_present": { "state": "no", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "n/a:required_sound_absent", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.5, "signed_error": 1.5 }, "temporal_causal_coherence": { "value": 2, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 }, "physical_social_plausibility": { "value": 2, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 } } }, "gemini-3.7-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "separate_fart_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "n/a:required_sound_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 } } }, "gemini-3.6-flash": { "factual": { "chair_creak_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "separate_fart_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "n/a:required_sound_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 } } } }, "panel": { "chair_creak_present": { "vote_counts": { "no": 2, "yes": 1 }, "consensus": "no", "agrees_with_human_consensus": true, "calibration_eligible": true }, "separate_fart_event_present": { "vote_counts": { "no": 2, "yes": 1 }, "consensus": "no", "agrees_with_human_consensus": false, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "vote_counts": { "n/a:required_sound_absent": 3 }, "consensus": "n/a:required_sound_absent", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "K5DFBJ", "scene_id": "FB-014", "configuration_id": "flux-3-draft", "factual": { "single_event_present": { "question_id": "FB-014/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "event_connected_to_lift": { "question_id": "FB-014/event_connected_to_lift", "role": "primary_causal", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "body_and_clothing_plausible": { "question_id": "FB-014/body_and_clothing_plausible", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-014/event_connected_to_lift": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes", "FB-014/event_connected_to_lift": "yes" } }, "full_factual_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes", "FB-014/event_connected_to_lift": "yes", "FB-014/body_and_clothing_plausible": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 3, "H03": 3, "H04": 2 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3, "median": 3.0, "sd": 0.7071, "min": 2, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 3, "H02": 3, "H03": 3, "H04": 2 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.75, "median": 3.0, "sd": 0.433, "min": 2, "max": 3 }, "physical_social_plausibility": { "raw": { "H01": 3, "H02": 3, "H03": 3, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3, "median": 3.0, "sd": 0.0, "min": 3, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "KBZ29T", "scene_id": "FB-512", "configuration_id": "veo-3.1-fast", "factual": { "single_event_present": { "question_id": "FB-512/single_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "yes", "H03": "no", "H04": "no" }, "counts": { "yes": 1, "no": 3, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "both_people_reacted": { "question_id": "FB-512/both_people_reacted", "role": "other", "raw": { "H01": "n/a:not_observable", "H02": "unclear", "H03": "n/a:not_observable", "H04": "yes" }, "counts": { "yes": 1, "no": 0, "unclear": 1, "na": 2 }, "na_reason_counts": { "not_observable": 2 }, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 }, "recovered_without_further_emission": { "question_id": "FB-512/recovered_without_further_emission", "role": "primary_causal", "raw": { "H01": "n/a:primary_event_absent", "H02": "unclear", "H03": "no", "H04": "yes" }, "counts": { "yes": 1, "no": 1, "unclear": 1, "na": 1 }, "na_reason_counts": { "primary_event_absent": 1 }, "consensus": null, "consensus_exists": false, "consensus_votes": 1, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.0 } }, "outcomes": { "event_success": { "value": "not_success", "applicable": true, "inputs": { "FB-512/single_event_present": "no" } }, "primary_causal_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-512/recovered_without_further_emission": null } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-512/single_event_present": "no", "FB-512/recovered_without_further_emission": null } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-512/single_event_present": "no", "FB-512/both_people_reacted": null, "FB-512/recovered_without_further_emission": null } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 3, "H02": 3, "H03": 2, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.75, "median": 3.0, "sd": 0.433, "min": 2, "max": 3 }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": 2, "H03": 1, "H04": 1 }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "causal_prerequisite_absent": 1 }, "mean": 1.3333, "median": 1, "sd": 0.4714, "min": 1, "max": 2 }, "physical_social_plausibility": { "raw": { "H01": 3, "H02": 2, "H03": 1, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.25, "median": 2.5, "sd": 0.8292, "min": 1, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": null, "calibration_eligible": false }, "recovered_without_further_emission": { "state": "no", "confidence": 2, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "temporal_causal_coherence": { "value": 2, "rating_eligible": true, "abs_error": 0.6667, "signed_error": 0.6667 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "recovered_without_further_emission": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1.6667, "signed_error": 1.6667 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "recovered_without_further_emission": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1.6667, "signed_error": 1.6667 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 3 }, "consensus": "no", "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false }, "recovered_without_further_emission": { "vote_counts": { "no": 3 }, "consensus": "no", "agrees_with_human_consensus": null, "calibration_eligible": false } } } }, { "sample_id": "L49LSH", "scene_id": "FB-C02", "configuration_id": "minimax-h3-768p-local", "factual": { "single_event_present": { "question_id": "FB-C02/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "source_was_red_jacket_woman": { "question_id": "FB-C02/source_was_red_jacket_woman", "role": "primary_causal", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "bystanders_directed_attention": { "question_id": "FB-C02/bystanders_directed_attention", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/source_was_red_jacket_woman": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes", "FB-C02/source_was_red_jacket_woman": "yes" } }, "full_factual_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes", "FB-C02/source_was_red_jacket_woman": "yes", "FB-C02/bystanders_directed_attention": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 4, "H03": 1, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.25, "median": 4.0, "sd": 1.299, "min": 1, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 4, "H02": 3, "H03": 4, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.75, "median": 4.0, "sd": 0.433, "min": 3, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": 4, "H02": 4, "H03": 3, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.75, "median": 4.0, "sd": 0.433, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "LV5G5T", "scene_id": "FB-074", "configuration_id": "grok-imagine-video-1.5", "factual": { "chair_creak_present": { "question_id": "FB-074/chair_creak_present", "role": "other", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 4, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "separate_fart_event_present": { "question_id": "FB-074/separate_fart_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 4, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "creak_and_fart_distinguishable": { "question_id": "FB-074/creak_and_fart_distinguishable", "role": "primary_causal", "raw": { "H01": "unclear", "H02": "n/a:required_sound_absent", "H03": "n/a:required_sound_absent", "H04": "n/a:required_sound_absent" }, "counts": { "yes": 0, "no": 0, "unclear": 1, "na": 3 }, "na_reason_counts": { "required_sound_absent": 3 }, "consensus": "n/a:required_sound_absent", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 } }, "outcomes": { "event_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "no" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "no", "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/chair_creak_present": "no", "FB-074/separate_fart_event_present": "no", "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "primary_event_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "causal_prerequisite_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "physical_social_plausibility": { "raw": { "H01": null, "H02": null, "H03": 4, "H04": null }, "numeric_reviewers": 1, "na_count": 3, "na_reason_counts": { "not_observable": 3 }, "mean": 4, "median": 4, "sd": null, "min": 4, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "chair_creak_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "n/a:required_sound_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 2, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 2, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.7-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.6-flash": { "factual": { "chair_creak_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "separate_fart_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "n/a:required_sound_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } } }, "panel": { "chair_creak_present": { "vote_counts": { "no": 2, "yes": 1 }, "consensus": "no", "agrees_with_human_consensus": true, "calibration_eligible": true }, "separate_fart_event_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "vote_counts": { "n/a:required_sound_absent": 2, "yes": 1 }, "consensus": "n/a:required_sound_absent", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "LV9JB2", "scene_id": "FB-074", "configuration_id": "happyhorse-1.1", "factual": { "chair_creak_present": { "question_id": "FB-074/chair_creak_present", "role": "other", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "yes" }, "counts": { "yes": 1, "no": 3, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "separate_fart_event_present": { "question_id": "FB-074/separate_fart_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "creak_and_fart_distinguishable": { "question_id": "FB-074/creak_and_fart_distinguishable", "role": "primary_causal", "raw": { "H01": "n/a:required_sound_absent", "H02": "n/a:required_sound_absent", "H03": "n/a:required_sound_absent", "H04": "yes" }, "counts": { "yes": 1, "no": 0, "unclear": 0, "na": 3 }, "na_reason_counts": { "required_sound_absent": 3 }, "consensus": "n/a:required_sound_absent", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "yes" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "yes", "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/chair_creak_present": "no", "FB-074/separate_fart_event_present": "yes", "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 2, "H02": 2, "H03": 1, "H04": 1 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 1.5, "median": 1.5, "sd": 0.5, "min": 1, "max": 2 }, "temporal_causal_coherence": { "raw": { "H01": 3, "H02": 3, "H03": 2, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3, "median": 3.0, "sd": 0.7071, "min": 2, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": 3, "H02": 3, "H03": 3, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.25, "median": 3.0, "sd": 0.433, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.5, "signed_error": 1.5 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 } } }, "gemini-3.7-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.5, "signed_error": 1.5 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 } } }, "gemini-3.6-flash": { "factual": { "chair_creak_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "n/a:required_sound_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.5, "signed_error": 1.5 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 } } } }, "panel": { "chair_creak_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "separate_fart_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "vote_counts": { "n/a:required_sound_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true } } } }, { "sample_id": "MH6X9Z", "scene_id": "FB-074", "configuration_id": "ltx-2.5-fast", "factual": { "chair_creak_present": { "question_id": "FB-074/chair_creak_present", "role": "other", "raw": { "H01": "yes", "H02": "no", "H03": "no", "H04": "yes" }, "counts": { "yes": 2, "no": 2, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.3333 }, "separate_fart_event_present": { "question_id": "FB-074/separate_fart_event_present", "role": "primary_event", "raw": { "H01": "unclear", "H02": "no", "H03": "no", "H04": "yes" }, "counts": { "yes": 1, "no": 2, "unclear": 1, "na": 0 }, "na_reason_counts": {}, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 }, "creak_and_fart_distinguishable": { "question_id": "FB-074/creak_and_fart_distinguishable", "role": "primary_causal", "raw": { "H01": "yes", "H02": "n/a:required_sound_absent", "H03": "n/a:required_sound_absent", "H04": "yes" }, "counts": { "yes": 2, "no": 0, "unclear": 0, "na": 2 }, "na_reason_counts": { "required_sound_absent": 2 }, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.3333 } }, "outcomes": { "event_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": null } }, "primary_causal_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/creak_and_fart_distinguishable": null } }, "strict_scene_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": null, "FB-074/creak_and_fart_distinguishable": null } }, "full_factual_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/chair_creak_present": null, "FB-074/separate_fart_event_present": null, "FB-074/creak_and_fart_distinguishable": null } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 1, "H02": null, "H03": null, "H04": 1 }, "numeric_reviewers": 2, "na_count": 2, "na_reason_counts": { "primary_event_absent": 2 }, "mean": 1, "median": 1.0, "sd": 0.0, "min": 1, "max": 1 }, "temporal_causal_coherence": { "raw": { "H01": 3, "H02": null, "H03": null, "H04": 4 }, "numeric_reviewers": 2, "na_count": 2, "na_reason_counts": { "causal_prerequisite_absent": 2 }, "mean": 3.5, "median": 3.5, "sd": 0.5, "min": 3, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": null, "H02": 3, "H03": 3, "H04": 3 }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "not_observable": 1 }, "mean": 3, "median": 3, "sd": 0.0, "min": 3, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 } } }, "gemini-3.7-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 } } }, "gemini-3.6-flash": { "factual": { "chair_creak_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "creak_and_fart_distinguishable": { "state": "n/a:required_sound_absent", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 } } } }, "panel": { "chair_creak_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false }, "separate_fart_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false }, "creak_and_fart_distinguishable": { "vote_counts": { "n/a:required_sound_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false } } } }, { "sample_id": "P6R96U", "scene_id": "FB-C02", "configuration_id": "minimax-h3-2k-api", "factual": { "single_event_present": { "question_id": "FB-C02/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "source_was_red_jacket_woman": { "question_id": "FB-C02/source_was_red_jacket_woman", "role": "primary_causal", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "bystanders_directed_attention": { "question_id": "FB-C02/bystanders_directed_attention", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/source_was_red_jacket_woman": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes", "FB-C02/source_was_red_jacket_woman": "yes" } }, "full_factual_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes", "FB-C02/source_was_red_jacket_woman": "yes", "FB-C02/bystanders_directed_attention": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 4, "H03": 3, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.5, "median": 3.5, "sd": 0.5, "min": 3, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 4, "H02": 2, "H03": 4, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.25, "median": 3.5, "sd": 0.8292, "min": 2, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": 4, "H02": 2, "H03": 3, "H04": 2 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.75, "median": 2.5, "sd": 0.8292, "min": 2, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "P94D96", "scene_id": "FB-014", "configuration_id": "grok-imagine-video-1.5", "factual": { "single_event_present": { "question_id": "FB-014/single_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 4, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "event_connected_to_lift": { "question_id": "FB-014/event_connected_to_lift", "role": "primary_causal", "raw": { "H01": "n/a:primary_event_absent", "H02": "n/a:primary_event_absent", "H03": "n/a:primary_event_absent", "H04": "n/a:primary_event_absent" }, "counts": { "yes": 0, "no": 0, "unclear": 0, "na": 4 }, "na_reason_counts": { "primary_event_absent": 4 }, "consensus": "n/a:primary_event_absent", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "body_and_clothing_plausible": { "question_id": "FB-014/body_and_clothing_plausible", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "n/a:not_observable" }, "counts": { "yes": 3, "no": 0, "unclear": 0, "na": 1 }, "na_reason_counts": { "not_observable": 1 }, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 } }, "outcomes": { "event_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/single_event_present": "no" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/event_connected_to_lift": "n/a:primary_event_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/single_event_present": "no", "FB-014/event_connected_to_lift": "n/a:primary_event_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/single_event_present": "no", "FB-014/event_connected_to_lift": "n/a:primary_event_absent", "FB-014/body_and_clothing_plausible": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "primary_event_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "causal_prerequisite_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "physical_social_plausibility": { "raw": { "H01": null, "H02": 3, "H03": 4, "H04": 3 }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "not_observable": 1 }, "mean": 3.3333, "median": 3, "sd": 0.4714, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.3333, "signed_error": -0.3333 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.3333, "signed_error": -0.3333 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.3333, "signed_error": -0.3333 } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "vote_counts": { "n/a:primary_event_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "PN3WRY", "scene_id": "FB-512", "configuration_id": "minimax-h3-2k-api", "factual": { "single_event_present": { "question_id": "FB-512/single_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "unclear", "H03": "yes", "H04": "yes" }, "counts": { "yes": 2, "no": 1, "unclear": 1, "na": 0 }, "na_reason_counts": {}, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 }, "both_people_reacted": { "question_id": "FB-512/both_people_reacted", "role": "other", "raw": { "H01": "n/a:primary_event_absent", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 3, "no": 0, "unclear": 0, "na": 1 }, "na_reason_counts": { "primary_event_absent": 1 }, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "recovered_without_further_emission": { "question_id": "FB-512/recovered_without_further_emission", "role": "primary_causal", "raw": { "H01": "n/a:primary_event_absent", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 3, "no": 0, "unclear": 0, "na": 1 }, "na_reason_counts": { "primary_event_absent": 1 }, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 } }, "outcomes": { "event_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-512/single_event_present": null } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-512/recovered_without_further_emission": "yes" } }, "strict_scene_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-512/single_event_present": null, "FB-512/recovered_without_further_emission": "yes" } }, "full_factual_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-512/single_event_present": null, "FB-512/both_people_reacted": "yes", "FB-512/recovered_without_further_emission": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": null, "H02": 1, "H03": 1, "H04": 1 }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "primary_event_absent": 1 }, "mean": 1, "median": 1, "sd": 0.0, "min": 1, "max": 1 }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": 1, "H03": 2, "H04": 4 }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "causal_prerequisite_absent": 1 }, "mean": 2.3333, "median": 2, "sd": 1.2472, "min": 1, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": null, "H02": 3, "H03": 3, "H04": 4 }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "not_observable": 1 }, "mean": 3.3333, "median": 3, "sd": 0.4714, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": null, "calibration_eligible": false }, "both_people_reacted": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 2, "signed_error": 2 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 1.6667, "signed_error": 1.6667 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.3333, "signed_error": -0.3333 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 2, "signed_error": 2 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 1.6667, "signed_error": 1.6667 }, "physical_social_plausibility": { "value": 4, "rating_eligible": true, "abs_error": 0.6667, "signed_error": 0.6667 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": null, "calibration_eligible": false }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 2, "signed_error": 2 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 1.6667, "signed_error": 1.6667 }, "physical_social_plausibility": { "value": 4, "rating_eligible": true, "abs_error": 0.6667, "signed_error": 0.6667 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false }, "both_people_reacted": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "R26HLK", "scene_id": "FB-074", "configuration_id": "veo-3.1-fast", "factual": { "chair_creak_present": { "question_id": "FB-074/chair_creak_present", "role": "other", "raw": { "H01": "unclear", "H02": "unclear", "H03": "yes", "H04": "unclear" }, "counts": { "yes": 1, "no": 0, "unclear": 3, "na": 0 }, "na_reason_counts": {}, "consensus": "unclear", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "separate_fart_event_present": { "question_id": "FB-074/separate_fart_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "yes", "H03": "yes", "H04": "unclear" }, "counts": { "yes": 2, "no": 1, "unclear": 1, "na": 0 }, "na_reason_counts": {}, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 }, "creak_and_fart_distinguishable": { "question_id": "FB-074/creak_and_fart_distinguishable", "role": "primary_causal", "raw": { "H01": "n/a:required_sound_absent", "H02": "unclear", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 2, "unclear": 1, "na": 1 }, "na_reason_counts": { "required_sound_absent": 1 }, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 } }, "outcomes": { "event_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": null } }, "primary_causal_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/creak_and_fart_distinguishable": null } }, "strict_scene_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": null, "FB-074/creak_and_fart_distinguishable": null } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/chair_creak_present": "unclear", "FB-074/separate_fart_event_present": null, "FB-074/creak_and_fart_distinguishable": null } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 3, "H03": 1, "H04": 1 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.25, "median": 2.0, "sd": 1.299, "min": 1, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": 3, "H03": 1, "H04": 3 }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "causal_prerequisite_absent": 1 }, "mean": 2.3333, "median": 3, "sd": 0.9428, "min": 1, "max": 3 }, "physical_social_plausibility": { "raw": { "H01": 4, "H02": 3, "H03": 3, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.25, "median": 3.0, "sd": 0.433, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.6667, "signed_error": 0.6667 }, "physical_social_plausibility": { "value": 4, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 } } }, "gemini-3.7-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 1.6667, "signed_error": 1.6667 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 } } }, "gemini-3.6-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.6667, "signed_error": 0.6667 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 } } } }, "panel": { "chair_creak_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "separate_fart_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false }, "creak_and_fart_distinguishable": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false } } } }, { "sample_id": "RJJ3V5", "scene_id": "FB-512", "configuration_id": "minimax-h3-768p-local", "factual": { "single_event_present": { "question_id": "FB-512/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "both_people_reacted": { "question_id": "FB-512/both_people_reacted", "role": "other", "raw": { "H01": "unclear", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 3, "no": 0, "unclear": 1, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "recovered_without_further_emission": { "question_id": "FB-512/recovered_without_further_emission", "role": "primary_causal", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-512/recovered_without_further_emission": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/recovered_without_further_emission": "yes" } }, "full_factual_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/both_people_reacted": "yes", "FB-512/recovered_without_further_emission": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 5, "H03": 3, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4, "median": 4.0, "sd": 0.7071, "min": 3, "max": 5 }, "temporal_causal_coherence": { "raw": { "H01": 3, "H02": 5, "H03": 4, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4, "median": 4.0, "sd": 0.7071, "min": 3, "max": 5 }, "physical_social_plausibility": { "raw": { "H01": 3, "H02": 4, "H03": 4, "H04": 2 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.25, "median": 3.5, "sd": 0.8292, "min": 2, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "RMW286", "scene_id": "FB-C02", "configuration_id": "kling-video-3.0-omni", "factual": { "single_event_present": { "question_id": "FB-C02/single_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 4, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "source_was_red_jacket_woman": { "question_id": "FB-C02/source_was_red_jacket_woman", "role": "primary_causal", "raw": { "H01": "n/a:primary_event_absent", "H02": "yes", "H03": "n/a:primary_event_absent", "H04": "n/a:primary_event_absent" }, "counts": { "yes": 1, "no": 0, "unclear": 0, "na": 3 }, "na_reason_counts": { "primary_event_absent": 3 }, "consensus": "n/a:primary_event_absent", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "bystanders_directed_attention": { "question_id": "FB-C02/bystanders_directed_attention", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "n/a:causal_prerequisite_absent" }, "counts": { "yes": 3, "no": 0, "unclear": 0, "na": 1 }, "na_reason_counts": { "causal_prerequisite_absent": 1 }, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 } }, "outcomes": { "event_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "no" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/source_was_red_jacket_woman": "n/a:primary_event_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "no", "FB-C02/source_was_red_jacket_woman": "n/a:primary_event_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "no", "FB-C02/source_was_red_jacket_woman": "n/a:primary_event_absent", "FB-C02/bystanders_directed_attention": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "primary_event_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "temporal_causal_coherence": { "raw": { "H01": 3, "H02": 2, "H03": null, "H04": null }, "numeric_reviewers": 2, "na_count": 2, "na_reason_counts": { "causal_prerequisite_absent": 2 }, "mean": 2.5, "median": 2.5, "sd": 0.5, "min": 2, "max": 3 }, "physical_social_plausibility": { "raw": { "H01": 3, "H02": 2, "H03": 3, "H04": null }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "not_observable": 1 }, "mean": 2.6667, "median": 3, "sd": 0.4714, "min": 2, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.3333, "signed_error": 0.3333 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "n/a:causal_prerequisite_absent", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.3333, "signed_error": 0.3333 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.3333, "signed_error": 0.3333 } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 2, "yes": 1 }, "consensus": "no", "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "vote_counts": { "n/a:primary_event_absent": 2, "yes": 1 }, "consensus": "n/a:primary_event_absent", "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "vote_counts": { "n/a:causal_prerequisite_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "RXTC7G", "scene_id": "FB-074", "configuration_id": "vidu-omni-pro-3.2a", "factual": { "chair_creak_present": { "question_id": "FB-074/chair_creak_present", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "separate_fart_event_present": { "question_id": "FB-074/separate_fart_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "creak_and_fart_distinguishable": { "question_id": "FB-074/creak_and_fart_distinguishable", "role": "primary_causal", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-074/creak_and_fart_distinguishable": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "yes", "FB-074/creak_and_fart_distinguishable": "yes" } }, "full_factual_success": { "value": "success", "applicable": true, "inputs": { "FB-074/chair_creak_present": "yes", "FB-074/separate_fart_event_present": "yes", "FB-074/creak_and_fart_distinguishable": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 4, "H03": 4, "H04": 5 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4.25, "median": 4.0, "sd": 0.433, "min": 4, "max": 5 }, "temporal_causal_coherence": { "raw": { "H01": 4, "H02": 4, "H03": 3, "H04": 5 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4, "median": 4.0, "sd": 0.7071, "min": 3, "max": 5 }, "physical_social_plausibility": { "raw": { "H01": 4, "H02": 3, "H03": 2, "H04": 5 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.5, "median": 3.5, "sd": 1.118, "min": 2, "max": 5 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "chair_creak_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "n/a:required_sound_absent", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.25, "signed_error": -1.25 }, "temporal_causal_coherence": { "value": 2, "rating_eligible": true, "abs_error": 2, "signed_error": -2 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 } } }, "gemini-3.7-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.25, "signed_error": -1.25 }, "temporal_causal_coherence": { "value": 2, "rating_eligible": true, "abs_error": 2, "signed_error": -2 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 } } }, "gemini-3.6-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.25, "signed_error": -1.25 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 } } } }, "panel": { "chair_creak_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "separate_fart_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "vote_counts": { "n/a:required_sound_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "SKE7V8", "scene_id": "FB-512", "configuration_id": "seedance-2.5", "factual": { "single_event_present": { "question_id": "FB-512/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "both_people_reacted": { "question_id": "FB-512/both_people_reacted", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "recovered_without_further_emission": { "question_id": "FB-512/recovered_without_further_emission", "role": "primary_causal", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-512/recovered_without_further_emission": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/recovered_without_further_emission": "yes" } }, "full_factual_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/both_people_reacted": "yes", "FB-512/recovered_without_further_emission": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 3, "H03": 4, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.75, "median": 4.0, "sd": 0.433, "min": 3, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 5, "H02": 3, "H03": 4, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.75, "median": 3.5, "sd": 0.8292, "min": 3, "max": 5 }, "physical_social_plausibility": { "raw": { "H01": 5, "H02": 2, "H03": 3, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.5, "median": 3.5, "sd": 1.118, "min": 2, "max": 5 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "SXAZFW", "scene_id": "FB-C02", "configuration_id": "ltx-2.5-pro", "factual": { "single_event_present": { "question_id": "FB-C02/single_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 4, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "source_was_red_jacket_woman": { "question_id": "FB-C02/source_was_red_jacket_woman", "role": "primary_causal", "raw": { "H01": "yes", "H02": "n/a:primary_event_absent", "H03": "n/a:primary_event_absent", "H04": "n/a:primary_event_absent" }, "counts": { "yes": 1, "no": 0, "unclear": 0, "na": 3 }, "na_reason_counts": { "primary_event_absent": 3 }, "consensus": "n/a:primary_event_absent", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "bystanders_directed_attention": { "question_id": "FB-C02/bystanders_directed_attention", "role": "other", "raw": { "H01": "n/a:causal_prerequisite_absent", "H02": "n/a:causal_prerequisite_absent", "H03": "n/a:causal_prerequisite_absent", "H04": "n/a:causal_prerequisite_absent" }, "counts": { "yes": 0, "no": 0, "unclear": 0, "na": 4 }, "na_reason_counts": { "causal_prerequisite_absent": 4 }, "consensus": "n/a:causal_prerequisite_absent", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "no" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/source_was_red_jacket_woman": "n/a:primary_event_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "no", "FB-C02/source_was_red_jacket_woman": "n/a:primary_event_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-C02/single_event_present": "no", "FB-C02/source_was_red_jacket_woman": "n/a:primary_event_absent", "FB-C02/bystanders_directed_attention": "n/a:causal_prerequisite_absent" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "primary_event_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "causal_prerequisite_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "physical_social_plausibility": { "raw": { "H01": 3, "H02": 3, "H03": 3, "H04": null }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "not_observable": 1 }, "mean": 3, "median": 3, "sd": 0.0, "min": 3, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "unclear", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "no", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 2, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "unclear", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "no", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "n/a:causal_prerequisite_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0, "signed_error": 0 } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "source_was_red_jacket_woman": { "vote_counts": { "n/a:primary_event_absent": 1, "unclear": 2 }, "consensus": "unclear", "agrees_with_human_consensus": false, "calibration_eligible": true }, "bystanders_directed_attention": { "vote_counts": { "n/a:causal_prerequisite_absent": 1, "no": 2 }, "consensus": "no", "agrees_with_human_consensus": false, "calibration_eligible": true } } } }, { "sample_id": "SXQRDJ", "scene_id": "FB-014", "configuration_id": "happyhorse-1.1", "factual": { "single_event_present": { "question_id": "FB-014/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "unclear", "H03": "no", "H04": "no" }, "counts": { "yes": 1, "no": 2, "unclear": 1, "na": 0 }, "na_reason_counts": {}, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 }, "event_connected_to_lift": { "question_id": "FB-014/event_connected_to_lift", "role": "primary_causal", "raw": { "H01": "yes", "H02": "unclear", "H03": "n/a:primary_event_absent", "H04": "n/a:primary_event_absent" }, "counts": { "yes": 1, "no": 0, "unclear": 1, "na": 2 }, "na_reason_counts": { "primary_event_absent": 2 }, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 }, "body_and_clothing_plausible": { "question_id": "FB-014/body_and_clothing_plausible", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "n/a:not_observable" }, "counts": { "yes": 3, "no": 0, "unclear": 0, "na": 1 }, "na_reason_counts": { "not_observable": 1 }, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 } }, "outcomes": { "event_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-014/single_event_present": null } }, "primary_causal_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-014/event_connected_to_lift": null } }, "strict_scene_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-014/single_event_present": null, "FB-014/event_connected_to_lift": null } }, "full_factual_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-014/single_event_present": null, "FB-014/event_connected_to_lift": null, "FB-014/body_and_clothing_plausible": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 3, "H02": 1, "H03": null, "H04": null }, "numeric_reviewers": 2, "na_count": 2, "na_reason_counts": { "primary_event_absent": 2 }, "mean": 2, "median": 2.0, "sd": 1.0, "min": 1, "max": 3 }, "temporal_causal_coherence": { "raw": { "H01": 4, "H02": 3, "H03": null, "H04": null }, "numeric_reviewers": 2, "na_count": 2, "na_reason_counts": { "causal_prerequisite_absent": 2 }, "mean": 3.5, "median": 3.5, "sd": 0.5, "min": 3, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": 4, "H02": 3, "H03": 4, "H04": null }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "not_observable": 1 }, "mean": 3.6667, "median": 4, "sd": 0.4714, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 5, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 4, "rating_eligible": true, "abs_error": 0.3333, "signed_error": 0.3333 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 4, "rating_eligible": true, "abs_error": 0.3333, "signed_error": 0.3333 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "event_connected_to_lift": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 5, "rating_eligible": true, "abs_error": 1.3333, "signed_error": 1.3333 } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false }, "event_connected_to_lift": { "vote_counts": { "n/a:primary_event_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false }, "body_and_clothing_plausible": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "SZY2DC", "scene_id": "FB-C02", "configuration_id": "wan-3", "factual": { "single_event_present": { "question_id": "FB-C02/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "source_was_red_jacket_woman": { "question_id": "FB-C02/source_was_red_jacket_woman", "role": "primary_causal", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "bystanders_directed_attention": { "question_id": "FB-C02/bystanders_directed_attention", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/source_was_red_jacket_woman": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes", "FB-C02/source_was_red_jacket_woman": "yes" } }, "full_factual_success": { "value": "success", "applicable": true, "inputs": { "FB-C02/single_event_present": "yes", "FB-C02/source_was_red_jacket_woman": "yes", "FB-C02/bystanders_directed_attention": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 3, "H03": 4, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.75, "median": 4.0, "sd": 0.433, "min": 3, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 4, "H02": 4, "H03": 5, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4.25, "median": 4.0, "sd": 0.433, "min": 4, "max": 5 }, "physical_social_plausibility": { "raw": { "H01": 5, "H02": 4, "H03": 4, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4.25, "median": 4.0, "sd": 0.433, "min": 4, "max": 5 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.25, "signed_error": -1.25 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.25, "signed_error": -1.25 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1.25, "signed_error": -1.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.25, "signed_error": -1.25 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "source_was_red_jacket_woman": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "bystanders_directed_attention": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "TA85B8", "scene_id": "FB-512", "configuration_id": "sora-2-pro", "factual": { "single_event_present": { "question_id": "FB-512/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "both_people_reacted": { "question_id": "FB-512/both_people_reacted", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "recovered_without_further_emission": { "question_id": "FB-512/recovered_without_further_emission", "role": "primary_causal", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-512/recovered_without_further_emission": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/recovered_without_further_emission": "yes" } }, "full_factual_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/both_people_reacted": "yes", "FB-512/recovered_without_further_emission": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 4, "H03": 4, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4, "median": 4.0, "sd": 0.0, "min": 4, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 4, "H02": 3, "H03": 4, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.75, "median": 4.0, "sd": 0.433, "min": 3, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": 4, "H02": 3, "H03": 4, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.75, "median": 4.0, "sd": 0.433, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "UF4W9B", "scene_id": "FB-014", "configuration_id": "seedance-2.5", "factual": { "single_event_present": { "question_id": "FB-014/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "event_connected_to_lift": { "question_id": "FB-014/event_connected_to_lift", "role": "primary_causal", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "body_and_clothing_plausible": { "question_id": "FB-014/body_and_clothing_plausible", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-014/event_connected_to_lift": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes", "FB-014/event_connected_to_lift": "yes" } }, "full_factual_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes", "FB-014/event_connected_to_lift": "yes", "FB-014/body_and_clothing_plausible": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 4, "H03": 4, "H04": 5 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4.25, "median": 4.0, "sd": 0.433, "min": 4, "max": 5 }, "temporal_causal_coherence": { "raw": { "H01": 5, "H02": 4, "H03": 4, "H04": 5 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4.5, "median": 4.5, "sd": 0.5, "min": 4, "max": 5 }, "physical_social_plausibility": { "raw": { "H01": 4, "H02": 3, "H03": 4, "H04": 5 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4, "median": 4.0, "sd": 0.7071, "min": 3, "max": 5 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.25, "signed_error": -1.25 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.25, "signed_error": -1.25 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.25, "signed_error": -1.25 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "V65YL7", "scene_id": "FB-074", "configuration_id": "minimax-h3-2k-api", "factual": { "chair_creak_present": { "question_id": "FB-074/chair_creak_present", "role": "other", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 4, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "separate_fart_event_present": { "question_id": "FB-074/separate_fart_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 4, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "creak_and_fart_distinguishable": { "question_id": "FB-074/creak_and_fart_distinguishable", "role": "primary_causal", "raw": { "H01": "no", "H02": "n/a:required_sound_absent", "H03": "n/a:required_sound_absent", "H04": "n/a:required_sound_absent" }, "counts": { "yes": 0, "no": 1, "unclear": 0, "na": 3 }, "na_reason_counts": { "required_sound_absent": 3 }, "consensus": "n/a:required_sound_absent", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 } }, "outcomes": { "event_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "no" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/separate_fart_event_present": "no", "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-074/chair_creak_present": "no", "FB-074/separate_fart_event_present": "no", "FB-074/creak_and_fart_distinguishable": "n/a:required_sound_absent" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "primary_event_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "causal_prerequisite_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "physical_social_plausibility": { "raw": { "H01": null, "H02": 3, "H03": 3, "H04": null }, "numeric_reviewers": 2, "na_count": 2, "na_reason_counts": { "not_observable": 2 }, "mean": 3, "median": 3.0, "sd": 0.0, "min": 3, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.7-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.6-flash": { "factual": { "chair_creak_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "separate_fart_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } } }, "panel": { "chair_creak_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "separate_fart_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "creak_and_fart_distinguishable": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true } } } }, { "sample_id": "VUCT7K", "scene_id": "FB-512", "configuration_id": "ltx-2.5-fast", "factual": { "single_event_present": { "question_id": "FB-512/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "no", "H04": "yes" }, "counts": { "yes": 3, "no": 1, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "both_people_reacted": { "question_id": "FB-512/both_people_reacted", "role": "other", "raw": { "H01": "unclear", "H02": "yes", "H03": "n/a:primary_event_absent", "H04": "yes" }, "counts": { "yes": 2, "no": 0, "unclear": 1, "na": 1 }, "na_reason_counts": { "primary_event_absent": 1 }, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 }, "recovered_without_further_emission": { "question_id": "FB-512/recovered_without_further_emission", "role": "primary_causal", "raw": { "H01": "unclear", "H02": "yes", "H03": "n/a:primary_event_absent", "H04": "yes" }, "counts": { "yes": 2, "no": 0, "unclear": 1, "na": 1 }, "na_reason_counts": { "primary_event_absent": 1 }, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.1667 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes" } }, "primary_causal_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-512/recovered_without_further_emission": null } }, "strict_scene_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/recovered_without_further_emission": null } }, "full_factual_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/both_people_reacted": null, "FB-512/recovered_without_further_emission": null } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 1, "H02": 1, "H03": null, "H04": 1 }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "primary_event_absent": 1 }, "mean": 1, "median": 1, "sd": 0.0, "min": 1, "max": 1 }, "temporal_causal_coherence": { "raw": { "H01": 3, "H02": 1, "H03": null, "H04": 3 }, "numeric_reviewers": 3, "na_count": 1, "na_reason_counts": { "causal_prerequisite_absent": 1 }, "mean": 2.3333, "median": 3, "sd": 0.9428, "min": 1, "max": 3 }, "physical_social_plausibility": { "raw": { "H01": 2, "H02": 2, "H03": 2, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.25, "median": 2.0, "sd": 0.433, "min": 2, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "recovered_without_further_emission": { "state": "no", "confidence": 2, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 2, "signed_error": 2 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.6667, "signed_error": 0.6667 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 2, "signed_error": 2 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 1.6667, "signed_error": 1.6667 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "both_people_reacted": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "recovered_without_further_emission": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": 0.75 } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "vote_counts": { "n/a:primary_event_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false }, "recovered_without_further_emission": { "vote_counts": { "n/a:primary_event_absent": 1, "no": 1, "yes": 1 }, "consensus": null, "agrees_with_human_consensus": null, "calibration_eligible": false } } } }, { "sample_id": "WCYF6F", "scene_id": "FB-512", "configuration_id": "grok-imagine-video-1.5", "factual": { "single_event_present": { "question_id": "FB-512/single_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 4, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "both_people_reacted": { "question_id": "FB-512/both_people_reacted", "role": "other", "raw": { "H01": "n/a:primary_event_absent", "H02": "n/a:primary_event_absent", "H03": "n/a:primary_event_absent", "H04": "n/a:not_observable" }, "counts": { "yes": 0, "no": 0, "unclear": 0, "na": 4 }, "na_reason_counts": { "not_observable": 1, "primary_event_absent": 3 }, "consensus": "n/a:primary_event_absent", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "recovered_without_further_emission": { "question_id": "FB-512/recovered_without_further_emission", "role": "primary_causal", "raw": { "H01": "n/a:primary_event_absent", "H02": "n/a:primary_event_absent", "H03": "n/a:primary_event_absent", "H04": "n/a:primary_event_absent" }, "counts": { "yes": 0, "no": 0, "unclear": 0, "na": 4 }, "na_reason_counts": { "primary_event_absent": 4 }, "consensus": "n/a:primary_event_absent", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "not_success", "applicable": true, "inputs": { "FB-512/single_event_present": "no" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-512/recovered_without_further_emission": "n/a:primary_event_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-512/single_event_present": "no", "FB-512/recovered_without_further_emission": "n/a:primary_event_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-512/single_event_present": "no", "FB-512/both_people_reacted": "n/a:primary_event_absent", "FB-512/recovered_without_further_emission": "n/a:primary_event_absent" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "primary_event_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "causal_prerequisite_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "physical_social_plausibility": { "raw": { "H01": null, "H02": 3, "H03": 4, "H04": null }, "numeric_reviewers": 2, "na_count": 2, "na_reason_counts": { "not_observable": 2 }, "mean": 3.5, "median": 3.5, "sd": 0.5, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "both_people_reacted": { "vote_counts": { "n/a:primary_event_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "recovered_without_further_emission": { "vote_counts": { "n/a:primary_event_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true } } } }, { "sample_id": "WJHT3C", "scene_id": "FB-014", "configuration_id": "minimax-h3-2k-api", "factual": { "single_event_present": { "question_id": "FB-014/single_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 4, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "event_connected_to_lift": { "question_id": "FB-014/event_connected_to_lift", "role": "primary_causal", "raw": { "H01": "n/a:primary_event_absent", "H02": "yes", "H03": "n/a:primary_event_absent", "H04": "n/a:primary_event_absent" }, "counts": { "yes": 1, "no": 0, "unclear": 0, "na": 3 }, "na_reason_counts": { "primary_event_absent": 3 }, "consensus": "n/a:primary_event_absent", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "body_and_clothing_plausible": { "question_id": "FB-014/body_and_clothing_plausible", "role": "other", "raw": { "H01": "n/a:not_observable", "H02": "yes", "H03": "yes", "H04": "n/a:not_observable" }, "counts": { "yes": 2, "no": 0, "unclear": 0, "na": 2 }, "na_reason_counts": { "not_observable": 2 }, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.3333 } }, "outcomes": { "event_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/single_event_present": "no" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/event_connected_to_lift": "n/a:primary_event_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/single_event_present": "no", "FB-014/event_connected_to_lift": "n/a:primary_event_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/single_event_present": "no", "FB-014/event_connected_to_lift": "n/a:primary_event_absent", "FB-014/body_and_clothing_plausible": null } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "primary_event_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": 3, "H03": null, "H04": null }, "numeric_reviewers": 1, "na_count": 3, "na_reason_counts": { "causal_prerequisite_absent": 3 }, "mean": 3, "median": 3, "sd": null, "min": 3, "max": 3 }, "physical_social_plausibility": { "raw": { "H01": null, "H02": 3, "H03": 3, "H04": null }, "numeric_reviewers": 2, "na_count": 2, "na_reason_counts": { "not_observable": 2 }, "mean": 3, "median": 3.0, "sd": 0.0, "min": 3, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 2, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false } } } }, { "sample_id": "XND6TL", "scene_id": "FB-014", "configuration_id": "veo-3.1-fast", "factual": { "single_event_present": { "question_id": "FB-014/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "no", "H04": "yes" }, "counts": { "yes": 3, "no": 1, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 }, "event_connected_to_lift": { "question_id": "FB-014/event_connected_to_lift", "role": "primary_causal", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 4, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "body_and_clothing_plausible": { "question_id": "FB-014/body_and_clothing_plausible", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/event_connected_to_lift": "no" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes", "FB-014/event_connected_to_lift": "no" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes", "FB-014/event_connected_to_lift": "no", "FB-014/body_and_clothing_plausible": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 1, "H02": 2, "H03": 1, "H04": 1 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 1.25, "median": 1.0, "sd": 0.433, "min": 1, "max": 2 }, "temporal_causal_coherence": { "raw": { "H01": 2, "H02": 2, "H03": 1, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2, "median": 2.0, "sd": 0.7071, "min": 1, "max": 3 }, "physical_social_plausibility": { "raw": { "H01": 3, "H02": 2, "H03": 3, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.75, "median": 3.0, "sd": 0.433, "min": 2, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.75, "signed_error": 1.75 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": 1 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.75, "signed_error": 1.75 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 2, "signed_error": 2 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 1.75, "signed_error": 1.75 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": 1 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "XVFNKH", "scene_id": "FB-014", "configuration_id": "wan-3", "factual": { "single_event_present": { "question_id": "FB-014/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "event_connected_to_lift": { "question_id": "FB-014/event_connected_to_lift", "role": "primary_causal", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "body_and_clothing_plausible": { "question_id": "FB-014/body_and_clothing_plausible", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-014/event_connected_to_lift": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes", "FB-014/event_connected_to_lift": "yes" } }, "full_factual_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes", "FB-014/event_connected_to_lift": "yes", "FB-014/body_and_clothing_plausible": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 4, "H03": 2, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.5, "median": 4.0, "sd": 0.866, "min": 2, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 3, "H02": 4, "H03": 3, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.5, "median": 3.5, "sd": 0.5, "min": 3, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": 4, "H02": 4, "H03": 3, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.75, "median": 4.0, "sd": 0.433, "min": 3, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0.5, "signed_error": 0.5 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0.5, "signed_error": 0.5 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.75, "signed_error": -0.75 } } } }, "panel": { "single_event_present": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "YJCLF4", "scene_id": "FB-014", "configuration_id": "ltx-2.5-fast", "factual": { "single_event_present": { "question_id": "FB-014/single_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 4, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "event_connected_to_lift": { "question_id": "FB-014/event_connected_to_lift", "role": "primary_causal", "raw": { "H01": "n/a:primary_event_absent", "H02": "n/a:primary_event_absent", "H03": "n/a:primary_event_absent", "H04": "n/a:primary_event_absent" }, "counts": { "yes": 0, "no": 0, "unclear": 0, "na": 4 }, "na_reason_counts": { "primary_event_absent": 4 }, "consensus": "n/a:primary_event_absent", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "body_and_clothing_plausible": { "question_id": "FB-014/body_and_clothing_plausible", "role": "other", "raw": { "H01": "yes", "H02": "n/a:not_observable", "H03": "yes", "H04": "n/a:not_observable" }, "counts": { "yes": 2, "no": 0, "unclear": 0, "na": 2 }, "na_reason_counts": { "not_observable": 2 }, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.3333 } }, "outcomes": { "event_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/single_event_present": "no" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/event_connected_to_lift": "n/a:primary_event_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/single_event_present": "no", "FB-014/event_connected_to_lift": "n/a:primary_event_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/single_event_present": "no", "FB-014/event_connected_to_lift": "n/a:primary_event_absent", "FB-014/body_and_clothing_plausible": null } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "primary_event_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "causal_prerequisite_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "physical_social_plausibility": { "raw": { "H01": null, "H02": null, "H03": 3, "H04": null }, "numeric_reviewers": 1, "na_count": 3, "na_reason_counts": { "not_observable": 3 }, "mean": 3, "median": 3, "sd": null, "min": 3, "max": 3 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 2, "yes": 1 }, "consensus": "no", "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "vote_counts": { "n/a:primary_event_absent": 2, "yes": 1 }, "consensus": "n/a:primary_event_absent", "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false } } } }, { "sample_id": "YMLKA7", "scene_id": "FB-512", "configuration_id": "wan-3", "factual": { "single_event_present": { "question_id": "FB-512/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "both_people_reacted": { "question_id": "FB-512/both_people_reacted", "role": "other", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 4, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "recovered_without_further_emission": { "question_id": "FB-512/recovered_without_further_emission", "role": "primary_causal", "raw": { "H01": "no", "H02": "yes", "H03": "no", "H04": "yes" }, "counts": { "yes": 2, "no": 2, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.3333 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes" } }, "primary_causal_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-512/recovered_without_further_emission": null } }, "strict_scene_success": { "value": "no_consensus", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/recovered_without_further_emission": null } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-512/single_event_present": "yes", "FB-512/both_people_reacted": "no", "FB-512/recovered_without_further_emission": null } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 3, "H02": 3, "H03": 2, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.75, "median": 3.0, "sd": 0.433, "min": 2, "max": 3 }, "temporal_causal_coherence": { "raw": { "H01": 3, "H02": 3, "H03": 1, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.5, "median": 3.0, "sd": 0.866, "min": 1, "max": 3 }, "physical_social_plausibility": { "raw": { "H01": 3, "H02": 4, "H03": 3, "H04": 1 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 2.75, "median": 3.0, "sd": 1.0897, "min": 1, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "no", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": 0.5 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 1.5, "signed_error": 1.5 }, "physical_social_plausibility": { "value": 4, "rating_eligible": true, "abs_error": 1.25, "signed_error": 1.25 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": 0.5 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": 0.25 } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "recovered_without_further_emission": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false } } } }, { "sample_id": "YUNREY", "scene_id": "FB-014", "configuration_id": "sora-2", "factual": { "single_event_present": { "question_id": "FB-014/single_event_present", "role": "primary_event", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "event_connected_to_lift": { "question_id": "FB-014/event_connected_to_lift", "role": "primary_causal", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "body_and_clothing_plausible": { "question_id": "FB-014/body_and_clothing_plausible", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "yes" }, "counts": { "yes": 4, "no": 0, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "yes", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes" } }, "primary_causal_success": { "value": "success", "applicable": true, "inputs": { "FB-014/event_connected_to_lift": "yes" } }, "strict_scene_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes", "FB-014/event_connected_to_lift": "yes" } }, "full_factual_success": { "value": "success", "applicable": true, "inputs": { "FB-014/single_event_present": "yes", "FB-014/event_connected_to_lift": "yes", "FB-014/body_and_clothing_plausible": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": 4, "H02": 4, "H03": 3, "H04": 3 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.5, "median": 3.5, "sd": 0.5, "min": 3, "max": 4 }, "temporal_causal_coherence": { "raw": { "H01": 4, "H02": 4, "H03": 4, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 4, "median": 4.0, "sd": 0.0, "min": 4, "max": 4 }, "physical_social_plausibility": { "raw": { "H01": 2, "H02": 4, "H03": 3, "H04": 4 }, "numeric_reviewers": 4, "na_count": 0, "na_reason_counts": {}, "mean": 3.25, "median": 3.5, "sd": 0.8292, "min": 2, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "temporal_causal_coherence": { "value": 4, "rating_eligible": true, "abs_error": 0, "signed_error": 0 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.5, "signed_error": -0.5 }, "temporal_causal_coherence": { "value": 3, "rating_eligible": true, "abs_error": 1, "signed_error": -1 }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": true, "abs_error": 0.25, "signed_error": -0.25 } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 2, "yes": 1 }, "consensus": "no", "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "vote_counts": { "n/a:primary_event_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "YUTRHH", "scene_id": "FB-014", "configuration_id": "kling-video-3.0-omni", "factual": { "single_event_present": { "question_id": "FB-014/single_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 4, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "event_connected_to_lift": { "question_id": "FB-014/event_connected_to_lift", "role": "primary_causal", "raw": { "H01": "n/a:primary_event_absent", "H02": "n/a:primary_event_absent", "H03": "n/a:primary_event_absent", "H04": "n/a:primary_event_absent" }, "counts": { "yes": 0, "no": 0, "unclear": 0, "na": 4 }, "na_reason_counts": { "primary_event_absent": 4 }, "consensus": "n/a:primary_event_absent", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "body_and_clothing_plausible": { "question_id": "FB-014/body_and_clothing_plausible", "role": "other", "raw": { "H01": "yes", "H02": "yes", "H03": "yes", "H04": "n/a:not_observable" }, "counts": { "yes": 3, "no": 0, "unclear": 0, "na": 1 }, "na_reason_counts": { "not_observable": 1 }, "consensus": "yes", "consensus_exists": true, "consensus_votes": 3, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.5 } }, "outcomes": { "event_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/single_event_present": "no" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/event_connected_to_lift": "n/a:primary_event_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/single_event_present": "no", "FB-014/event_connected_to_lift": "n/a:primary_event_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-014/single_event_present": "no", "FB-014/event_connected_to_lift": "n/a:primary_event_absent", "FB-014/body_and_clothing_plausible": "yes" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "primary_event_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "causal_prerequisite_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "physical_social_plausibility": { "raw": { "H01": null, "H02": 2, "H03": 4, "H04": null }, "numeric_reviewers": 2, "na_count": 2, "na_reason_counts": { "not_observable": 2 }, "mean": 3, "median": 3.0, "sd": 1.0, "min": 2, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "event_connected_to_lift": { "state": "n/a:primary_event_absent", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 2, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "event_connected_to_lift": { "vote_counts": { "n/a:primary_event_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "body_and_clothing_plausible": { "vote_counts": { "yes": 3 }, "consensus": "yes", "agrees_with_human_consensus": true, "calibration_eligible": true } } } }, { "sample_id": "ZB6E8W", "scene_id": "FB-512", "configuration_id": "sora-2", "factual": { "single_event_present": { "question_id": "FB-512/single_event_present", "role": "primary_event", "raw": { "H01": "no", "H02": "no", "H03": "no", "H04": "no" }, "counts": { "yes": 0, "no": 4, "unclear": 0, "na": 0 }, "na_reason_counts": {}, "consensus": "no", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 }, "both_people_reacted": { "question_id": "FB-512/both_people_reacted", "role": "other", "raw": { "H01": "n/a:not_observable", "H02": "n/a:primary_event_absent", "H03": "n/a:primary_event_absent", "H04": "n/a:not_observable" }, "counts": { "yes": 0, "no": 0, "unclear": 0, "na": 4 }, "na_reason_counts": { "not_observable": 2, "primary_event_absent": 2 }, "consensus": null, "consensus_exists": false, "consensus_votes": 2, "contributing_reviewers": 4, "unanimous": false, "split": true, "pairwise_exact_agreement": 0.3333 }, "recovered_without_further_emission": { "question_id": "FB-512/recovered_without_further_emission", "role": "primary_causal", "raw": { "H01": "n/a:primary_event_absent", "H02": "n/a:primary_event_absent", "H03": "n/a:primary_event_absent", "H04": "n/a:primary_event_absent" }, "counts": { "yes": 0, "no": 0, "unclear": 0, "na": 4 }, "na_reason_counts": { "primary_event_absent": 4 }, "consensus": "n/a:primary_event_absent", "consensus_exists": true, "consensus_votes": 4, "contributing_reviewers": 4, "unanimous": true, "split": false, "pairwise_exact_agreement": 1.0 } }, "outcomes": { "event_success": { "value": "not_success", "applicable": true, "inputs": { "FB-512/single_event_present": "no" } }, "primary_causal_success": { "value": "not_success", "applicable": true, "inputs": { "FB-512/recovered_without_further_emission": "n/a:primary_event_absent" } }, "strict_scene_success": { "value": "not_success", "applicable": true, "inputs": { "FB-512/single_event_present": "no", "FB-512/recovered_without_further_emission": "n/a:primary_event_absent" } }, "full_factual_success": { "value": "not_success", "applicable": true, "inputs": { "FB-512/single_event_present": "no", "FB-512/both_people_reacted": null, "FB-512/recovered_without_further_emission": "n/a:primary_event_absent" } } }, "ratings": { "acoustic_plausibility": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "primary_event_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "temporal_causal_coherence": { "raw": { "H01": null, "H02": null, "H03": null, "H04": null }, "numeric_reviewers": 0, "na_count": 4, "na_reason_counts": { "causal_prerequisite_absent": 4 }, "mean": null, "median": null, "sd": null, "min": null, "max": null }, "physical_social_plausibility": { "raw": { "H01": null, "H02": null, "H03": 4, "H04": null }, "numeric_reviewers": 1, "na_count": 3, "na_reason_counts": { "not_observable": 3 }, "mean": 4, "median": 4, "sd": null, "min": 4, "max": 4 } }, "ai": { "judges": { "gemini-3.8-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.7-flash": { "factual": { "single_event_present": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true }, "both_people_reacted": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "recovered_without_further_emission": { "state": "yes", "confidence": 3, "agrees_with_human_consensus": false, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 4, "rating_eligible": false, "abs_error": null, "signed_error": null } } }, "gemini-3.6-flash": { "factual": { "single_event_present": { "state": "no", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true }, "both_people_reacted": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": null, "calibration_eligible": false }, "recovered_without_further_emission": { "state": "n/a:primary_event_absent", "confidence": 3, "agrees_with_human_consensus": true, "calibration_eligible": true } }, "ratings": { "acoustic_plausibility": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "temporal_causal_coherence": { "value": null, "rating_eligible": false, "abs_error": null, "signed_error": null }, "physical_social_plausibility": { "value": 3, "rating_eligible": false, "abs_error": null, "signed_error": null } } } }, "panel": { "single_event_present": { "vote_counts": { "no": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true }, "both_people_reacted": { "vote_counts": { "n/a:primary_event_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": null, "calibration_eligible": false }, "recovered_without_further_emission": { "vote_counts": { "n/a:primary_event_absent": 1, "yes": 2 }, "consensus": "yes", "agrees_with_human_consensus": false, "calibration_eligible": true } } } } ], "refusals": [ { "cell_id": "cell-kling-video-3.0-omni-fb-074", "configuration_id": "kling-video-3.0-omni", "scene_id": "FB-074", "canonical_outcome": "refusal", "message_category": "provider moderation", "provider_message": "Prompt blocked as it may violate TOS, scat detected in prompt", "provider_message_publication": "published", "human": null, "ai": null, "supplemental_variant": { "exists": true, "excluded_from_canonical_scoring": true } } ], "configurations": [ { "configuration_id": "wan-3", "label": "Wan 3", "family": "Alibaba Wan 3", "sample_ids": [ "7XGJH6", "SZY2DC", "XVFNKH", "YMLKA7" ], "refusal_cell_ids": [], "summary": { "scenes_total": 4, "scenes_evaluated": 4, "scenes_refused": 0, "event_success": { "success": 4, "not_success": 0, "no_consensus": 0, "evaluated": 4 }, "primary_causal_success": { "success": 2, "not_success": 0, "no_consensus": 2, "evaluated": 4 }, "strict_scene_success": { "success": 2, "not_success": 0, "no_consensus": 2, "evaluated": 4 }, "full_factual_success": { "success": 2, "not_success": 1, "no_consensus": 1, "evaluated": 4 }, "ratings": { "acoustic_plausibility": { "numeric_responses": 16, "na_responses": 0, "clips_with_numeric": 4, "mean": 3.1875, "median": 3.0, "sd": 0.8077 }, "temporal_causal_coherence": { "numeric_responses": 16, "na_responses": 0, "clips_with_numeric": 4, "mean": 3.375, "median": 3.0, "sd": 0.857 }, "physical_social_plausibility": { "numeric_responses": 16, "na_responses": 0, "clips_with_numeric": 4, "mean": 3.625, "median": 4.0, "sd": 0.857 } }, "human_questions": 12, "human_questions_with_consensus": 9, "ai_agreement": { "gemini-3.8-flash": { "agree": 7, "compared": 9 }, "gemini-3.7-flash": { "agree": 8, "compared": 9 }, "gemini-3.6-flash": { "agree": 8, "compared": 9 }, "panel": { "agree": 8, "compared": 9 } } } }, { "configuration_id": "flux-3-draft", "label": "FLUX 3 Draft", "family": "Black Forest Labs FLUX 3", "sample_ids": [ "3EYPKH", "DWPP3W", "E97W8V", "K5DFBJ" ], "refusal_cell_ids": [], "summary": { "scenes_total": 4, "scenes_evaluated": 4, "scenes_refused": 0, "event_success": { "success": 4, "not_success": 0, "no_consensus": 0, "evaluated": 4 }, "primary_causal_success": { "success": 2, "not_success": 2, "no_consensus": 0, "evaluated": 4 }, "strict_scene_success": { "success": 2, "not_success": 2, "no_consensus": 0, "evaluated": 4 }, "full_factual_success": { "success": 2, "not_success": 2, "no_consensus": 0, "evaluated": 4 }, "ratings": { "acoustic_plausibility": { "numeric_responses": 15, "na_responses": 1, "clips_with_numeric": 4, "mean": 2.4, "median": 2, "sd": 0.9522 }, "temporal_causal_coherence": { "numeric_responses": 15, "na_responses": 1, "clips_with_numeric": 4, "mean": 3.0667, "median": 3, "sd": 0.8537 }, "physical_social_plausibility": { "numeric_responses": 16, "na_responses": 0, "clips_with_numeric": 4, "mean": 3.125, "median": 3.0, "sd": 0.4841 } }, "human_questions": 12, "human_questions_with_consensus": 12, "ai_agreement": { "gemini-3.8-flash": { "agree": 11, "compared": 12 }, "gemini-3.7-flash": { "agree": 11, "compared": 12 }, "gemini-3.6-flash": { "agree": 8, "compared": 12 }, "panel": { "agree": 11, "compared": 12 } } } }, { "configuration_id": "flux-3-full", "label": "FLUX 3 Full", "family": "Black Forest Labs FLUX 3", "sample_ids": [ "7FY9MG", "D8P3PH", "EKXKK2", "GQZK3X" ], "refusal_cell_ids": [], "summary": { "scenes_total": 4, "scenes_evaluated": 4, "scenes_refused": 0, "event_success": { "success": 4, "not_success": 0, "no_consensus": 0, "evaluated": 4 }, "primary_causal_success": { "success": 1, "not_success": 1, "no_consensus": 2, "evaluated": 4 }, "strict_scene_success": { "success": 1, "not_success": 1, "no_consensus": 2, "evaluated": 4 }, "full_factual_success": { "success": 0, "not_success": 1, "no_consensus": 3, "evaluated": 4 }, "ratings": { "acoustic_plausibility": { "numeric_responses": 15, "na_responses": 1, "clips_with_numeric": 4, "mean": 3.1333, "median": 3, "sd": 0.8844 }, "temporal_causal_coherence": { "numeric_responses": 15, "na_responses": 1, "clips_with_numeric": 4, "mean": 3.0667, "median": 3, "sd": 0.6799 }, "physical_social_plausibility": { "numeric_responses": 16, "na_responses": 0, "clips_with_numeric": 4, "mean": 3.125, "median": 3.0, "sd": 0.5995 } }, "human_questions": 12, "human_questions_with_consensus": 7, "ai_agreement": { "gemini-3.8-flash": { "agree": 5, "compared": 7 }, "gemini-3.7-flash": { "agree": 5, "compared": 7 }, "gemini-3.6-flash": { "agree": 6, "compared": 7 }, "panel": { "agree": 5, "compared": 7 } } } }, { "configuration_id": "seedance-2.5", "label": "Seedance 2.5", "family": "ByteDance Seedance", "sample_ids": [ "ECWRMS", "GDGAFY", "SKE7V8", "UF4W9B" ], "refusal_cell_ids": [], "summary": { "scenes_total": 4, "scenes_evaluated": 4, "scenes_refused": 0, "event_success": { "success": 4, "not_success": 0, "no_consensus": 0, "evaluated": 4 }, "primary_causal_success": { "success": 4, "not_success": 0, "no_consensus": 0, "evaluated": 4 }, "strict_scene_success": { "success": 4, "not_success": 0, "no_consensus": 0, "evaluated": 4 }, "full_factual_success": { "success": 4, "not_success": 0, "no_consensus": 0, "evaluated": 4 }, "ratings": { "acoustic_plausibility": { "numeric_responses": 16, "na_responses": 0, "clips_with_numeric": 4, "mean": 3.9375, "median": 4.0, "sd": 0.6585 }, "temporal_causal_coherence": { "numeric_responses": 16, "na_responses": 0, "clips_with_numeric": 4, "mean": 4.1875, "median": 4.0, "sd": 0.7262 }, "physical_social_plausibility": { "numeric_responses": 16, "na_responses": 0, "clips_with_numeric": 4, "mean": 3.75, "median": 4.0, "sd": 0.9014 } }, "human_questions": 12, "human_questions_with_consensus": 12, "ai_agreement": { "gemini-3.8-flash": { "agree": 12, "compared": 12 }, "gemini-3.7-flash": { "agree": 12, "compared": 12 }, "gemini-3.6-flash": { "agree": 10, "compared": 12 }, "panel": { "agree": 12, "compared": 12 } } } }, { "configuration_id": "veo-3.1-fast", "label": "Veo 3.1 Fast", "family": "Google Veo 3.1", "sample_ids": [ "6D7MWZ", "KBZ29T", "R26HLK", "XND6TL" ], "refusal_cell_ids": [], "summary": { "scenes_total": 4, "scenes_evaluated": 4, "scenes_refused": 0, "event_success": { "success": 1, "not_success": 2, "no_consensus": 1, "evaluated": 4 }, "primary_causal_success": { "success": 0, "not_success": 1, "no_consensus": 3, "evaluated": 4 }, "strict_scene_success": { "success": 0, "not_success": 3, "no_consensus": 1, "evaluated": 4 }, "full_factual_success": { "success": 0, "not_success": 4, "no_consensus": 0, "evaluated": 4 }, "ratings": { "acoustic_plausibility": { "numeric_responses": 16, "na_responses": 0, "clips_with_numeric": 4, "mean": 2.3125, "median": 2.5, "sd": 1.044 }, "temporal_causal_coherence": { "numeric_responses": 14, "na_responses": 2, "clips_with_numeric": 4, "mean": 1.7143, "median": 1.5, "sd": 0.7954 }, "physical_social_plausibility": { "numeric_responses": 16, "na_responses": 0, "clips_with_numeric": 4, "mean": 2.375, "median": 3.0, "sd": 0.927 } }, "human_questions": 12, "human_questions_with_consensus": 7, "ai_agreement": { "gemini-3.8-flash": { "agree": 3, "compared": 7 }, "gemini-3.7-flash": { "agree": 4, "compared": 7 }, "gemini-3.6-flash": { "agree": 3, "compared": 7 }, "panel": { "agree": 4, "compared": 7 } } } }, { "configuration_id": "veo-3.1-standard", "label": "Veo 3.1 Standard", "family": "Google Veo 3.1", "sample_ids": [ "3Z7JPK", "845GNZ", "9X4F73", "CS9BVJ" ], "refusal_cell_ids": [], "summary": { "scenes_total": 4, "scenes_evaluated": 4, "scenes_refused": 0, "event_success": { "success": 3, "not_success": 0, "no_consensus": 1, "evaluated": 4 }, "primary_causal_success": { "success": 1, "not_success": 0, "no_consensus": 3, "evaluated": 4 }, "strict_scene_success": { "success": 1, "not_success": 0, "no_consensus": 3, "evaluated": 4 }, "full_factual_success": { "success": 1, "not_success": 0, "no_consensus": 3, "evaluated": 4 }, "ratings": { "acoustic_plausibility": { "numeric_responses": 16, "na_responses": 0, "clips_with_numeric": 4, "mean": 3.1875, "median": 3.5, "sd": 0.8817 }, "temporal_causal_coherence": { "numeric_responses": 16, "na_responses": 0, "clips_with_numeric": 4, "mean": 2.5625, "median": 2.0, "sd": 0.998 }, "physical_social_plausibility": { "numeric_responses": 16, "na_responses": 0, "clips_with_numeric": 4, "mean": 3.0625, "median": 3.0, "sd": 0.5555 } }, "human_questions": 12, "human_questions_with_consensus": 6, "ai_agreement": { "gemini-3.8-flash": { "agree": 5, "compared": 6 }, "gemini-3.7-flash": { "agree": 5, "compared": 6 }, "gemini-3.6-flash": { "agree": 4, "compared": 6 }, "panel": { "agree": 5, "compared": 6 } } } }, { "configuration_id": "happyhorse-1.1", "label": "HappyHorse 1.1", "family": "HappyHorse", "sample_ids": [ "6RR2XF", "BABQP3", "LV9JB2", "SXQRDJ" ], "refusal_cell_ids": [], "summary": { "scenes_total": 4, "scenes_evaluated": 4, "scenes_refused": 0, "event_success": { "success": 2, "not_success": 1, "no_consensus": 1, "evaluated": 4 }, "primary_causal_success": { "success": 1, "not_success": 2, "no_consensus": 1, "evaluated": 4 }, "strict_scene_success": { "success": 1, "not_success": 2, "no_consensus": 1, "evaluated": 4 }, "full_factual_success": { "success": 1, "not_success": 2, "no_consensus": 1, "evaluated": 4 }, "ratings": { "acoustic_plausibility": { "numeric_responses": 10, "na_responses": 6, "clips_with_numeric": 3, "mean": 1.6, "median": 1.5, "sd": 0.6633 }, "temporal_causal_coherence": { "numeric_responses": 11, "na_responses": 5, "clips_with_numeric": 4, "mean": 3.2727, "median": 3, "sd": 0.7497 }, "physical_social_plausibility": { "numeric_responses": 12, "na_responses": 4, "clips_with_numeric": 4, "mean": 3.5, "median": 3.5, "sd": 0.5 } }, "human_questions": 12, "human_questions_with_consensus": 9, "ai_agreement": { "gemini-3.8-flash": { "agree": 5, "compared": 9 }, "gemini-3.7-flash": { "agree": 5, "compared": 9 }, "gemini-3.6-flash": { "agree": 4, "compared": 9 }, "panel": { "agree": 5, "compared": 9 } } } }, { "configuration_id": "kling-video-3.0-omni", "label": "Kling VIDEO 3.0 Omni", "family": "Kling 3", "sample_ids": [ "5Q6WE5", "RMW286", "YUTRHH" ], "refusal_cell_ids": [ "cell-kling-video-3.0-omni-fb-074" ], "summary": { "scenes_total": 4, "scenes_evaluated": 3, "scenes_refused": 1, "event_success": { "success": 1, "not_success": 2, "no_consensus": 0, "evaluated": 3 }, "primary_causal_success": { "success": 1, "not_success": 2, "no_consensus": 0, "evaluated": 3 }, "strict_scene_success": { "success": 1, "not_success": 2, "no_consensus": 0, "evaluated": 3 }, "full_factual_success": { "success": 1, "not_success": 2, "no_consensus": 0, "evaluated": 3 }, "ratings": { "acoustic_plausibility": { "numeric_responses": 4, "na_responses": 8, "clips_with_numeric": 1, "mean": 3, "median": 3.0, "sd": 0.7071 }, "temporal_causal_coherence": { "numeric_responses": 6, "na_responses": 6, "clips_with_numeric": 2, "mean": 3.5, "median": 3.5, "sd": 0.9574 }, "physical_social_plausibility": { "numeric_responses": 9, "na_responses": 3, "clips_with_numeric": 3, "mean": 3.1111, "median": 3, "sd": 0.737 } }, "human_questions": 9, "human_questions_with_consensus": 9, "ai_agreement": { "gemini-3.8-flash": { "agree": 7, "compared": 9 }, "gemini-3.7-flash": { "agree": 6, "compared": 9 }, "gemini-3.6-flash": { "agree": 7, "compared": 9 }, "panel": { "agree": 7, "compared": 9 } } } }, { "configuration_id": "ltx-2.5-fast", "label": "LTX-2.5 Fast", "family": "LTX-2.5", "sample_ids": [ "3G6SS5", "MH6X9Z", "VUCT7K", "YJCLF4" ], "refusal_cell_ids": [], "summary": { "scenes_total": 4, "scenes_evaluated": 4, "scenes_refused": 0, "event_success": { "success": 1, "not_success": 2, "no_consensus": 1, "evaluated": 4 }, "primary_causal_success": { "success": 0, "not_success": 2, "no_consensus": 2, "evaluated": 4 }, "strict_scene_success": { "success": 0, "not_success": 2, "no_consensus": 2, "evaluated": 4 }, "full_factual_success": { "success": 0, "not_success": 2, "no_consensus": 2, "evaluated": 4 }, "ratings": { "acoustic_plausibility": { "numeric_responses": 6, "na_responses": 10, "clips_with_numeric": 3, "mean": 1, "median": 1.0, "sd": 0.0 }, "temporal_causal_coherence": { "numeric_responses": 6, "na_responses": 10, "clips_with_numeric": 3, "mean": 2.8333, "median": 3.0, "sd": 0.8975 }, "physical_social_plausibility": { "numeric_responses": 10, "na_responses": 6, "clips_with_numeric": 4, "mean": 2.7, "median": 3.0, "sd": 0.6403 } }, "human_questions": 12, "human_questions_with_consensus": 5, "ai_agreement": { "gemini-3.8-flash": { "agree": 3, "compared": 5 }, "gemini-3.7-flash": { "agree": 1, "compared": 5 }, "gemini-3.6-flash": { "agree": 2, "compared": 5 }, "panel": { "agree": 3, "compared": 5 } } } }, { "configuration_id": "ltx-2.5-pro", "label": "LTX-2.5 Pro", "family": "LTX-2.5", "sample_ids": [ "D2UWK6", "HP3MTU", "J594DG", "SXAZFW" ], "refusal_cell_ids": [], "summary": { "scenes_total": 4, "scenes_evaluated": 4, "scenes_refused": 0, "event_success": { "success": 0, "not_success": 4, "no_consensus": 0, "evaluated": 4 }, "primary_causal_success": { "success": 0, "not_success": 4, "no_consensus": 0, "evaluated": 4 }, "strict_scene_success": { "success": 0, "not_success": 4, "no_consensus": 0, "evaluated": 4 }, "full_factual_success": { "success": 0, "not_success": 4, "no_consensus": 0, "evaluated": 4 }, "ratings": { "acoustic_plausibility": { "numeric_responses": 2, "na_responses": 14, "clips_with_numeric": 2, "mean": 1, "median": 1.0, "sd": 0.0 }, "temporal_causal_coherence": { "numeric_responses": 2, "na_responses": 14, "clips_with_numeric": 2, "mean": 3.5, "median": 3.5, "sd": 0.5 }, "physical_social_plausibility": { "numeric_responses": 11, "na_responses": 5, "clips_with_numeric": 4, "mean": 2.9091, "median": 3, "sd": 0.5143 } }, "human_questions": 12, "human_questions_with_consensus": 11, "ai_agreement": { "gemini-3.8-flash": { "agree": 4, "compared": 11 }, "gemini-3.7-flash": { "agree": 4, "compared": 11 }, "gemini-3.6-flash": { "agree": 10, "compared": 11 }, "panel": { "agree": 4, "compared": 11 } } } }, { "configuration_id": "minimax-h3-2k-api", "label": "MiniMax H3 2K API", "family": "MiniMax H3", "sample_ids": [ "P6R96U", "PN3WRY", "V65YL7", "WJHT3C" ], "refusal_cell_ids": [], "summary": { "scenes_total": 4, "scenes_evaluated": 4, "scenes_refused": 0, "event_success": { "success": 1, "not_success": 2, "no_consensus": 1, "evaluated": 4 }, "primary_causal_success": { "success": 2, "not_success": 2, "no_consensus": 0, "evaluated": 4 }, "strict_scene_success": { "success": 1, "not_success": 2, "no_consensus": 1, "evaluated": 4 }, "full_factual_success": { "success": 1, "not_success": 2, "no_consensus": 1, "evaluated": 4 }, "ratings": { "acoustic_plausibility": { "numeric_responses": 7, "na_responses": 9, "clips_with_numeric": 2, "mean": 2.4286, "median": 3, "sd": 1.2936 }, "temporal_causal_coherence": { "numeric_responses": 8, "na_responses": 8, "clips_with_numeric": 3, "mean": 2.875, "median": 3.0, "sd": 1.0533 }, "physical_social_plausibility": { "numeric_responses": 11, "na_responses": 5, "clips_with_numeric": 4, "mean": 3, "median": 3, "sd": 0.603 } }, "human_questions": 12, "human_questions_with_consensus": 10, "ai_agreement": { "gemini-3.8-flash": { "agree": 5, "compared": 10 }, "gemini-3.7-flash": { "agree": 5, "compared": 10 }, "gemini-3.6-flash": { "agree": 5, "compared": 10 }, "panel": { "agree": 5, "compared": 10 } } } }, { "configuration_id": "minimax-h3-768p-local", "label": "MiniMax H3 768p local", "family": "MiniMax H3", "sample_ids": [ "8SJDU5", "JGL4LW", "L49LSH", "RJJ3V5" ], "refusal_cell_ids": [], "summary": { "scenes_total": 4, "scenes_evaluated": 4, "scenes_refused": 0, "event_success": { "success": 4, "not_success": 0, "no_consensus": 0, "evaluated": 4 }, "primary_causal_success": { "success": 2, "not_success": 1, "no_consensus": 1, "evaluated": 4 }, "strict_scene_success": { "success": 2, "not_success": 1, "no_consensus": 1, "evaluated": 4 }, "full_factual_success": { "success": 2, "not_success": 1, "no_consensus": 1, "evaluated": 4 }, "ratings": { "acoustic_plausibility": { "numeric_responses": 15, "na_responses": 1, "clips_with_numeric": 4, "mean": 2.7333, "median": 3, "sd": 1.34 }, "temporal_causal_coherence": { "numeric_responses": 15, "na_responses": 1, "clips_with_numeric": 4, "mean": 3, "median": 3, "sd": 1.1547 }, "physical_social_plausibility": { "numeric_responses": 16, "na_responses": 0, "clips_with_numeric": 4, "mean": 3.0625, "median": 3.0, "sd": 0.7474 } }, "human_questions": 12, "human_questions_with_consensus": 11, "ai_agreement": { "gemini-3.8-flash": { "agree": 11, "compared": 11 }, "gemini-3.7-flash": { "agree": 9, "compared": 11 }, "gemini-3.6-flash": { "agree": 10, "compared": 11 }, "panel": { "agree": 10, "compared": 11 } } } }, { "configuration_id": "sora-2", "label": "Sora 2", "family": "OpenAI Sora", "sample_ids": [ "7K724T", "EFXRPX", "YUNREY", "ZB6E8W" ], "refusal_cell_ids": [], "summary": { "scenes_total": 4, "scenes_evaluated": 4, "scenes_refused": 0, "event_success": { "success": 1, "not_success": 1, "no_consensus": 2, "evaluated": 4 }, "primary_causal_success": { "success": 1, "not_success": 2, "no_consensus": 1, "evaluated": 4 }, "strict_scene_success": { "success": 1, "not_success": 2, "no_consensus": 1, "evaluated": 4 }, "full_factual_success": { "success": 1, "not_success": 2, "no_consensus": 1, "evaluated": 4 }, "ratings": { "acoustic_plausibility": { "numeric_responses": 11, "na_responses": 5, "clips_with_numeric": 3, "mean": 2.2727, "median": 2, "sd": 1.1355 }, "temporal_causal_coherence": { "numeric_responses": 11, "na_responses": 5, "clips_with_numeric": 3, "mean": 2.9091, "median": 3, "sd": 0.9 }, "physical_social_plausibility": { "numeric_responses": 11, "na_responses": 5, "clips_with_numeric": 4, "mean": 2.8182, "median": 3, "sd": 0.8332 } }, "human_questions": 12, "human_questions_with_consensus": 6, "ai_agreement": { "gemini-3.8-flash": { "agree": 4, "compared": 6 }, "gemini-3.7-flash": { "agree": 3, "compared": 6 }, "gemini-3.6-flash": { "agree": 3, "compared": 6 }, "panel": { "agree": 3, "compared": 6 } } } }, { "configuration_id": "sora-2-pro", "label": "Sora 2 Pro", "family": "OpenAI Sora", "sample_ids": [ "3MNRCL", "5JSTTP", "D6JHRC", "TA85B8" ], "refusal_cell_ids": [], "summary": { "scenes_total": 4, "scenes_evaluated": 4, "scenes_refused": 0, "event_success": { "success": 2, "not_success": 1, "no_consensus": 1, "evaluated": 4 }, "primary_causal_success": { "success": 1, "not_success": 3, "no_consensus": 0, "evaluated": 4 }, "strict_scene_success": { "success": 1, "not_success": 3, "no_consensus": 0, "evaluated": 4 }, "full_factual_success": { "success": 1, "not_success": 3, "no_consensus": 0, "evaluated": 4 }, "ratings": { "acoustic_plausibility": { "numeric_responses": 10, "na_responses": 6, "clips_with_numeric": 3, "mean": 3.1, "median": 3.0, "sd": 0.8307 }, "temporal_causal_coherence": { "numeric_responses": 10, "na_responses": 6, "clips_with_numeric": 3, "mean": 3.3, "median": 3.5, "sd": 0.781 }, "physical_social_plausibility": { "numeric_responses": 12, "na_responses": 4, "clips_with_numeric": 4, "mean": 2.9167, "median": 3.0, "sd": 0.862 } }, "human_questions": 12, "human_questions_with_consensus": 10, "ai_agreement": { "gemini-3.8-flash": { "agree": 8, "compared": 10 }, "gemini-3.7-flash": { "agree": 5, "compared": 10 }, "gemini-3.6-flash": { "agree": 9, "compared": 10 }, "panel": { "agree": 8, "compared": 10 } } } }, { "configuration_id": "vidu-omni-pro-3.2a", "label": "Vidu Omni Pro 3.2a", "family": "Vidu", "sample_ids": [ "4EPZWT", "7AKCCT", "J8X46G", "RXTC7G" ], "refusal_cell_ids": [], "summary": { "scenes_total": 4, "scenes_evaluated": 4, "scenes_refused": 0, "event_success": { "success": 4, "not_success": 0, "no_consensus": 0, "evaluated": 4 }, "primary_causal_success": { "success": 4, "not_success": 0, "no_consensus": 0, "evaluated": 4 }, "strict_scene_success": { "success": 4, "not_success": 0, "no_consensus": 0, "evaluated": 4 }, "full_factual_success": { "success": 4, "not_success": 0, "no_consensus": 0, "evaluated": 4 }, "ratings": { "acoustic_plausibility": { "numeric_responses": 16, "na_responses": 0, "clips_with_numeric": 4, "mean": 4.0625, "median": 4.0, "sd": 0.2421 }, "temporal_causal_coherence": { "numeric_responses": 16, "na_responses": 0, "clips_with_numeric": 4, "mean": 4.25, "median": 4.0, "sd": 0.6614 }, "physical_social_plausibility": { "numeric_responses": 16, "na_responses": 0, "clips_with_numeric": 4, "mean": 4, "median": 4.0, "sd": 0.7906 } }, "human_questions": 12, "human_questions_with_consensus": 12, "ai_agreement": { "gemini-3.8-flash": { "agree": 9, "compared": 12 }, "gemini-3.7-flash": { "agree": 11, "compared": 12 }, "gemini-3.6-flash": { "agree": 11, "compared": 12 }, "panel": { "agree": 11, "compared": 12 } } } }, { "configuration_id": "grok-imagine-video-1.5", "label": "Grok Imagine Video 1.5", "family": "xAI Grok Imagine", "sample_ids": [ "F9ZVRK", "LV5G5T", "P94D96", "WCYF6F" ], "refusal_cell_ids": [], "summary": { "scenes_total": 4, "scenes_evaluated": 4, "scenes_refused": 0, "event_success": { "success": 0, "not_success": 4, "no_consensus": 0, "evaluated": 4 }, "primary_causal_success": { "success": 0, "not_success": 4, "no_consensus": 0, "evaluated": 4 }, "strict_scene_success": { "success": 0, "not_success": 4, "no_consensus": 0, "evaluated": 4 }, "full_factual_success": { "success": 0, "not_success": 4, "no_consensus": 0, "evaluated": 4 }, "ratings": { "acoustic_plausibility": { "numeric_responses": 1, "na_responses": 15, "clips_with_numeric": 1, "mean": 1, "median": 1, "sd": null }, "temporal_causal_coherence": { "numeric_responses": 1, "na_responses": 15, "clips_with_numeric": 1, "mean": 3, "median": 3, "sd": null }, "physical_social_plausibility": { "numeric_responses": 9, "na_responses": 7, "clips_with_numeric": 4, "mean": 3.4444, "median": 3, "sd": 0.4969 } }, "human_questions": 12, "human_questions_with_consensus": 11, "ai_agreement": { "gemini-3.8-flash": { "agree": 3, "compared": 11 }, "gemini-3.7-flash": { "agree": 1, "compared": 11 }, "gemini-3.6-flash": { "agree": 9, "compared": 11 }, "panel": { "agree": 3, "compared": 11 } } } } ], "reliability": { "per_question": { "FB-C02/single_event_present": { "cells": 16, "consensus": 14, "no_consensus": 2, "consensus_rate": 0.875, "unanimous": 10, "split": 6, "unanimous_rate": 0.625, "pairwise_exact_agreement": 0.7708, "alpha": 0.5815, "alpha_ci95": [ 0.2971, 0.8149 ], "alpha_metric": "nominal over exact states (N/A reasons distinct)", "raw_vote_counts": { "no": 24, "unclear": 4, "yes": 36 } }, "FB-C02/source_was_red_jacket_woman": { "cells": 16, "consensus": 14, "no_consensus": 2, "consensus_rate": 0.875, "unanimous": 6, "split": 10, "unanimous_rate": 0.375, "pairwise_exact_agreement": 0.6562, "alpha": 0.4504, "alpha_ci95": [ 0.2335, 0.609 ], "alpha_metric": "nominal over exact states (N/A reasons distinct)", "raw_vote_counts": { "n/a:primary_event_absent": 17, "unclear": 14, "yes": 33 } }, "FB-C02/bystanders_directed_attention": { "cells": 16, "consensus": 10, "no_consensus": 6, "consensus_rate": 0.625, "unanimous": 7, "split": 9, "unanimous_rate": 0.4375, "pairwise_exact_agreement": 0.6042, "alpha": 0.3243, "alpha_ci95": [ 0.1026, 0.5372 ], "alpha_metric": "nominal over exact states (N/A reasons distinct)", "raw_vote_counts": { "n/a:causal_prerequisite_absent": 12, "n/a:not_observable": 2, "no": 7, "unclear": 4, "yes": 39 } }, "FB-014/single_event_present": { "cells": 16, "consensus": 15, "no_consensus": 1, "consensus_rate": 0.9375, "unanimous": 10, "split": 6, "unanimous_rate": 0.625, "pairwise_exact_agreement": 0.7917, "alpha": 0.5996, "alpha_ci95": [ 0.3024, 0.8408 ], "alpha_metric": "nominal over exact states (N/A reasons distinct)", "raw_vote_counts": { "no": 29, "unclear": 1, "yes": 34 } }, "FB-014/event_connected_to_lift": { "cells": 16, "consensus": 13, "no_consensus": 3, "consensus_rate": 0.8125, "unanimous": 10, "split": 6, "unanimous_rate": 0.625, "pairwise_exact_agreement": 0.75, "alpha": 0.609, "alpha_ci95": [ 0.3342, 0.8389 ], "alpha_metric": "nominal over exact states (N/A reasons distinct)", "raw_vote_counts": { "n/a:primary_event_absent": 26, "no": 7, "unclear": 3, "yes": 28 } }, "FB-014/body_and_clothing_plausible": { "cells": 16, "consensus": 13, "no_consensus": 3, "consensus_rate": 0.8125, "unanimous": 9, "split": 7, "unanimous_rate": 0.5625, "pairwise_exact_agreement": 0.75, "alpha": 0.0667, "alpha_ci95": [ -0.08, 0.2105 ], "alpha_metric": "nominal over exact states (N/A reasons distinct)", "raw_vote_counts": { "n/a:not_observable": 10, "yes": 54 } }, "FB-074/chair_creak_present": { "cells": 15, "consensus": 9, "no_consensus": 6, "consensus_rate": 0.6, "unanimous": 5, "split": 10, "unanimous_rate": 0.3333, "pairwise_exact_agreement": 0.5667, "alpha": 0.2872, "alpha_ci95": [ -0.0183, 0.5305 ], "alpha_metric": "nominal over exact states (N/A reasons distinct)", "raw_vote_counts": { "no": 32, "unclear": 10, "yes": 18 } }, "FB-074/separate_fart_event_present": { "cells": 15, "consensus": 11, "no_consensus": 4, "consensus_rate": 0.7333, "unanimous": 9, "split": 6, "unanimous_rate": 0.6, "pairwise_exact_agreement": 0.7333, "alpha": 0.4679, "alpha_ci95": [ 0.1368, 0.7528 ], "alpha_metric": "nominal over exact states (N/A reasons distinct)", "raw_vote_counts": { "no": 17, "unclear": 4, "yes": 39 } }, "FB-074/creak_and_fart_distinguishable": { "cells": 15, "consensus": 9, "no_consensus": 6, "consensus_rate": 0.6, "unanimous": 2, "split": 13, "unanimous_rate": 0.1333, "pairwise_exact_agreement": 0.4667, "alpha": 0.1712, "alpha_ci95": [ -0.0716, 0.3611 ], "alpha_metric": "nominal over exact states (N/A reasons distinct)", "raw_vote_counts": { "n/a:required_sound_absent": 31, "no": 6, "unclear": 6, "yes": 17 } }, "FB-512/single_event_present": { "cells": 16, "consensus": 15, "no_consensus": 1, "consensus_rate": 0.9375, "unanimous": 12, "split": 4, "unanimous_rate": 0.75, "pairwise_exact_agreement": 0.8542, "alpha": 0.6632, "alpha_ci95": [ 0.2696, 0.9302 ], "alpha_metric": "nominal over exact states (N/A reasons distinct)", "raw_vote_counts": { "no": 18, "unclear": 1, "yes": 45 } }, "FB-512/both_people_reacted": { "cells": 16, "consensus": 12, "no_consensus": 4, "consensus_rate": 0.75, "unanimous": 8, "split": 8, "unanimous_rate": 0.5, "pairwise_exact_agreement": 0.6771, "alpha": 0.4833, "alpha_ci95": [ 0.1967, 0.7064 ], "alpha_metric": "nominal over exact states (N/A reasons distinct)", "raw_vote_counts": { "n/a:not_observable": 6, "n/a:primary_event_absent": 11, "no": 5, "unclear": 5, "yes": 37 } }, "FB-512/recovered_without_further_emission": { "cells": 16, "consensus": 12, "no_consensus": 4, "consensus_rate": 0.75, "unanimous": 9, "split": 7, "unanimous_rate": 0.5625, "pairwise_exact_agreement": 0.6979, "alpha": 0.4984, "alpha_ci95": [ 0.2222, 0.7489 ], "alpha_metric": "nominal over exact states (N/A reasons distinct)", "raw_vote_counts": { "n/a:primary_event_absent": 15, "no": 5, "unclear": 7, "yes": 37 } } }, "per_scene": { "FB-C02": { "cells": 48, "consensus": 38, "no_consensus": 10, "consensus_rate": 0.7917, "unanimous": 23, "split": 25, "unanimous_rate": 0.4792, "pairwise_exact_agreement": 0.6771, "alpha": 0.4922, "alpha_ci95": [ 0.3657, 0.6037 ], "alpha_metric": "nominal over exact states (N/A reasons distinct)", "raw_vote_counts": { "n/a:causal_prerequisite_absent": 12, "n/a:not_observable": 2, "n/a:primary_event_absent": 17, "no": 31, "unclear": 22, "yes": 108 } }, "FB-014": { "cells": 48, "consensus": 41, "no_consensus": 7, "consensus_rate": 0.8542, "unanimous": 29, "split": 19, "unanimous_rate": 0.6042, "pairwise_exact_agreement": 0.7639, "alpha": 0.5939, "alpha_ci95": [ 0.4268, 0.7141 ], "alpha_metric": "nominal over exact states (N/A reasons distinct)", "raw_vote_counts": { "n/a:not_observable": 10, "n/a:primary_event_absent": 26, "no": 36, "unclear": 4, "yes": 116 } }, "FB-074": { "cells": 45, "consensus": 29, "no_consensus": 16, "consensus_rate": 0.6444, "unanimous": 16, "split": 29, "unanimous_rate": 0.3556, "pairwise_exact_agreement": 0.5889, "alpha": 0.4123, "alpha_ci95": [ 0.2791, 0.5212 ], "alpha_metric": "nominal over exact states (N/A reasons distinct)", "raw_vote_counts": { "n/a:required_sound_absent": 31, "no": 55, "unclear": 20, "yes": 74 } }, "FB-512": { "cells": 48, "consensus": 39, "no_consensus": 9, "consensus_rate": 0.8125, "unanimous": 29, "split": 19, "unanimous_rate": 0.6042, "pairwise_exact_agreement": 0.7431, "alpha": 0.5521, "alpha_ci95": [ 0.3871, 0.7041 ], "alpha_metric": "nominal over exact states (N/A reasons distinct)", "raw_vote_counts": { "n/a:not_observable": 6, "n/a:primary_event_absent": 26, "no": 28, "unclear": 13, "yes": 119 } } }, "overall": { "cells": 189, "consensus": 147, "no_consensus": 42, "consensus_rate": 0.7778, "unanimous": 97, "split": 92, "unanimous_rate": 0.5132, "pairwise_exact_agreement": 0.6949, "alpha": 0.5235, "alpha_ci95": [ 0.4519, 0.5879 ], "alpha_metric": "nominal over exact states (N/A reasons distinct)", "raw_vote_counts": { "n/a:causal_prerequisite_absent": 12, "n/a:not_observable": 18, "n/a:primary_event_absent": 69, "n/a:required_sound_absent": 31, "no": 150, "unclear": 59, "yes": 417 } }, "pairwise_reviewers": { "H01-H02": 0.6772, "H01-H03": 0.6878, "H01-H04": 0.7037, "H02-H03": 0.7725, "H02-H04": 0.6667, "H03-H04": 0.6614 }, "ratings": { "acoustic_plausibility": { "numeric": 176, "not_applicable": 76, "alpha": 0.5404, "alpha_ci95": [ 0.3669, 0.669 ], "alpha_metric": "interval, N/A missing (ordinal is not in the frozen implementation)", "pairwise_exact_agreement": 0.4239, "pairwise_within_one_point": 0.8395, "mean_within_clip_sd": 0.5672 }, "temporal_causal_coherence": { "numeric": 178, "not_applicable": 74, "alpha": 0.5145, "alpha_ci95": [ 0.3528, 0.6329 ], "alpha_metric": "interval, N/A missing (ordinal is not in the frozen implementation)", "pairwise_exact_agreement": 0.3361, "pairwise_within_one_point": 0.8487, "mean_within_clip_sd": 0.5876 }, "physical_social_plausibility": { "numeric": 213, "not_applicable": 39, "alpha": 0.3394, "alpha_ci95": [ 0.132, 0.4997 ], "alpha_metric": "interval, N/A missing (ordinal is not in the frozen implementation)", "pairwise_exact_agreement": 0.4362, "pairwise_within_one_point": 0.9043, "mean_within_clip_sd": 0.4837 } }, "caveats": [ { "scope": "scene", "target": "FB-074", "kind": "low_consensus_reliability", "text": "FB-074 has comparatively low human consensus and reliability; conclusions about any configuration in this scene carry an explicit uncertainty caveat." }, { "scope": "question", "target": "FB-014/body_and_clothing_plausible", "kind": "prevalence_low_alpha", "text": "High raw agreement but low Krippendorff alpha, because almost every answer is the same; read the raw vote proportions beside alpha rather than treating alpha alone as poor reviewer quality." } ] }, "ai_calibration": { "thresholds": { "ordinary_gate": 0.7, "primary_causal_gate": 0.75, "rating_mae": 1.0, "rating_min_human_numeric": 3, "panel_requires": "2 of 3" }, "gates": { "FB-C02/single_event_present": { "role": "primary_event", "threshold": 0.7, "human_consensus_samples": 14, "human_no_consensus_samples": 2, "judges": { "gemini-3.8-flash": { "agree": 8, "compared": 14, "rate": 0.5714, "wilson95": [ 0.3259, 0.7862 ], "status": "not_calibrated" }, "gemini-3.7-flash": { "agree": 9, "compared": 14, "rate": 0.6429, "wilson95": [ 0.3876, 0.8366 ], "status": "not_calibrated" }, "gemini-3.6-flash": { "agree": 10, "compared": 14, "rate": 0.7143, "wilson95": [ 0.4535, 0.8828 ], "status": "calibrated" } }, "panel": { "agree": 9, "compared": 14, "rate": 0.6429, "wilson95": [ 0.3876, 0.8366 ], "status": "not_calibrated", "panel_without_majority": 0 } }, "FB-C02/source_was_red_jacket_woman": { "role": "primary_causal", "threshold": 0.75, "human_consensus_samples": 14, "human_no_consensus_samples": 2, "judges": { "gemini-3.8-flash": { "agree": 8, "compared": 14, "rate": 0.5714, "wilson95": [ 0.3259, 0.7862 ], "status": "not_calibrated_for_primary_causal_gate" }, "gemini-3.7-flash": { "agree": 9, "compared": 14, "rate": 0.6429, "wilson95": [ 0.3876, 0.8366 ], "status": "not_calibrated_for_primary_causal_gate" }, "gemini-3.6-flash": { "agree": 8, "compared": 14, "rate": 0.5714, "wilson95": [ 0.3259, 0.7862 ], "status": "not_calibrated_for_primary_causal_gate" } }, "panel": { "agree": 9, "compared": 14, "rate": 0.6429, "wilson95": [ 0.3876, 0.8366 ], "status": "not_calibrated_for_primary_causal_gate", "panel_without_majority": 0 } }, "FB-C02/bystanders_directed_attention": { "role": "other", "threshold": 0.7, "human_consensus_samples": 10, "human_no_consensus_samples": 6, "judges": { "gemini-3.8-flash": { "agree": 7, "compared": 10, "rate": 0.7, "wilson95": [ 0.3968, 0.8922 ], "status": "calibrated" }, "gemini-3.7-flash": { "agree": 7, "compared": 10, "rate": 0.7, "wilson95": [ 0.3968, 0.8922 ], "status": "calibrated" }, "gemini-3.6-flash": { "agree": 9, "compared": 10, "rate": 0.9, "wilson95": [ 0.5958, 0.9821 ], "status": "calibrated" } }, "panel": { "agree": 8, "compared": 10, "rate": 0.8, "wilson95": [ 0.4902, 0.9433 ], "status": "calibrated", "panel_without_majority": 0 } }, "FB-014/single_event_present": { "role": "primary_event", "threshold": 0.7, "human_consensus_samples": 15, "human_no_consensus_samples": 1, "judges": { "gemini-3.8-flash": { "agree": 11, "compared": 15, "rate": 0.7333, "wilson95": [ 0.4805, 0.891 ], "status": "calibrated" }, "gemini-3.7-flash": { "agree": 8, "compared": 15, "rate": 0.5333, "wilson95": [ 0.3012, 0.7519 ], "status": "not_calibrated" }, "gemini-3.6-flash": { "agree": 10, "compared": 15, "rate": 0.6667, "wilson95": [ 0.4171, 0.8482 ], "status": "not_calibrated" } }, "panel": { "agree": 9, "compared": 15, "rate": 0.6, "wilson95": [ 0.3575, 0.8018 ], "status": "not_calibrated", "panel_without_majority": 0 } }, "FB-014/event_connected_to_lift": { "role": "primary_causal", "threshold": 0.75, "human_consensus_samples": 13, "human_no_consensus_samples": 3, "judges": { "gemini-3.8-flash": { "agree": 8, "compared": 13, "rate": 0.6154, "wilson95": [ 0.3552, 0.8229 ], "status": "not_calibrated_for_primary_causal_gate" }, "gemini-3.7-flash": { "agree": 6, "compared": 13, "rate": 0.4615, "wilson95": [ 0.2321, 0.7086 ], "status": "not_calibrated_for_primary_causal_gate" }, "gemini-3.6-flash": { "agree": 9, "compared": 13, "rate": 0.6923, "wilson95": [ 0.4237, 0.8732 ], "status": "not_calibrated_for_primary_causal_gate" } }, "panel": { "agree": 7, "compared": 13, "rate": 0.5385, "wilson95": [ 0.2914, 0.7679 ], "status": "not_calibrated_for_primary_causal_gate", "panel_without_majority": 0 } }, "FB-014/body_and_clothing_plausible": { "role": "other", "threshold": 0.7, "human_consensus_samples": 13, "human_no_consensus_samples": 3, "judges": { "gemini-3.8-flash": { "agree": 13, "compared": 13, "rate": 1.0, "wilson95": [ 0.7719, 1.0 ], "status": "calibrated" }, "gemini-3.7-flash": { "agree": 13, "compared": 13, "rate": 1.0, "wilson95": [ 0.7719, 1.0 ], "status": "calibrated" }, "gemini-3.6-flash": { "agree": 13, "compared": 13, "rate": 1.0, "wilson95": [ 0.7719, 1.0 ], "status": "calibrated" } }, "panel": { "agree": 13, "compared": 13, "rate": 1.0, "wilson95": [ 0.7719, 1.0 ], "status": "calibrated", "panel_without_majority": 0 } }, "FB-074/chair_creak_present": { "role": "other", "threshold": 0.7, "human_consensus_samples": 9, "human_no_consensus_samples": 6, "judges": { "gemini-3.8-flash": { "agree": 5, "compared": 9, "rate": 0.5556, "wilson95": [ 0.2667, 0.8112 ], "status": "not_calibrated" }, "gemini-3.7-flash": { "agree": 3, "compared": 9, "rate": 0.3333, "wilson95": [ 0.1206, 0.6458 ], "status": "not_calibrated" }, "gemini-3.6-flash": { "agree": 6, "compared": 9, "rate": 0.6667, "wilson95": [ 0.3542, 0.8794 ], "status": "not_calibrated" } }, "panel": { "agree": 6, "compared": 9, "rate": 0.6667, "wilson95": [ 0.3542, 0.8794 ], "status": "not_calibrated", "panel_without_majority": 0 } }, "FB-074/separate_fart_event_present": { "role": "primary_event", "threshold": 0.7, "human_consensus_samples": 11, "human_no_consensus_samples": 4, "judges": { "gemini-3.8-flash": { "agree": 8, "compared": 11, "rate": 0.7273, "wilson95": [ 0.4344, 0.9025 ], "status": "calibrated" }, "gemini-3.7-flash": { "agree": 7, "compared": 11, "rate": 0.6364, "wilson95": [ 0.3538, 0.8483 ], "status": "not_calibrated" }, "gemini-3.6-flash": { "agree": 8, "compared": 11, "rate": 0.7273, "wilson95": [ 0.4344, 0.9025 ], "status": "calibrated" } }, "panel": { "agree": 7, "compared": 11, "rate": 0.6364, "wilson95": [ 0.3538, 0.8483 ], "status": "not_calibrated", "panel_without_majority": 0 } }, "FB-074/creak_and_fart_distinguishable": { "role": "primary_causal", "threshold": 0.75, "human_consensus_samples": 9, "human_no_consensus_samples": 6, "judges": { "gemini-3.8-flash": { "agree": 5, "compared": 9, "rate": 0.5556, "wilson95": [ 0.2667, 0.8112 ], "status": "not_calibrated_for_primary_causal_gate" }, "gemini-3.7-flash": { "agree": 4, "compared": 9, "rate": 0.4444, "wilson95": [ 0.1888, 0.7333 ], "status": "not_calibrated_for_primary_causal_gate" }, "gemini-3.6-flash": { "agree": 7, "compared": 9, "rate": 0.7778, "wilson95": [ 0.4526, 0.9368 ], "status": "calibrated" } }, "panel": { "agree": 6, "compared": 9, "rate": 0.6667, "wilson95": [ 0.3542, 0.8794 ], "status": "not_calibrated_for_primary_causal_gate", "panel_without_majority": 0 } }, "FB-512/single_event_present": { "role": "primary_event", "threshold": 0.7, "human_consensus_samples": 15, "human_no_consensus_samples": 1, "judges": { "gemini-3.8-flash": { "agree": 11, "compared": 15, "rate": 0.7333, "wilson95": [ 0.4805, 0.891 ], "status": "calibrated" }, "gemini-3.7-flash": { "agree": 11, "compared": 15, "rate": 0.7333, "wilson95": [ 0.4805, 0.891 ], "status": "calibrated" }, "gemini-3.6-flash": { "agree": 11, "compared": 15, "rate": 0.7333, "wilson95": [ 0.4805, 0.891 ], "status": "calibrated" } }, "panel": { "agree": 12, "compared": 15, "rate": 0.8, "wilson95": [ 0.5481, 0.9295 ], "status": "calibrated", "panel_without_majority": 0 } }, "FB-512/both_people_reacted": { "role": "other", "threshold": 0.7, "human_consensus_samples": 12, "human_no_consensus_samples": 4, "judges": { "gemini-3.8-flash": { "agree": 10, "compared": 12, "rate": 0.8333, "wilson95": [ 0.552, 0.953 ], "status": "calibrated" }, "gemini-3.7-flash": { "agree": 10, "compared": 12, "rate": 0.8333, "wilson95": [ 0.552, 0.953 ], "status": "calibrated" }, "gemini-3.6-flash": { "agree": 8, "compared": 12, "rate": 0.6667, "wilson95": [ 0.3906, 0.8619 ], "status": "not_calibrated" } }, "panel": { "agree": 10, "compared": 12, "rate": 0.8333, "wilson95": [ 0.552, 0.953 ], "status": "calibrated", "panel_without_majority": 0 } }, "FB-512/recovered_without_further_emission": { "role": "primary_causal", "threshold": 0.75, "human_consensus_samples": 12, "human_no_consensus_samples": 4, "judges": { "gemini-3.8-flash": { "agree": 8, "compared": 12, "rate": 0.6667, "wilson95": [ 0.3906, 0.8619 ], "status": "not_calibrated_for_primary_causal_gate" }, "gemini-3.7-flash": { "agree": 8, "compared": 12, "rate": 0.6667, "wilson95": [ 0.3906, 0.8619 ], "status": "not_calibrated_for_primary_causal_gate" }, "gemini-3.6-flash": { "agree": 10, "compared": 12, "rate": 0.8333, "wilson95": [ 0.552, 0.953 ], "status": "calibrated" } }, "panel": { "agree": 8, "compared": 12, "rate": 0.6667, "wilson95": [ 0.3906, 0.8619 ], "status": "not_calibrated_for_primary_causal_gate", "panel_without_majority": 0 } } }, "pooled": { "ordinary_gates": { "gemini-3.8-flash": { "agree": 73, "compared": 99, "rate": 0.7374, "wilson95": [ 0.643, 0.814 ] }, "gemini-3.7-flash": { "agree": 68, "compared": 99, "rate": 0.6869, "wilson95": [ 0.59, 0.7698 ] }, "gemini-3.6-flash": { "agree": 75, "compared": 99, "rate": 0.7576, "wilson95": [ 0.6646, 0.8313 ] }, "panel": { "agree": 74, "compared": 99, "rate": 0.7475, "wilson95": [ 0.6538, 0.8227 ] } }, "primary_causal_gates": { "gemini-3.8-flash": { "agree": 29, "compared": 48, "rate": 0.6042, "wilson95": [ 0.4631, 0.7298 ] }, "gemini-3.7-flash": { "agree": 27, "compared": 48, "rate": 0.5625, "wilson95": [ 0.4228, 0.693 ] }, "gemini-3.6-flash": { "agree": 34, "compared": 48, "rate": 0.7083, "wilson95": [ 0.5682, 0.8176 ] }, "panel": { "agree": 30, "compared": 48, "rate": 0.625, "wilson95": [ 0.4836, 0.7478 ] } }, "all_gates": { "gemini-3.8-flash": { "agree": 102, "compared": 147, "rate": 0.6939, "wilson95": [ 0.6152, 0.7627 ] }, "gemini-3.7-flash": { "agree": 95, "compared": 147, "rate": 0.6463, "wilson95": [ 0.5661, 0.7189 ] }, "gemini-3.6-flash": { "agree": 109, "compared": 147, "rate": 0.7415, "wilson95": [ 0.6652, 0.8055 ] }, "panel": { "agree": 104, "compared": 147, "rate": 0.7075, "wilson95": [ 0.6294, 0.775 ] } } }, "ratings": { "acoustic_plausibility": { "human_eligible_samples": 43, "judges": { "gemini-3.8-flash": { "eligible_samples": 41, "mae": 0.7886, "bias": 0.0691, "spearman": null, "pearson": null, "correlation_defined": false, "ai_distinct_values": 1, "ai_modal_value": 3, "ai_modal_share": 1.0, "ai_variance": 0, "mae_threshold_met": true, "discrimination_demonstrated": false, "discrimination_basis": "All three judges answered 3 on every eligible clip; the AI output has no variance, so correlation is undefined.", "summary_label": "MAE criterion met · useful discrimination not demonstrated", "ai_na_where_humans_rated": 2, "ai_numeric_where_humans_did_not": 16 }, "gemini-3.7-flash": { "eligible_samples": 42, "mae": 0.7798, "bias": 0.0774, "spearman": null, "pearson": null, "correlation_defined": false, "ai_distinct_values": 1, "ai_modal_value": 3, "ai_modal_share": 1.0, "ai_variance": 0, "mae_threshold_met": true, "discrimination_demonstrated": false, "discrimination_basis": "All three judges answered 3 on every eligible clip; the AI output has no variance, so correlation is undefined.", "summary_label": "MAE criterion met · useful discrimination not demonstrated", "ai_na_where_humans_rated": 1, "ai_numeric_where_humans_did_not": 18 }, "gemini-3.6-flash": { "eligible_samples": 35, "mae": 0.719, "bias": -0.081, "spearman": null, "pearson": null, "correlation_defined": false, "ai_distinct_values": 1, "ai_modal_value": 3, "ai_modal_share": 1.0, "ai_variance": 0, "mae_threshold_met": true, "discrimination_demonstrated": false, "discrimination_basis": "All three judges answered 3 on every eligible clip; the AI output has no variance, so correlation is undefined.", "summary_label": "MAE criterion met · useful discrimination not demonstrated", "ai_na_where_humans_rated": 8, "ai_numeric_where_humans_did_not": 9 } } }, "temporal_causal_coherence": { "human_eligible_samples": 43, "judges": { "gemini-3.8-flash": { "eligible_samples": 41, "mae": 0.6159, "bias": -0.1159, "spearman": 0.4841, "pearson": 0.4842, "correlation_defined": true, "ai_distinct_values": 3, "ai_modal_value": 3, "ai_modal_share": 0.6098, "ai_variance": 0.3879, "mae_threshold_met": true, "discrimination_demonstrated": false, "discrimination_basis": "AI ratings are strongly concentrated around 3.", "summary_label": "MAE criterion met · useful discrimination not demonstrated", "ai_na_where_humans_rated": 2, "ai_numeric_where_humans_did_not": 16 }, "gemini-3.7-flash": { "eligible_samples": 42, "mae": 0.7183, "bias": 0.1587, "spearman": 0.2317, "pearson": 0.2523, "correlation_defined": true, "ai_distinct_values": 3, "ai_modal_value": 3, "ai_modal_share": 0.5, "ai_variance": 0.4042, "mae_threshold_met": true, "discrimination_demonstrated": false, "discrimination_basis": "AI ratings are strongly concentrated around 3.", "summary_label": "MAE criterion met · useful discrimination not demonstrated", "ai_na_where_humans_rated": 1, "ai_numeric_where_humans_did_not": 18 }, "gemini-3.6-flash": { "eligible_samples": 35, "mae": 0.7452, "bias": -0.1405, "spearman": 0.2322, "pearson": 0.2048, "correlation_defined": true, "ai_distinct_values": 3, "ai_modal_value": 3, "ai_modal_share": 0.8, "ai_variance": 0.1927, "mae_threshold_met": true, "discrimination_demonstrated": false, "discrimination_basis": "AI ratings are strongly concentrated around 3.", "summary_label": "MAE criterion met · useful discrimination not demonstrated", "ai_na_where_humans_rated": 8, "ai_numeric_where_humans_did_not": 9 } } }, "physical_social_plausibility": { "human_eligible_samples": 50, "judges": { "gemini-3.8-flash": { "eligible_samples": 50, "mae": 0.5183, "bias": -0.195, "spearman": 0.2012, "pearson": 0.2838, "correlation_defined": true, "ai_distinct_values": 3, "ai_modal_value": 3, "ai_modal_share": 0.88, "ai_variance": 0.12, "mae_threshold_met": true, "discrimination_demonstrated": false, "discrimination_basis": "AI ratings are strongly concentrated around 3.", "summary_label": "MAE criterion met · useful discrimination not demonstrated", "ai_na_where_humans_rated": 0, "ai_numeric_where_humans_did_not": 13 }, "gemini-3.7-flash": { "eligible_samples": 50, "mae": 0.5317, "bias": -0.075, "spearman": 0.137, "pearson": 0.1953, "correlation_defined": true, "ai_distinct_values": 3, "ai_modal_value": 3, "ai_modal_share": 0.8, "ai_variance": 0.1856, "mae_threshold_met": true, "discrimination_demonstrated": false, "discrimination_basis": "AI ratings are strongly concentrated around 3.", "summary_label": "MAE criterion met · useful discrimination not demonstrated", "ai_na_where_humans_rated": 0, "ai_numeric_where_humans_did_not": 13 }, "gemini-3.6-flash": { "eligible_samples": 50, "mae": 0.525, "bias": -0.135, "spearman": 0.1258, "pearson": 0.1174, "correlation_defined": true, "ai_distinct_values": 3, "ai_modal_value": 3, "ai_modal_share": 0.96, "ai_variance": 0.0964, "mae_threshold_met": true, "discrimination_demonstrated": false, "discrimination_basis": "AI ratings are strongly concentrated around 3.", "summary_label": "MAE criterion met · useful discrimination not demonstrated", "ai_na_where_humans_rated": 0, "ai_numeric_where_humans_did_not": 13 } } } }, "headline": { "gemini-3.8-flash": { "ordinary_gates_calibrated": 6, "ordinary_gates": 8, "primary_causal_gates_calibrated": 0, "primary_causal_gates": 4, "not_calibrated_for_primary_causal_gate": [ "FB-014/event_connected_to_lift", "FB-074/creak_and_fart_distinguishable", "FB-512/recovered_without_further_emission", "FB-C02/source_was_red_jacket_woman" ], "not_calibrated": [ "FB-074/chair_creak_present", "FB-C02/single_event_present" ], "not_evaluable": [], "rating_mae_threshold_met": 3, "rating_discrimination_demonstrated": 0 }, "gemini-3.7-flash": { "ordinary_gates_calibrated": 4, "ordinary_gates": 8, "primary_causal_gates_calibrated": 0, "primary_causal_gates": 4, "not_calibrated_for_primary_causal_gate": [ "FB-014/event_connected_to_lift", "FB-074/creak_and_fart_distinguishable", "FB-512/recovered_without_further_emission", "FB-C02/source_was_red_jacket_woman" ], "not_calibrated": [ "FB-014/single_event_present", "FB-074/chair_creak_present", "FB-074/separate_fart_event_present", "FB-C02/single_event_present" ], "not_evaluable": [], "rating_mae_threshold_met": 3, "rating_discrimination_demonstrated": 0 }, "gemini-3.6-flash": { "ordinary_gates_calibrated": 5, "ordinary_gates": 8, "primary_causal_gates_calibrated": 2, "primary_causal_gates": 4, "not_calibrated_for_primary_causal_gate": [ "FB-014/event_connected_to_lift", "FB-C02/source_was_red_jacket_woman" ], "not_calibrated": [ "FB-014/single_event_present", "FB-074/chair_creak_present", "FB-512/both_people_reacted" ], "not_evaluable": [], "rating_mae_threshold_met": 3, "rating_discrimination_demonstrated": 0 }, "panel": { "ordinary_gates_calibrated": 4, "ordinary_gates": 8, "primary_causal_gates_calibrated": 0, "primary_causal_gates": 4, "not_calibrated_for_primary_causal_gate": [ "FB-014/event_connected_to_lift", "FB-074/creak_and_fart_distinguishable", "FB-512/recovered_without_further_emission", "FB-C02/source_was_red_jacket_woman" ], "not_calibrated": [ "FB-014/single_event_present", "FB-074/chair_creak_present", "FB-074/separate_fart_event_present", "FB-C02/single_event_present" ], "not_evaluable": [], "rating_mae_threshold_met": null, "rating_discrimination_demonstrated": null } }, "disagreements": { "human_consensus_any_judge_disagrees": [ { "sample_id": "3EYPKH", "question_id": "FB-512/single_event_present" }, { "sample_id": "3EYPKH", "question_id": "FB-512/both_people_reacted" }, { "sample_id": "3EYPKH", "question_id": "FB-512/recovered_without_further_emission" }, { "sample_id": "3G6SS5", "question_id": "FB-C02/single_event_present" }, { "sample_id": "3G6SS5", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "3MNRCL", "question_id": "FB-074/chair_creak_present" }, { "sample_id": "3MNRCL", "question_id": "FB-074/creak_and_fart_distinguishable" }, { "sample_id": "5JSTTP", "question_id": "FB-C02/bystanders_directed_attention" }, { "sample_id": "6D7MWZ", "question_id": "FB-C02/single_event_present" }, { "sample_id": "6D7MWZ", "question_id": "FB-C02/bystanders_directed_attention" }, { "sample_id": "6RR2XF", "question_id": "FB-512/single_event_present" }, { "sample_id": "6RR2XF", "question_id": "FB-512/both_people_reacted" }, { "sample_id": "6RR2XF", "question_id": "FB-512/recovered_without_further_emission" }, { "sample_id": "BABQP3", "question_id": "FB-C02/single_event_present" }, { "sample_id": "BABQP3", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "CS9BVJ", "question_id": "FB-512/single_event_present" }, { "sample_id": "CS9BVJ", "question_id": "FB-512/both_people_reacted" }, { "sample_id": "D6JHRC", "question_id": "FB-014/single_event_present" }, { "sample_id": "D6JHRC", "question_id": "FB-014/event_connected_to_lift" }, { "sample_id": "D8P3PH", "question_id": "FB-014/single_event_present" }, { "sample_id": "DWPP3W", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "EFXRPX", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "F9ZVRK", "question_id": "FB-C02/single_event_present" }, { "sample_id": "F9ZVRK", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "GDGAFY", "question_id": "FB-074/chair_creak_present" }, { "sample_id": "GDGAFY", "question_id": "FB-074/creak_and_fart_distinguishable" }, { "sample_id": "GQZK3X", "question_id": "FB-512/single_event_present" }, { "sample_id": "GQZK3X", "question_id": "FB-512/recovered_without_further_emission" }, { "sample_id": "HP3MTU", "question_id": "FB-074/separate_fart_event_present" }, { "sample_id": "HP3MTU", "question_id": "FB-074/creak_and_fart_distinguishable" }, { "sample_id": "J594DG", "question_id": "FB-014/single_event_present" }, { "sample_id": "J594DG", "question_id": "FB-014/event_connected_to_lift" }, { "sample_id": "J8X46G", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "JGL4LW", "question_id": "FB-074/chair_creak_present" }, { "sample_id": "JGL4LW", "question_id": "FB-074/separate_fart_event_present" }, { "sample_id": "LV5G5T", "question_id": "FB-074/chair_creak_present" }, { "sample_id": "LV5G5T", "question_id": "FB-074/separate_fart_event_present" }, { "sample_id": "LV5G5T", "question_id": "FB-074/creak_and_fart_distinguishable" }, { "sample_id": "LV9JB2", "question_id": "FB-074/chair_creak_present" }, { "sample_id": "LV9JB2", "question_id": "FB-074/creak_and_fart_distinguishable" }, { "sample_id": "P94D96", "question_id": "FB-014/single_event_present" }, { "sample_id": "P94D96", "question_id": "FB-014/event_connected_to_lift" }, { "sample_id": "R26HLK", "question_id": "FB-074/chair_creak_present" }, { "sample_id": "RMW286", "question_id": "FB-C02/single_event_present" }, { "sample_id": "RMW286", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "RMW286", "question_id": "FB-C02/bystanders_directed_attention" }, { "sample_id": "RXTC7G", "question_id": "FB-074/chair_creak_present" }, { "sample_id": "RXTC7G", "question_id": "FB-074/creak_and_fart_distinguishable" }, { "sample_id": "SXAZFW", "question_id": "FB-C02/single_event_present" }, { "sample_id": "SXAZFW", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "SXAZFW", "question_id": "FB-C02/bystanders_directed_attention" }, { "sample_id": "V65YL7", "question_id": "FB-074/chair_creak_present" }, { "sample_id": "V65YL7", "question_id": "FB-074/separate_fart_event_present" }, { "sample_id": "V65YL7", "question_id": "FB-074/creak_and_fart_distinguishable" }, { "sample_id": "VUCT7K", "question_id": "FB-512/single_event_present" }, { "sample_id": "WCYF6F", "question_id": "FB-512/single_event_present" }, { "sample_id": "WCYF6F", "question_id": "FB-512/both_people_reacted" }, { "sample_id": "WCYF6F", "question_id": "FB-512/recovered_without_further_emission" }, { "sample_id": "WJHT3C", "question_id": "FB-014/single_event_present" }, { "sample_id": "WJHT3C", "question_id": "FB-014/event_connected_to_lift" }, { "sample_id": "XND6TL", "question_id": "FB-014/single_event_present" }, { "sample_id": "XND6TL", "question_id": "FB-014/event_connected_to_lift" }, { "sample_id": "YJCLF4", "question_id": "FB-014/single_event_present" }, { "sample_id": "YJCLF4", "question_id": "FB-014/event_connected_to_lift" }, { "sample_id": "YMLKA7", "question_id": "FB-512/single_event_present" }, { "sample_id": "YMLKA7", "question_id": "FB-512/both_people_reacted" }, { "sample_id": "YUNREY", "question_id": "FB-014/single_event_present" }, { "sample_id": "YUNREY", "question_id": "FB-014/event_connected_to_lift" }, { "sample_id": "YUTRHH", "question_id": "FB-014/single_event_present" }, { "sample_id": "YUTRHH", "question_id": "FB-014/event_connected_to_lift" }, { "sample_id": "ZB6E8W", "question_id": "FB-512/single_event_present" }, { "sample_id": "ZB6E8W", "question_id": "FB-512/recovered_without_further_emission" } ], "human_consensus_most_judges_disagree": [ { "sample_id": "3EYPKH", "question_id": "FB-512/recovered_without_further_emission" }, { "sample_id": "3G6SS5", "question_id": "FB-C02/single_event_present" }, { "sample_id": "3G6SS5", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "5JSTTP", "question_id": "FB-C02/bystanders_directed_attention" }, { "sample_id": "6D7MWZ", "question_id": "FB-C02/single_event_present" }, { "sample_id": "BABQP3", "question_id": "FB-C02/single_event_present" }, { "sample_id": "BABQP3", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "CS9BVJ", "question_id": "FB-512/single_event_present" }, { "sample_id": "D6JHRC", "question_id": "FB-014/event_connected_to_lift" }, { "sample_id": "D8P3PH", "question_id": "FB-014/single_event_present" }, { "sample_id": "F9ZVRK", "question_id": "FB-C02/single_event_present" }, { "sample_id": "F9ZVRK", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "GQZK3X", "question_id": "FB-512/recovered_without_further_emission" }, { "sample_id": "HP3MTU", "question_id": "FB-074/separate_fart_event_present" }, { "sample_id": "HP3MTU", "question_id": "FB-074/creak_and_fart_distinguishable" }, { "sample_id": "J594DG", "question_id": "FB-014/single_event_present" }, { "sample_id": "J594DG", "question_id": "FB-014/event_connected_to_lift" }, { "sample_id": "J8X46G", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "JGL4LW", "question_id": "FB-074/separate_fart_event_present" }, { "sample_id": "LV5G5T", "question_id": "FB-074/separate_fart_event_present" }, { "sample_id": "LV9JB2", "question_id": "FB-074/chair_creak_present" }, { "sample_id": "LV9JB2", "question_id": "FB-074/creak_and_fart_distinguishable" }, { "sample_id": "P94D96", "question_id": "FB-014/single_event_present" }, { "sample_id": "P94D96", "question_id": "FB-014/event_connected_to_lift" }, { "sample_id": "R26HLK", "question_id": "FB-074/chair_creak_present" }, { "sample_id": "SXAZFW", "question_id": "FB-C02/single_event_present" }, { "sample_id": "SXAZFW", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "SXAZFW", "question_id": "FB-C02/bystanders_directed_attention" }, { "sample_id": "V65YL7", "question_id": "FB-074/chair_creak_present" }, { "sample_id": "V65YL7", "question_id": "FB-074/separate_fart_event_present" }, { "sample_id": "V65YL7", "question_id": "FB-074/creak_and_fart_distinguishable" }, { "sample_id": "WCYF6F", "question_id": "FB-512/single_event_present" }, { "sample_id": "WCYF6F", "question_id": "FB-512/both_people_reacted" }, { "sample_id": "WCYF6F", "question_id": "FB-512/recovered_without_further_emission" }, { "sample_id": "WJHT3C", "question_id": "FB-014/single_event_present" }, { "sample_id": "WJHT3C", "question_id": "FB-014/event_connected_to_lift" }, { "sample_id": "XND6TL", "question_id": "FB-014/event_connected_to_lift" }, { "sample_id": "YMLKA7", "question_id": "FB-512/both_people_reacted" }, { "sample_id": "YUNREY", "question_id": "FB-014/single_event_present" }, { "sample_id": "YUTRHH", "question_id": "FB-014/single_event_present" }, { "sample_id": "YUTRHH", "question_id": "FB-014/event_connected_to_lift" }, { "sample_id": "ZB6E8W", "question_id": "FB-512/single_event_present" }, { "sample_id": "ZB6E8W", "question_id": "FB-512/recovered_without_further_emission" } ], "human_consensus_all_judges_disagree": [ { "sample_id": "3EYPKH", "question_id": "FB-512/recovered_without_further_emission" }, { "sample_id": "3G6SS5", "question_id": "FB-C02/single_event_present" }, { "sample_id": "3G6SS5", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "5JSTTP", "question_id": "FB-C02/bystanders_directed_attention" }, { "sample_id": "6D7MWZ", "question_id": "FB-C02/single_event_present" }, { "sample_id": "BABQP3", "question_id": "FB-C02/single_event_present" }, { "sample_id": "BABQP3", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "CS9BVJ", "question_id": "FB-512/single_event_present" }, { "sample_id": "F9ZVRK", "question_id": "FB-C02/single_event_present" }, { "sample_id": "F9ZVRK", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "HP3MTU", "question_id": "FB-074/separate_fart_event_present" }, { "sample_id": "J8X46G", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "R26HLK", "question_id": "FB-074/chair_creak_present" }, { "sample_id": "V65YL7", "question_id": "FB-074/chair_creak_present" }, { "sample_id": "V65YL7", "question_id": "FB-074/separate_fart_event_present" }, { "sample_id": "V65YL7", "question_id": "FB-074/creak_and_fart_distinguishable" }, { "sample_id": "WJHT3C", "question_id": "FB-014/single_event_present" }, { "sample_id": "WJHT3C", "question_id": "FB-014/event_connected_to_lift" }, { "sample_id": "XND6TL", "question_id": "FB-014/event_connected_to_lift" }, { "sample_id": "YMLKA7", "question_id": "FB-512/both_people_reacted" } ], "ai_unanimous_humans_without_consensus": [ { "sample_id": "3G6SS5", "question_id": "FB-C02/bystanders_directed_attention" }, { "sample_id": "3Z7JPK", "question_id": "FB-C02/single_event_present" }, { "sample_id": "3Z7JPK", "question_id": "FB-C02/bystanders_directed_attention" }, { "sample_id": "6D7MWZ", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "7FY9MG", "question_id": "FB-074/chair_creak_present" }, { "sample_id": "7FY9MG", "question_id": "FB-074/creak_and_fart_distinguishable" }, { "sample_id": "7K724T", "question_id": "FB-074/creak_and_fart_distinguishable" }, { "sample_id": "7XGJH6", "question_id": "FB-074/chair_creak_present" }, { "sample_id": "7XGJH6", "question_id": "FB-074/creak_and_fart_distinguishable" }, { "sample_id": "8SJDU5", "question_id": "FB-014/event_connected_to_lift" }, { "sample_id": "BABQP3", "question_id": "FB-C02/bystanders_directed_attention" }, { "sample_id": "D6JHRC", "question_id": "FB-014/body_and_clothing_plausible" }, { "sample_id": "F9ZVRK", "question_id": "FB-C02/bystanders_directed_attention" }, { "sample_id": "GQZK3X", "question_id": "FB-512/both_people_reacted" }, { "sample_id": "KBZ29T", "question_id": "FB-512/both_people_reacted" }, { "sample_id": "KBZ29T", "question_id": "FB-512/recovered_without_further_emission" }, { "sample_id": "MH6X9Z", "question_id": "FB-074/separate_fart_event_present" }, { "sample_id": "PN3WRY", "question_id": "FB-512/single_event_present" }, { "sample_id": "R26HLK", "question_id": "FB-074/separate_fart_event_present" }, { "sample_id": "R26HLK", "question_id": "FB-074/creak_and_fart_distinguishable" }, { "sample_id": "YJCLF4", "question_id": "FB-014/body_and_clothing_plausible" } ], "ai_unanimous_against_human_consensus": [ { "sample_id": "3G6SS5", "question_id": "FB-C02/single_event_present" }, { "sample_id": "3G6SS5", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "6D7MWZ", "question_id": "FB-C02/single_event_present" }, { "sample_id": "BABQP3", "question_id": "FB-C02/single_event_present" }, { "sample_id": "BABQP3", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "CS9BVJ", "question_id": "FB-512/single_event_present" }, { "sample_id": "F9ZVRK", "question_id": "FB-C02/single_event_present" }, { "sample_id": "HP3MTU", "question_id": "FB-074/separate_fart_event_present" }, { "sample_id": "J8X46G", "question_id": "FB-C02/source_was_red_jacket_woman" }, { "sample_id": "R26HLK", "question_id": "FB-074/chair_creak_present" }, { "sample_id": "V65YL7", "question_id": "FB-074/chair_creak_present" }, { "sample_id": "V65YL7", "question_id": "FB-074/separate_fart_event_present" }, { "sample_id": "V65YL7", "question_id": "FB-074/creak_and_fart_distinguishable" }, { "sample_id": "WJHT3C", "question_id": "FB-014/single_event_present" }, { "sample_id": "WJHT3C", "question_id": "FB-014/event_connected_to_lift" }, { "sample_id": "XND6TL", "question_id": "FB-014/event_connected_to_lift" }, { "sample_id": "YMLKA7", "question_id": "FB-512/both_people_reacted" } ] } }, "notes": [] }