{
  "source_file": "satcove-consensus-benchmark-2026-10-22.xlsx",
  "source_sha256": "14c33d4b5730b70f0e1c0f7c17caa2bfe9084ce1f761b45d40db027d12080ced",
  "view": "primary protocol only: letters of 'Données brutes' minus the 654 recovered cells (recovered_cells.csv)",
  "note": "Jury size = number of models with a valid primary-protocol letter for that question; questions with fewer than 4 are excluded.",
  "recovered_cells_removed": 654,
  "overall": {
    "questions": 2460,
    "ties": 189,
    "majorities": 2271,
    "majority_correct": 1741,
    "majority_accuracy_pct": 76.7,
    "majority_ci95": [
      74.9,
      78.4
    ],
    "majority_accuracy_ties_as_failures_pct": 70.8,
    "unanimous": 1240,
    "unanimous_wrong": 86,
    "unanimous_wrong_pct": 6.9,
    "unanimous_wrong_ci95": [
      5.7,
      8.5
    ]
  },
  "by_jury_size": {
    "4": {
      "questions": 407,
      "ties": 84,
      "majorities": 323,
      "majority_correct": 147,
      "majority_accuracy_pct": 45.5,
      "majority_ci95": [
        40.2,
        51.0
      ],
      "majority_accuracy_ties_as_failures_pct": 36.1,
      "unanimous": 54,
      "unanimous_wrong": 12,
      "unanimous_wrong_pct": 22.2,
      "unanimous_wrong_ci95": [
        13.2,
        34.9
      ]
    },
    "5": {
      "questions": 1689,
      "ties": 84,
      "majorities": 1605,
      "majority_correct": 1295,
      "majority_accuracy_pct": 80.7,
      "majority_ci95": [
        78.7,
        82.5
      ],
      "majority_accuracy_ties_as_failures_pct": 76.7,
      "unanimous": 959,
      "unanimous_wrong": 65,
      "unanimous_wrong_pct": 6.8,
      "unanimous_wrong_ci95": [
        5.4,
        8.5
      ]
    },
    "6": {
      "questions": 364,
      "ties": 21,
      "majorities": 343,
      "majority_correct": 299,
      "majority_accuracy_pct": 87.2,
      "majority_ci95": [
        83.2,
        90.3
      ],
      "majority_accuracy_ties_as_failures_pct": 82.1,
      "unanimous": 227,
      "unanimous_wrong": 9,
      "unanimous_wrong_pct": 4.0,
      "unanimous_wrong_ci95": [
        2.1,
        7.4
      ]
    }
  },
  "false_consensus_by_jury_size": {
    "4": {
      "unanimous": 54,
      "unanimous_wrong": 12
    },
    "5": {
      "unanimous": 959,
      "unanimous_wrong": 65
    },
    "6": {
      "unanimous": 227,
      "unanimous_wrong": 9
    }
  },
  "model_scores_primary_only_own_footprint": {
    "Claude": {
      "answered": 2025,
      "correct": 1645,
      "accuracy_pct": 81.2,
      "ci95": [
        79.5,
        82.9
      ]
    },
    "GPT": {
      "answered": 2500,
      "correct": 1595,
      "accuracy_pct": 63.8,
      "ci95": [
        61.9,
        65.7
      ]
    },
    "Gemini": {
      "answered": 2414,
      "correct": 1714,
      "accuracy_pct": 71.0,
      "ci95": [
        69.2,
        72.8
      ]
    },
    "Mistral": {
      "answered": 2492,
      "correct": 1682,
      "accuracy_pct": 67.5,
      "ci95": [
        65.6,
        69.3
      ]
    },
    "Perplexity": {
      "answered": 447,
      "correct": 326,
      "accuracy_pct": 72.9,
      "ci95": [
        68.6,
        76.8
      ]
    },
    "Grok": {
      "answered": 2499,
      "correct": 1591,
      "accuracy_pct": 63.7,
      "ci95": [
        61.8,
        65.5
      ]
    }
  },
  "perplexity_533": {
    "questions_perplexity_asked": 533,
    "model_scores_primary_only_on_533": {
      "Claude": {
        "answered": 438,
        "correct": 365,
        "accuracy_pct": 83.3,
        "ci95": [
          79.6,
          86.5
        ]
      },
      "GPT": {
        "answered": 533,
        "correct": 341,
        "accuracy_pct": 64.0,
        "ci95": [
          59.8,
          67.9
        ]
      },
      "Gemini": {
        "answered": 517,
        "correct": 373,
        "accuracy_pct": 72.1,
        "ci95": [
          68.1,
          75.8
        ]
      },
      "Mistral": {
        "answered": 533,
        "correct": 366,
        "accuracy_pct": 68.7,
        "ci95": [
          64.6,
          72.5
        ]
      },
      "Perplexity": {
        "answered": 447,
        "correct": 326,
        "accuracy_pct": 72.9,
        "ci95": [
          68.6,
          76.8
        ]
      },
      "Grok": {
        "answered": 532,
        "correct": 353,
        "accuracy_pct": 66.4,
        "ci95": [
          62.2,
          70.2
        ]
      }
    },
    "questions_where_all_six_have_primary_letter": 364,
    "model_scores_on_those": {
      "Claude": {
        "answered": 364,
        "correct": 312,
        "accuracy_pct": 85.7,
        "ci95": [
          81.7,
          88.9
        ]
      },
      "GPT": {
        "answered": 364,
        "correct": 279,
        "accuracy_pct": 76.6,
        "ci95": [
          72.0,
          80.7
        ]
      },
      "Gemini": {
        "answered": 364,
        "correct": 303,
        "accuracy_pct": 83.2,
        "ci95": [
          79.1,
          86.7
        ]
      },
      "Mistral": {
        "answered": 364,
        "correct": 281,
        "accuracy_pct": 77.2,
        "ci95": [
          72.6,
          81.2
        ]
      },
      "Perplexity": {
        "answered": 364,
        "correct": 293,
        "accuracy_pct": 80.5,
        "ci95": [
          76.1,
          84.2
        ]
      },
      "Grok": {
        "answered": 364,
        "correct": 276,
        "accuracy_pct": 75.8,
        "ci95": [
          71.2,
          79.9
        ]
      }
    },
    "jury6_metrics": {
      "questions": 364,
      "ties": 21,
      "majorities": 343,
      "majority_correct": 299,
      "majority_accuracy_pct": 87.2,
      "majority_ci95": [
        83.2,
        90.3
      ],
      "majority_accuracy_ties_as_failures_pct": 82.1,
      "unanimous": 227,
      "unanimous_wrong": 9,
      "unanimous_wrong_pct": 4.0,
      "unanimous_wrong_ci95": [
        2.1,
        7.4
      ]
    }
  },
  "final_view_for_comparison": {
    "questions": 2500,
    "ties": 197,
    "majorities": 2303,
    "majority_correct": 1775,
    "majority_accuracy_pct": 77.1,
    "majority_ci95": [
      75.3,
      78.7
    ],
    "majority_accuracy_ties_as_failures_pct": 71.0,
    "unanimous": 1221,
    "unanimous_wrong": 73,
    "unanimous_wrong_pct": 6.0,
    "unanimous_wrong_ci95": [
      4.8,
      7.5
    ]
  }
}