{
  "title": "Published AI hallucination benchmarks, compiled from the original sources",
  "note": "Satcove does not run these benchmarks. Figures are copied from each source on the check date; verify at the source.",
  "version": "1.0",
  "license": "CC BY 4.0",
  "checked": "2026-10-15",
  "citation": "Satcove (2026). AI hallucination benchmarks compiled, version 1.0. https://satcove.com/ai-hallucination-benchmarks",
  "benchmarks": [
    {
      "name": "Vectara Hallucination Leaderboard",
      "sourceUrl": "https://github.com/vectara/hallucination-leaderboard",
      "sourceUpdated": "2026-09-22",
      "evaluator": "HHEM-2.3 (Vectara's hallucination evaluation model)",
      "measures": "How often a model adds information that is not in a document when asked to summarise it. It tests faithfulness to a supplied text, not general knowledge.",
      "dataset": "More than 7,700 articles across news, technology, science, medicine, legal, sports, business and education; the dataset is not public.",
      "license": "Apache-2.0 (repository); no licence is stated for the dataset.",
      "rows": [
        {
          "model": "antgroup/finix_s1_32b",
          "hallucination": 1.8,
          "consistency": 98.2,
          "answerRate": 99.5,
          "avgWords": 172.4
        },
        {
          "model": "openai/gpt-5.4-nano-2026-03-17",
          "hallucination": 3.1,
          "consistency": 96.9,
          "answerRate": 100,
          "avgWords": 144.4
        },
        {
          "model": "google/gemini-2.5-flash-lite",
          "hallucination": 3.3,
          "consistency": 96.7,
          "answerRate": 99.5,
          "avgWords": 95.7
        },
        {
          "model": "microsoft/Phi-4",
          "hallucination": 3.7,
          "consistency": 96.3,
          "answerRate": 80.7,
          "avgWords": 120.9
        },
        {
          "model": "meta-llama/Llama-3.3-70B-Instruct-Turbo",
          "hallucination": 4.1,
          "consistency": 95.9,
          "answerRate": 99.5,
          "avgWords": 64.6
        },
        {
          "model": "snowflake/snowflake-arctic-instruct",
          "hallucination": 4.3,
          "consistency": 95.7,
          "answerRate": 62.7,
          "avgWords": 81.4
        },
        {
          "model": "google/gemma-3-12b-it",
          "hallucination": 4.4,
          "consistency": 95.6,
          "answerRate": 97.4,
          "avgWords": 89.7
        },
        {
          "model": "mistralai/mistral-large-2411",
          "hallucination": 4.5,
          "consistency": 95.5,
          "answerRate": 99.9,
          "avgWords": 85
        },
        {
          "model": "qwen/qwen3-8b",
          "hallucination": 4.8,
          "consistency": 95.2,
          "answerRate": 99.9,
          "avgWords": 83.6
        },
        {
          "model": "amazon/nova-pro-v1:0",
          "hallucination": 5.1,
          "consistency": 94.9,
          "answerRate": 99.3,
          "avgWords": 66.2
        },
        {
          "model": "amazon/nova-2-lite-v1:0",
          "hallucination": 5.1,
          "consistency": 94.9,
          "answerRate": 99.6,
          "avgWords": 94.1
        },
        {
          "model": "mistralai/mistral-small-2501",
          "hallucination": 5.1,
          "consistency": 94.9,
          "answerRate": 97.9,
          "avgWords": 98.8
        },
        {
          "model": "ibm-granite/granite-4.0-h-small",
          "hallucination": 5.2,
          "consistency": 94.8,
          "answerRate": 100,
          "avgWords": 107.4
        },
        {
          "model": "google/gemma-4-26b-a4b-it",
          "hallucination": 5.2,
          "consistency": 94.8,
          "answerRate": 99.8,
          "avgWords": 67.1
        },
        {
          "model": "ai21labs/jamba-mini-2",
          "hallucination": 5.3,
          "consistency": 94.7,
          "answerRate": 99.6,
          "avgWords": 109.4
        }
      ]
    },
    {
      "name": "AA-Omniscience (Artificial Analysis)",
      "sourceUrl": "https://artificialanalysis.ai/evaluations/omniscience",
      "paperUrl": "https://arxiv.org/abs/2511.13029",
      "sourceUpdated": "not stated on the page",
      "measures": "General knowledge reliability. It rewards correct answers, penalises wrong answers and does not penalise refusals. The index runs from -100 to 100.",
      "definitions": [
        "Index: higher is better. Rewards correct answers, penalises hallucinations, does not penalise abstaining.",
        "Accuracy: higher is better. Correct answers divided by all questions, whether or not the model chose to answer.",
        "Hallucination rate: lower is better. Incorrect answers divided by the sum of incorrect, partial and not-attempted responses."
      ],
      "top": [
        {
          "metric": "Omniscience Index",
          "rank": 1,
          "model": "Claude Opus 5.5 (Max, Default Fallback)",
          "score": "46"
        },
        {
          "metric": "Omniscience Index",
          "rank": 2,
          "model": "GPT-6 Astra (High)",
          "score": "44"
        },
        {
          "metric": "Omniscience Index",
          "rank": 3,
          "model": "Claude Fable 5.1 (Max, Default Fallback)",
          "score": "43"
        },
        {
          "metric": "Accuracy",
          "rank": 1,
          "model": "Claude Fable 5.1 (Max, Default Fallback)",
          "score": "67%"
        },
        {
          "metric": "Accuracy",
          "rank": 2,
          "model": "Claude Opus 5.5 (Max, Default Fallback)",
          "score": "66%"
        },
        {
          "metric": "Accuracy",
          "rank": 3,
          "model": "Claude Fable 5.1 (Xhigh, Default Fallback)",
          "score": "66%"
        },
        {
          "metric": "Hallucination rate (lower is better)",
          "rank": 1,
          "model": "MiniCPM5-1B (Non-reasoning)",
          "score": "1%"
        },
        {
          "metric": "Hallucination rate (lower is better)",
          "rank": 2,
          "model": "G9v3-3B",
          "score": "12%"
        },
        {
          "metric": "Hallucination rate (lower is better)",
          "rank": 3,
          "model": "G9v3-39A5B",
          "score": "13%"
        }
      ]
    }
  ],
  "vectaraByProvider": {
    "totalModelsOnLeaderboard": 108,
    "providers": [
      {
        "provider": "Anthropic (Claude)",
        "rows": [
          {
            "model": "anthropic/claude-haiku-4-5-20251001",
            "hallucination": 9.8,
            "consistency": 90.2,
            "answerRate": 99.5
          },
          {
            "model": "anthropic/claude-sonnet-4-20250514",
            "hallucination": 10.3,
            "consistency": 89.7,
            "answerRate": 98.6
          },
          {
            "model": "anthropic/claude-sonnet-4-6",
            "hallucination": 10.6,
            "consistency": 89.4,
            "answerRate": 99.9
          },
          {
            "model": "anthropic/claude-opus-4-5-20251101",
            "hallucination": 10.9,
            "consistency": 89.1,
            "answerRate": 98.7
          },
          {
            "model": "anthropic/claude-opus-4-1-20250805",
            "hallucination": 11.8,
            "consistency": 88.2,
            "answerRate": 92.4
          },
          {
            "model": "anthropic/claude-opus-4-20250514",
            "hallucination": 12,
            "consistency": 88,
            "answerRate": 91
          },
          {
            "model": "anthropic/claude-sonnet-4-5-20250929",
            "hallucination": 12,
            "consistency": 88,
            "answerRate": 95.6
          },
          {
            "model": "anthropic/claude-opus-4-7",
            "hallucination": 12,
            "consistency": 88,
            "answerRate": 98
          },
          {
            "model": "anthropic/claude-opus-4-6",
            "hallucination": 12.2,
            "consistency": 87.8,
            "answerRate": 99.8
          }
        ]
      },
      {
        "provider": "OpenAI (GPT)",
        "rows": [
          {
            "model": "openai/gpt-5.4-nano-2026-03-17",
            "hallucination": 3.1,
            "consistency": 96.9,
            "answerRate": 100
          },
          {
            "model": "openai/gpt-5.4-mini-2026-03-17",
            "hallucination": 5.5,
            "consistency": 94.5,
            "answerRate": 100
          },
          {
            "model": "openai/gpt-4.1-2025-04-14",
            "hallucination": 5.6,
            "consistency": 94.4,
            "answerRate": 99.9
          },
          {
            "model": "openai/gpt-6-sol",
            "hallucination": 6.5,
            "consistency": 93.5,
            "answerRate": 100
          },
          {
            "model": "openai/gpt-5.4-2026-03-05",
            "hallucination": 7,
            "consistency": 93,
            "answerRate": 99.9
          },
          {
            "model": "openai/gpt-5.4-pro-2026-03-05",
            "hallucination": 8.3,
            "consistency": 91.7,
            "answerRate": 100
          },
          {
            "model": "openai/gpt-5.2-low-2025-12-11",
            "hallucination": 8.4,
            "consistency": 91.6,
            "answerRate": 100
          },
          {
            "model": "openai/gpt-6-astra",
            "hallucination": 8.7,
            "consistency": 91.3,
            "answerRate": 100
          },
          {
            "model": "openai/gpt-5.5",
            "hallucination": 9.3,
            "consistency": 90.7,
            "answerRate": 100
          },
          {
            "model": "openai/gpt-4o-2024-08-06",
            "hallucination": 9.6,
            "consistency": 90.4,
            "answerRate": 93.8
          },
          {
            "model": "openai/gpt-5-nano-2025-08-07",
            "hallucination": 10.5,
            "consistency": 89.5,
            "answerRate": 100
          },
          {
            "model": "openai/gpt-5.2-high-2025-12-11",
            "hallucination": 10.8,
            "consistency": 89.2,
            "answerRate": 100
          },
          {
            "model": "openai/gpt-5.1-low-2025-11-13",
            "hallucination": 10.9,
            "consistency": 89.1,
            "answerRate": 100
          },
          {
            "model": "openai/gpt-5.1-high-2025-11-13",
            "hallucination": 12.1,
            "consistency": 87.9,
            "answerRate": 100
          },
          {
            "model": "openai/gpt-5.6-sol",
            "hallucination": 12.4,
            "consistency": 87.6,
            "answerRate": 99.1
          },
          {
            "model": "openai/gpt-5-mini-2025-08-07",
            "hallucination": 12.9,
            "consistency": 87.1,
            "answerRate": 99.9
          },
          {
            "model": "openai/gpt-oss-120b",
            "hallucination": 14.2,
            "consistency": 85.8,
            "answerRate": 99.9
          },
          {
            "model": "openai/gpt-5-minimal-2025-08-07",
            "hallucination": 14.7,
            "consistency": 85.3,
            "answerRate": 99.9
          },
          {
            "model": "openai/gpt-5-high-2025-08-07",
            "hallucination": 15.1,
            "consistency": 84.9,
            "answerRate": 99.9
          },
          {
            "model": "openai/o4-mini-low-2025-04-16",
            "hallucination": 18.6,
            "consistency": 81.4,
            "answerRate": 98.7
          },
          {
            "model": "openai/o4-mini-high-2025-04-16",
            "hallucination": 18.6,
            "consistency": 81.4,
            "answerRate": 99.2
          },
          {
            "model": "openai/o3-pro",
            "hallucination": 23.3,
            "consistency": 76.7,
            "answerRate": 100
          }
        ]
      },
      {
        "provider": "Google (Gemini)",
        "rows": [
          {
            "model": "google/gemini-2.5-flash-lite",
            "hallucination": 3.3,
            "consistency": 96.7,
            "answerRate": 99.5
          },
          {
            "model": "google/gemini-2.5-pro",
            "hallucination": 7,
            "consistency": 93,
            "answerRate": 99.1
          },
          {
            "model": "google/gemini-2.5-flash",
            "hallucination": 7.8,
            "consistency": 92.2,
            "answerRate": 99
          },
          {
            "model": "google/gemini-3.1-flash-lite-preview",
            "hallucination": 8.2,
            "consistency": 91.8,
            "answerRate": 99.6
          },
          {
            "model": "google/gemini-3.1-pro-preview",
            "hallucination": 10.4,
            "consistency": 89.6,
            "answerRate": 99.4
          },
          {
            "model": "google/gemini-3-flash-preview",
            "hallucination": 13.5,
            "consistency": 86.5,
            "answerRate": 99.8
          },
          {
            "model": "google/gemini-3-pro-preview",
            "hallucination": 13.6,
            "consistency": 86.4,
            "answerRate": 99.4
          }
        ]
      },
      {
        "provider": "xAI (Grok)",
        "rows": [
          {
            "model": "xai-org/grok-3",
            "hallucination": 5.8,
            "consistency": 94.2,
            "answerRate": 93
          },
          {
            "model": "xai-org/grok-4-1-fast-non-reasoning",
            "hallucination": 17.8,
            "consistency": 82.2,
            "answerRate": 98.5
          },
          {
            "model": "xai-org/grok-4-1-fast-reasoning",
            "hallucination": 19.2,
            "consistency": 80.8,
            "answerRate": 99.7
          },
          {
            "model": "xai-org/grok-4-fast-non-reasoning",
            "hallucination": 19.7,
            "consistency": 80.3,
            "answerRate": 99.2
          },
          {
            "model": "xai-org/grok-4-fast-reasoning",
            "hallucination": 20.2,
            "consistency": 79.8,
            "answerRate": 99.5
          }
        ]
      },
      {
        "provider": "Mistral",
        "rows": [
          {
            "model": "mistralai/mistral-large-2411",
            "hallucination": 4.5,
            "consistency": 95.5,
            "answerRate": 99.9
          },
          {
            "model": "mistralai/mistral-small-2501",
            "hallucination": 5.1,
            "consistency": 94.9,
            "answerRate": 97.9
          },
          {
            "model": "mistralai/ministral-3b-2410",
            "hallucination": 7.3,
            "consistency": 92.7,
            "answerRate": 99.9
          },
          {
            "model": "mistralai/ministral-8b-2410",
            "hallucination": 7.4,
            "consistency": 92.6,
            "answerRate": 99.9
          },
          {
            "model": "mistralai/mistral-3-large-2512",
            "hallucination": 14.5,
            "consistency": 85.5,
            "answerRate": 98.8
          },
          {
            "model": "mistralai/ministral-3-14b-2512",
            "hallucination": 19.4,
            "consistency": 80.6,
            "answerRate": 99.6
          },
          {
            "model": "mistralai/ministral-3-8b-2512",
            "hallucination": 21.7,
            "consistency": 78.3,
            "answerRate": 99.1
          },
          {
            "model": "mistralai/mistral-medium-2508",
            "hallucination": 22.7,
            "consistency": 77.3,
            "answerRate": 99.7
          },
          {
            "model": "mistralai/ministral-3-3b-2512",
            "hallucination": 24.2,
            "consistency": 75.8,
            "answerRate": 74.3
          }
        ]
      }
    ]
  },
  "notRead": [
    {
      "name": "FACTS Grounding (Google DeepMind / Kaggle)",
      "sourceUrl": "https://www.kaggle.com/benchmarks/google/facts-grounding",
      "note": "The leaderboard could not be read automatically on the check date, so no score is published here."
    }
  ]
}