{
  "$schema": "https://schema.org/Dataset",
  "name": "CodeReviewBench leaderboard",
  "description": "How well AI models find real bugs in code review, measured on merged pull requests from production open-source repositories.",
  "url": "https://codereviewbench.com/leaderboard",
  "license": "https://github.com/kodustech/codereviewbench",
  "measuredOn": {
    "pullRequests": 30,
    "confirmedBugs": 95,
    "repositories": [
      "cal.com",
      "discourse",
      "grafana",
      "keycloak",
      "sentry"
    ],
    "languages": [
      "go",
      "java",
      "python",
      "ruby",
      "typescript"
    ]
  },
  "generatedAt": "2026-08-31T23:21:14.517Z",
  "metricDefinitions": {
    "recall": "Share of human-reported bugs the model found, micro-averaged across all pull requests.",
    "precision": "Share of the model findings that matched a human-reported bug.",
    "f1": "Harmonic mean of recall and precision.",
    "costPerPR": "Measured API cost divided by number of pull requests reviewed. Null when the model ran on a subscription rather than metered API."
  },
  "models": [
    {
      "id": "deepseek-v4-pro",
      "name": "DeepSeek V4 Pro",
      "provider": "DeepSeek",
      "recall": 44.2,
      "precision": 43.6,
      "f1": 43.9,
      "bugsFound": 42,
      "bugsTotal": 95,
      "pullRequests": 30,
      "costPerPR": 0.3,
      "accessPath": "api",
      "url": "https://codereviewbench.com/model/deepseek-v4-pro"
    },
    {
      "id": "kimi-k2.7-code",
      "name": "Kimi K2.7 Code",
      "provider": "Moonshot",
      "recall": 37.9,
      "precision": 50,
      "f1": 43.1,
      "bugsFound": 36,
      "bugsTotal": 95,
      "pullRequests": 30,
      "costPerPR": 0.55,
      "accessPath": "api",
      "url": "https://codereviewbench.com/model/kimi-k2.7-code"
    },
    {
      "id": "qwen3.8-max",
      "name": "Qwen3.8 Max",
      "provider": "Alibaba",
      "recall": 41,
      "precision": 43.9,
      "f1": 42.4,
      "bugsFound": 39,
      "bugsTotal": 95,
      "pullRequests": 30,
      "costPerPR": 1.21,
      "accessPath": "api",
      "url": "https://codereviewbench.com/model/qwen3.8-max"
    },
    {
      "id": "glm-5.2@fireworks",
      "name": "GLM-5.2",
      "provider": "Zhipu",
      "recall": 35.2,
      "precision": 47,
      "f1": 40.2,
      "bugsFound": 32,
      "bugsTotal": 91,
      "pullRequests": 29,
      "costPerPR": 0.88,
      "accessPath": "api",
      "url": "https://codereviewbench.com/model/glm-5.2--fireworks"
    },
    {
      "id": "glm-5.3-flash",
      "name": "GLM-5.3 Flash",
      "provider": "Zhipu",
      "recall": 39,
      "precision": 41,
      "f1": 40,
      "bugsFound": 37,
      "bugsTotal": 95,
      "pullRequests": 30,
      "costPerPR": 0.08,
      "accessPath": "api",
      "url": "https://codereviewbench.com/model/glm-5.3-flash"
    },
    {
      "id": "kimi-k3",
      "name": "Kimi K3",
      "provider": "Moonshot",
      "recall": 41,
      "precision": 38.7,
      "f1": 39.9,
      "bugsFound": 39,
      "bugsTotal": 95,
      "pullRequests": 30,
      "costPerPR": 1.62,
      "accessPath": "api",
      "url": "https://codereviewbench.com/model/kimi-k3"
    },
    {
      "id": "deepseek-v4-flash",
      "name": "DeepSeek V4 Flash",
      "provider": "DeepSeek",
      "recall": 36.8,
      "precision": 42.1,
      "f1": 39.3,
      "bugsFound": 35,
      "bugsTotal": 95,
      "pullRequests": 30,
      "costPerPR": 0.1,
      "accessPath": "api",
      "url": "https://codereviewbench.com/model/deepseek-v4-flash"
    },
    {
      "id": "muse-spark-1.2",
      "name": "Muse Spark 1.2",
      "provider": "Meta",
      "recall": 41,
      "precision": 35,
      "f1": 37.8,
      "bugsFound": 39,
      "bugsTotal": 95,
      "pullRequests": 30,
      "costPerPR": 0.49,
      "accessPath": "api",
      "url": "https://codereviewbench.com/model/muse-spark-1.2"
    },
    {
      "id": "qwen3.8-27b",
      "name": "Qwen3.8 27B",
      "provider": "Alibaba",
      "recall": 34.4,
      "precision": 38.1,
      "f1": 36.2,
      "bugsFound": 32,
      "bugsTotal": 93,
      "pullRequests": 29,
      "costPerPR": 0.34,
      "accessPath": "api",
      "url": "https://codereviewbench.com/model/qwen3.8-27b"
    },
    {
      "id": "minimax-m3@fireworks",
      "name": "MiniMax M3",
      "provider": "MiniMax",
      "recall": 25.3,
      "precision": 35.6,
      "f1": 29.6,
      "bugsFound": 24,
      "bugsTotal": 95,
      "pullRequests": 30,
      "costPerPR": 0.17,
      "accessPath": "api",
      "url": "https://codereviewbench.com/model/minimax-m3--fireworks"
    },
    {
      "id": "gemini-3.7-flash",
      "name": "Gemini 3.7 Flash",
      "provider": "Google",
      "recall": 11.6,
      "precision": 73.9,
      "f1": 20,
      "bugsFound": 11,
      "bugsTotal": 95,
      "pullRequests": 30,
      "costPerPR": 0.27,
      "accessPath": "api",
      "url": "https://codereviewbench.com/model/gemini-3.7-flash"
    }
  ]
}