{
  "name": "Codna head-to-head benchmark",
  "measuredOn": "2026-06-15",
  "method": "Each scenario was run on an identical isolated checkout. A fix counts only when the project’s own oracle test passes — for all three agents. Codna = deterministic engine + agent (bring-your-own-key). Baselines: OpenAI Codex CLI and Google Gemini CLI at their defaults.",
  "understanding": {
    "repos": 130,
    "languages": 110,
    "codeCoverageGitHub": "99.9%",
    "totalTime": "9.2s",
    "perRepoP50": "59ms",
    "perRepoP90": "134ms",
    "tokens": 0,
    "accuracy": "100%",
    "files": 823
  },
  "evidenceBundle": {
    "reductionRatio": 162,
    "rawTokens": 103111,
    "bundleTokens": 634,
    "fixesFrom": "~600",
    "insteadOf": "100,000+"
  },
  "totals": {
    "codna": {
      "tokens": 465324,
      "time": 108
    },
    "codex": {
      "tokens": 6022479,
      "time": 568
    },
    "gemini": {
      "tokens": 4458345,
      "time": 1000
    }
  },
  "multipliers": {
    "vsCodex": {
      "tokenX": 12.9,
      "speedX": 5.2
    },
    "vsGemini": {
      "tokenX": 9.6,
      "speedX": 9.2
    },
    "bestTokenReduction": 18.3
  },
  "verified": "8 / 8",
  "costPerVerifiedFixUsd": 0.021,
  "scenarios": [
    {
      "scenario": "click · make_default_short_help",
      "lang": "Python",
      "codnaTokens": 29151,
      "codexTokens": 728892,
      "geminiTokens": 533557,
      "codnaTime": 13.4,
      "codexTime": 80.8,
      "geminiTime": 118.8,
      "tokenX": 18.3,
      "speedX": 6
    },
    {
      "scenario": "click · split_arg_string",
      "lang": "Python",
      "codnaTokens": 29185,
      "codexTokens": 647437,
      "geminiTokens": 424672,
      "codnaTime": 12.1,
      "codexTime": 67.3,
      "geminiTime": 80.7,
      "tokenX": 14.6,
      "speedX": 5.5
    },
    {
      "scenario": "click · _expand_args",
      "lang": "Python",
      "codnaTokens": 33480,
      "codexTokens": 382891,
      "geminiTokens": 232652,
      "codnaTime": 6.8,
      "codexTime": 41.9,
      "geminiTime": 42.4,
      "tokenX": 6.9,
      "speedX": 6.2
    },
    {
      "scenario": "click · _split_opt",
      "lang": "Python",
      "codnaTokens": 45254,
      "codexTokens": 928492,
      "geminiTokens": 370221,
      "codnaTime": 11.1,
      "codexTime": 82.6,
      "geminiTime": 49.5,
      "tokenX": 8.2,
      "speedX": 4.5
    },
    {
      "scenario": "ky · prefix + baseUrl",
      "lang": "TypeScript",
      "codnaTokens": 78044,
      "codexTokens": 960994,
      "geminiTokens": 592551,
      "codnaTime": 20.7,
      "codexTime": 79.7,
      "geminiTime": 470.2,
      "tokenX": 7.6,
      "speedX": 3.8
    },
    {
      "scenario": "click · 3 bugs at once",
      "lang": "Python",
      "codnaTokens": 171644,
      "codexTokens": 1293234,
      "geminiTokens": 1673781,
      "codnaTime": 23.5,
      "codexTime": 121.1,
      "geminiTime": 140.1,
      "tokenX": 7.5,
      "speedX": 5.2
    },
    {
      "scenario": "jmespath · ends_with (new repo)",
      "lang": "Python",
      "codnaTokens": 29805,
      "codexTokens": 681130,
      "geminiTokens": 382401,
      "codnaTime": 6.5,
      "codexTime": 42,
      "geminiTime": 40.1,
      "tokenX": 12.8,
      "speedX": 6.1
    },
    {
      "scenario": "p-map · ordering (new repo)",
      "lang": "TypeScript",
      "codnaTokens": 48761,
      "codexTokens": 399409,
      "geminiTokens": 248510,
      "codnaTime": 14,
      "codexTime": 52.3,
      "geminiTime": 58.5,
      "tokenX": 5.1,
      "speedX": 3.7
    }
  ]
}