diff --git a/README.md b/README.md index 969cccf2..34565ed8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Which Programming Language Is Best for AI Coding Agents? -A quantitative benchmark comparing how efficiently [Claude Code](https://docs.anthropic.com/en/docs/claude-code) generates code across 13 programming languages. +A quantitative benchmark comparing how efficiently [Claude Code](https://docs.anthropic.com/en/docs/claude-code) generates code across 14 programming languages. For a detailed discussion, see the blog post: [Which Programming Language Is Best for Claude Code?](https://dev.to/mame/which-programming-language-is-best-for-claude-code-508a) / [日本語版](https://zenn.dev/mametter/articles/3e8580ec034201) @@ -27,7 +27,7 @@ The prompt is simply: "Read [SPEC-v1.txt](./SPEC-v1.txt), implement it, and make | Category | Languages | |----------|-----------| -| Dynamic | Python, Ruby, JavaScript, Perl, Lua | +| Dynamic | Python, Ruby, JavaScript, Perl, Lua, Julia | | Dynamic + type checker | Python/mypy, Ruby/Steep | | Static | TypeScript, Go, Rust, C, Java | | Functional | Scheme (dynamic), OCaml (static), Haskell (static) | @@ -54,9 +54,10 @@ Each language was run **20 times**. A custom hash algorithm (not SHA-256) is use | Lua | 40/40 | 143.6s ± 43.0s | $0.58 | 398 | | C | 40/40 | 155.8s ± 40.9s | $0.74 | 517 | | Haskell | 39/40 | 174.0s ± 44.2s | $0.74 | 224 | +| Julia | 40/40 | 186.0s ± 34.0s | $0.46 | 271 | | Ruby/Steep | 40/40 | 186.6s ± 69.7s | $0.84 | 304 | -Out of 600 runs (15 configurations × 2 phases × 20 trials), only 3 failed: Rust (2) and Haskell (1). +Out of 640 runs (16 configurations × 2 phases × 20 trials), only 3 failed: Rust (2) and Haskell (1). ### Total Time and Cost (v1 + v2) diff --git a/benchmark.rb b/benchmark.rb index 772f3bd3..b296e8f5 100755 --- a/benchmark.rb +++ b/benchmark.rb @@ -15,6 +15,7 @@ GO_DIR = File.join(Dir.home, '.local', 'go') NPM_PREFIX = File.join(Dir.home, '.local', 'npm') +CLAUDE_BIN = ENV['CLAUDE_BIN'] || 'claude' LANGUAGES = { 'rust' => { exts: %w[rs], version_cmd: 'rustc --version' }, @@ -36,6 +37,7 @@ 'scheme' => { exts: %w[scm], version_cmd: 'guile --version | head -1' }, 'ocaml' => { exts: %w[ml mli], version_cmd: 'ocaml --version' }, 'haskell' => { exts: %w[hs], version_cmd: 'ghc --version' }, + 'julia' => { exts: %w[jl], version_cmd: 'julia --version' }, } TRIALS = 3 @@ -165,7 +167,7 @@ def parse_claude_output(raw_output) def run_claude(prompt, dir:, log_path: nil) env_prefix = "unset CLAUDECODE && export PATH=#{extra_path}:$PATH && " - cmd = "#{env_prefix}claude -p #{Shellwords.escape(prompt)} --dangerously-skip-permissions --output-format json" + cmd = "#{env_prefix}#{CLAUDE_BIN} -p #{Shellwords.escape(prompt)} --dangerously-skip-permissions --output-format json" puts " Running Claude..." start_time = Time.now @@ -213,7 +215,7 @@ def run_tests(test_script, dir:) puts '=' * 60 puts -claude_version_result = run_cmd('claude --version 2>/dev/null || echo unknown') +claude_version_result = run_cmd("#{CLAUDE_BIN} --version 2>/dev/null || echo unknown") claude_version = claude_version_result[:stdout].strip puts "Claude Version: #{claude_version}" diff --git a/figures/total_cost.png b/figures/total_cost.png index f3494d18..1878439e 100644 Binary files a/figures/total_cost.png and b/figures/total_cost.png differ diff --git a/figures/total_lines.png b/figures/total_lines.png index 65068c0d..fd1c0c2f 100644 Binary files a/figures/total_lines.png and b/figures/total_lines.png differ diff --git a/figures/total_time.png b/figures/total_time.png index 4e0800fc..01c2615b 100644 Binary files a/figures/total_time.png and b/figures/total_time.png differ diff --git a/figures/total_time_vs_cost.png b/figures/total_time_vs_cost.png index cfc716b2..3e2b9d03 100644 Binary files a/figures/total_time_vs_cost.png and b/figures/total_time_vs_cost.png differ diff --git a/figures/total_time_vs_loc.png b/figures/total_time_vs_loc.png index 0e2d3cec..5d34b316 100644 Binary files a/figures/total_time_vs_loc.png and b/figures/total_time_vs_loc.png differ diff --git a/figures/total_turns.png b/figures/total_turns.png index ad8815f0..1cdcff46 100644 Binary files a/figures/total_turns.png and b/figures/total_turns.png differ diff --git a/figures/v1_cost.png b/figures/v1_cost.png index 44fe5f5f..5ce9b1ec 100644 Binary files a/figures/v1_cost.png and b/figures/v1_cost.png differ diff --git a/figures/v1_lines.png b/figures/v1_lines.png index ca734cd8..1eac01a6 100644 Binary files a/figures/v1_lines.png and b/figures/v1_lines.png differ diff --git a/figures/v1_time.png b/figures/v1_time.png index 26011e5a..9a4793ef 100644 Binary files a/figures/v1_time.png and b/figures/v1_time.png differ diff --git a/figures/v1_time_vs_cost.png b/figures/v1_time_vs_cost.png index c1d94dd6..32a5abb0 100644 Binary files a/figures/v1_time_vs_cost.png and b/figures/v1_time_vs_cost.png differ diff --git a/figures/v1_time_vs_loc.png b/figures/v1_time_vs_loc.png index 8cf31928..f634b1a2 100644 Binary files a/figures/v1_time_vs_loc.png and b/figures/v1_time_vs_loc.png differ diff --git a/figures/v1_turns.png b/figures/v1_turns.png index e9f8fc2e..7120aa21 100644 Binary files a/figures/v1_turns.png and b/figures/v1_turns.png differ diff --git a/figures/v2_cost.png b/figures/v2_cost.png index 80a71998..6a885e16 100644 Binary files a/figures/v2_cost.png and b/figures/v2_cost.png differ diff --git a/figures/v2_lines.png b/figures/v2_lines.png index 65068c0d..fd1c0c2f 100644 Binary files a/figures/v2_lines.png and b/figures/v2_lines.png differ diff --git a/figures/v2_time.png b/figures/v2_time.png index 7ae0cebf..4a62202a 100644 Binary files a/figures/v2_time.png and b/figures/v2_time.png differ diff --git a/figures/v2_time_vs_cost.png b/figures/v2_time_vs_cost.png index 39d4f389..e8eec5f4 100644 Binary files a/figures/v2_time_vs_cost.png and b/figures/v2_time_vs_cost.png differ diff --git a/figures/v2_time_vs_loc.png b/figures/v2_time_vs_loc.png index ebacd664..3bcaabf3 100644 Binary files a/figures/v2_time_vs_loc.png and b/figures/v2_time_vs_loc.png differ diff --git a/figures/v2_turns.png b/figures/v2_turns.png index bc3792b3..5ead8b75 100644 Binary files a/figures/v2_turns.png and b/figures/v2_turns.png differ diff --git a/plot.py b/plot.py index ed7d6ab6..f2ec2260 100644 --- a/plot.py +++ b/plot.py @@ -32,7 +32,7 @@ # Each group is separated by a gap in the plot. LANG_GROUPS = [ # Dynamic - ["ruby", "python", "javascript", "perl", "lua"], + ["ruby", "python", "javascript", "perl", "lua", "julia"], # Dynamic + type checker ["ruby/steep", "python/mypy"], # Static (imperative) @@ -60,6 +60,7 @@ "java": "Java", "ocaml": "OCaml", "haskell": "Haskell", + "julia": "Julia", } # Colour palette @@ -84,6 +85,8 @@ "scheme": "#888888", "ocaml": "#A0A0A0", "haskell": "#606060", + # Dynamic (JIT) + "julia": "#9558B2", } DEFAULT_COLOUR = "#999999" diff --git a/results/meta.json b/results/meta.json index 437776c6..8bea5d9f 100644 --- a/results/meta.json +++ b/results/meta.json @@ -1,7 +1,7 @@ { - "date": "2026-03-04 22:09:41", + "date": "2026-03-07 05:06:39", "claude_version": "2.1.68 (Claude Code)", - "trials": 10, + "trials": 20, "versions": { "rust": "rustc 1.93.1 (01f6ddf75 2026-02-11)", "go": "go version go1.23.6 linux/amd64", @@ -17,6 +17,7 @@ "lua": "Lua 5.4.6 Copyright (C) 1994-2023 Lua.org, PUC-Rio", "scheme": "guile (GNU Guile) 3.0.9", "ocaml": "The OCaml toplevel, version 4.14.1", - "haskell": "The Glorious Glasgow Haskell Compilation System, version 9.4.7" + "haskell": "The Glorious Glasgow Haskell Compilation System, version 9.4.7", + "julia": "julia version 1.12.5" } -} \ No newline at end of file +} diff --git a/results/report.md b/results/report.md index 0f725516..877cdc97 100644 --- a/results/report.md +++ b/results/report.md @@ -1,28 +1,29 @@ # Claude Code Language Benchmark Report ## Environment -- Date: 2026-03-04 22:09:41 +- Date: 2026-03-07 05:06:39 - Claude Version: 2.1.68 (Claude Code) -- Trials per language: 10 +- Trials per language: 20 ## Language Versions | Language | Version | |----------|---------| -| Rust | rustc 1.93.1 (01f6ddf75 2026-02-11) | -| Go | go version go1.23.6 linux/amd64 | -| C | gcc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0 | -| Typescript | tsx v4.21.0 | -| Javascript | v18.19.1 | -| Java | openjdk 25.0.2 2026-01-20 | -| Perl | This is perl 5, version 38, subversion 2 (v5.38.2) built for x86_64-linux-gnu-thread-multi | -| Python | Python 3.12.3 | -| Python/mypy | Python 3.12.3 | -| Ruby | ruby 4.0.1 (2026-01-13 revision e04267a14b) +PRISM [x86_64-linux] | -| Ruby/steep | ruby 4.0.1 (2026-01-13 revision e04267a14b) +PRISM [x86_64-linux] | -| Lua | Lua 5.4.6 Copyright (C) 1994-2023 Lua.org, PUC-Rio | -| Scheme | guile (GNU Guile) 3.0.9 | -| Ocaml | The OCaml toplevel, version 4.14.1 | -| Haskell | The Glorious Glasgow Haskell Compilation System, version 9.4.7 | +| Rust | unknown | +| Go | unknown | +| C | unknown | +| Typescript | unknown | +| Javascript | unknown | +| Java | unknown | +| Perl | unknown | +| Python | unknown | +| Python/mypy | unknown | +| Ruby | unknown | +| Ruby/steep | unknown | +| Lua | unknown | +| Scheme | unknown | +| Ocaml | unknown | +| Haskell | unknown | +| Julia | julia version 1.12.5 | ## Results Summary | Language | v1 Time | v1 Turns | v1 LOC | v1 Tests | v2 Time | v2 Turns | v2 LOC | v2 Tests | Total Time | Avg Cost | @@ -42,6 +43,7 @@ | Scheme | 66.7s±36.7s | 9.0 | 171 | 20/20 | 63.9s±10.0s | 10.6 | 310 | 20/20 | 130.6s±39.9s | $0.60 | | Ocaml | 80.9s±28.8s | 11.3 | 111 | 20/20 | 47.1s±6.0s | 9.2 | 216 | 20/20 | 128.1s±28.9s | $0.58 | | Haskell | 74.4s±39.1s | 10.3 | 119 | 19/20 | 99.6s±32.4s | 16.4 | 224 | 20/20 | 174.0s±44.2s | $0.74 | +| Julia | 62.1s±6.3s | 7.9 | 129 | 20/20 | 123.9s±30.3s | 7.8 | 271 | 20/20 | 186.0s±34.9s | $0.46 | ## Token Summary | Language | Avg Input | Avg Output | Avg Cache Create | Avg Cache Read | Avg Total | Avg Cost | @@ -61,6 +63,7 @@ | Scheme | 20 | 7,496 | 29,834 | 457,419 | 494,769 | $0.6027 | | Ocaml | 21 | 7,636 | 26,165 | 453,908 | 487,730 | $0.5815 | | Haskell | 27 | 8,636 | 29,185 | 681,994 | 719,841 | $0.7394 | +| Julia | 17 | 4,898 | 27,178 | 333,479 | 365,573 | $0.4591 | ## Full Results | Language | Trial | v1 Time | v1 Turns | v1 LOC | v1 Tests | v2 Time | v2 Turns | v2 LOC | v2 Tests | Total Time | Cost | @@ -365,6 +368,26 @@ | Scheme | 20 | 52.2s | 8 | 170 | 11/11 PASS | 62.8s | 11 | 305 | 30/30 PASS | 115.0s | $0.59 | | Ocaml | 20 | 63.5s | 11 | 105 | 11/11 PASS | 45.0s | 8 | 209 | 30/30 PASS | 108.5s | $0.54 | | Haskell | 20 | 45.9s | 9 | 107 | 11/11 PASS | 72.0s | 11 | 214 | 30/30 PASS | 117.9s | $0.51 | +| Julia | 1 | 64.8s | 8 | 123 | 11/11 PASS | 117.7s | 7 | 268 | 30/30 PASS | 182.5s | $0.46 | +| Julia | 2 | 62.0s | 8 | 132 | 11/11 PASS | 128.6s | 8 | 242 | 30/30 PASS | 190.6s | $0.45 | +| Julia | 3 | 64.6s | 8 | 133 | 11/11 PASS | 118.2s | 7 | 282 | 30/30 PASS | 182.8s | $0.46 | +| Julia | 4 | 50.6s | 8 | 124 | 11/11 PASS | 89.3s | 8 | 271 | 30/30 PASS | 139.9s | $0.42 | +| Julia | 5 | 70.2s | 8 | 141 | 11/11 PASS | 174.5s | 10 | 264 | 30/30 PASS | 244.7s | $0.51 | +| Julia | 6 | 66.1s | 8 | 125 | 11/11 PASS | 127.1s | 10 | 269 | 30/30 PASS | 193.2s | $0.50 | +| Julia | 7 | 68.1s | 8 | 125 | 11/11 PASS | 125.0s | 7 | 260 | 30/30 PASS | 193.1s | $0.46 | +| Julia | 8 | 63.5s | 8 | 128 | 11/11 PASS | 131.6s | 8 | 285 | 30/30 PASS | 195.1s | $0.47 | +| Julia | 9 | 57.2s | 8 | 145 | 11/11 PASS | 105.2s | 8 | 294 | 30/30 PASS | 162.4s | $0.48 | +| Julia | 10 | 69.5s | 8 | 130 | 11/11 PASS | 138.4s | 7 | 272 | 30/30 PASS | 207.9s | $0.46 | +| Julia | 11 | 63.2s | 8 | 122 | 11/11 PASS | 127.2s | 8 | 265 | 30/30 PASS | 190.4s | $0.46 | +| Julia | 12 | 51.0s | 8 | 125 | 11/11 PASS | 90.5s | 7 | 267 | 30/30 PASS | 141.5s | $0.45 | +| Julia | 13 | 57.7s | 8 | 122 | 11/11 PASS | 85.5s | 8 | 269 | 30/30 PASS | 143.2s | $0.46 | +| Julia | 14 | 66.2s | 6 | 126 | 11/11 PASS | 137.6s | 7 | 272 | 30/30 PASS | 203.8s | $0.42 | +| Julia | 15 | 65.7s | 8 | 128 | 11/11 PASS | 129.8s | 8 | 272 | 30/30 PASS | 195.5s | $0.46 | +| Julia | 16 | 66.1s | 8 | 131 | 11/11 PASS | 134.7s | 8 | 278 | 30/30 PASS | 200.8s | $0.46 | +| Julia | 17 | 54.3s | 8 | 124 | 11/11 PASS | 88.7s | 7 | 269 | 30/30 PASS | 143.0s | $0.46 | +| Julia | 18 | 62.5s | 8 | 129 | 11/11 PASS | 209.1s | 10 | 263 | 30/30 PASS | 271.6s | $0.49 | +| Julia | 19 | 50.4s | 8 | 130 | 11/11 PASS | 86.1s | 6 | 280 | 30/30 PASS | 136.5s | $0.40 | +| Julia | 20 | 67.6s | 8 | 134 | 11/11 PASS | 133.0s | 7 | 272 | 30/30 PASS | 200.6s | $0.46 | ## Full Tokens | Language | Trial | Phase | Input | Output | Cache Create | Cache Read | Total | Cost USD | @@ -969,4 +992,44 @@ | Ocaml | 20 | v2 | 8 | 2,386 | 14,419 | 160,467 | 177,280 | $0.2300 | | Haskell | 20 | v1 | 9 | 2,226 | 8,816 | 174,008 | 185,059 | $0.1978 | | Haskell | 20 | v2 | 11 | 3,189 | 15,968 | 261,536 | 280,704 | $0.3103 | +| Julia | 1 | v1 | 9 | 2,022 | 9,603 | 171,017 | 182,651 | $0.1961 | +| Julia | 1 | v2 | 8 | 2,810 | 18,428 | 153,128 | 174,374 | $0.2620 | +| Julia | 2 | v1 | 9 | 2,040 | 9,694 | 171,296 | 183,039 | $0.1973 | +| Julia | 2 | v2 | 9 | 3,175 | 13,452 | 170,340 | 186,976 | $0.2487 | +| Julia | 3 | v1 | 9 | 2,097 | 9,671 | 171,230 | 183,007 | $0.1985 | +| Julia | 3 | v2 | 8 | 2,734 | 18,504 | 153,391 | 174,637 | $0.2607 | +| Julia | 4 | v1 | 9 | 2,080 | 9,438 | 171,197 | 182,724 | $0.1966 | +| Julia | 4 | v2 | 8 | 2,913 | 12,681 | 143,049 | 158,651 | $0.2236 | +| Julia | 5 | v1 | 9 | 2,202 | 9,784 | 171,572 | 183,567 | $0.2020 | +| Julia | 5 | v2 | 10 | 2,888 | 19,841 | 220,613 | 243,352 | $0.3066 | +| Julia | 6 | v1 | 9 | 2,065 | 9,653 | 171,176 | 182,903 | $0.1976 | +| Julia | 6 | v2 | 10 | 2,676 | 20,180 | 222,655 | 245,521 | $0.3044 | +| Julia | 7 | v1 | 9 | 2,024 | 9,610 | 171,041 | 182,684 | $0.1962 | +| Julia | 7 | v2 | 8 | 2,747 | 18,396 | 153,139 | 174,290 | $0.2603 | +| Julia | 8 | v1 | 9 | 2,038 | 9,693 | 171,290 | 183,030 | $0.1972 | +| Julia | 8 | v2 | 8 | 2,929 | 18,974 | 154,764 | 176,675 | $0.2692 | +| Julia | 9 | v1 | 9 | 2,156 | 9,524 | 171,476 | 183,165 | $0.1992 | +| Julia | 9 | v2 | 9 | 3,019 | 18,224 | 187,650 | 208,902 | $0.2832 | +| Julia | 10 | v1 | 9 | 1,985 | 9,638 | 171,140 | 182,772 | $0.1955 | +| Julia | 10 | v2 | 8 | 2,779 | 18,461 | 153,261 | 174,509 | $0.2615 | +| Julia | 11 | v1 | 9 | 1,943 | 9,595 | 170,999 | 182,546 | $0.1941 | +| Julia | 11 | v2 | 8 | 2,819 | 18,515 | 153,475 | 174,817 | $0.2630 | +| Julia | 12 | v1 | 9 | 2,034 | 9,391 | 171,072 | 182,506 | $0.1951 | +| Julia | 12 | v2 | 8 | 2,772 | 17,733 | 153,149 | 173,662 | $0.2567 | +| Julia | 13 | v1 | 9 | 2,065 | 9,428 | 171,186 | 182,688 | $0.1962 | +| Julia | 13 | v2 | 9 | 2,534 | 17,640 | 185,675 | 205,858 | $0.2665 | +| Julia | 14 | v1 | 7 | 1,959 | 9,179 | 118,532 | 129,677 | $0.1656 | +| Julia | 14 | v2 | 8 | 2,747 | 17,714 | 153,308 | 173,777 | $0.2561 | +| Julia | 15 | v1 | 9 | 2,084 | 9,666 | 171,231 | 182,990 | $0.1982 | +| Julia | 15 | v2 | 8 | 2,844 | 18,837 | 154,600 | 176,289 | $0.2662 | +| Julia | 16 | v1 | 9 | 2,115 | 9,673 | 171,255 | 183,052 | $0.1990 | +| Julia | 16 | v2 | 8 | 2,851 | 18,667 | 153,724 | 175,250 | $0.2648 | +| Julia | 17 | v1 | 9 | 2,091 | 9,452 | 171,251 | 182,803 | $0.1970 | +| Julia | 17 | v2 | 8 | 2,802 | 17,867 | 153,488 | 174,165 | $0.2585 | +| Julia | 18 | v1 | 9 | 2,085 | 9,674 | 171,248 | 183,016 | $0.1983 | +| Julia | 18 | v2 | 10 | 3,225 | 17,357 | 208,384 | 228,976 | $0.2933 | +| Julia | 19 | v1 | 9 | 2,100 | 9,449 | 171,245 | 182,803 | $0.1972 | +| Julia | 19 | v2 | 7 | 2,677 | 12,047 | 116,535 | 131,266 | $0.2005 | +| Julia | 20 | v1 | 9 | 2,050 | 9,705 | 171,335 | 183,099 | $0.1976 | +| Julia | 20 | v2 | 8 | 2,780 | 18,526 | 153,467 | 174,781 | $0.2621 | diff --git a/results/results.json b/results/results.json index 1d3fdb0f..789051e0 100644 --- a/results/results.json +++ b/results/results.json @@ -10798,5 +10798,725 @@ "num_turns": 11, "duration_ms": 70386 } + }, + { + "language": "julia", + "trial": 1, + "v1_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-1-v1", + "v2_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-1-v2", + "v1_time": 64.8, + "v1_pass": true, + "v1_passed_count": 11, + "v1_failed_count": 0, + "v1_total_count": 11, + "v1_loc": 123, + "v2_time": 117.7, + "v2_pass": true, + "v2_passed_count": 30, + "v2_failed_count": 0, + "v2_total_count": 30, + "v2_loc": 268, + "v1_claude": { + "input_tokens": 9, + "output_tokens": 2022, + "cache_creation_tokens": 9603, + "cache_read_tokens": 171017, + "cost_usd": 0.19612225000000003, + "num_turns": 8, + "duration_ms": 63543 + }, + "v2_claude": { + "input_tokens": 8, + "output_tokens": 2810, + "cache_creation_tokens": 18428, + "cache_read_tokens": 153128, + "cost_usd": 0.26202899999999996, + "num_turns": 7, + "duration_ms": 116128 + } + }, + { + "language": "julia", + "trial": 2, + "v1_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-2-v1", + "v2_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-2-v2", + "v1_time": 62.0, + "v1_pass": true, + "v1_passed_count": 11, + "v1_failed_count": 0, + "v1_total_count": 11, + "v1_loc": 132, + "v2_time": 128.6, + "v2_pass": true, + "v2_passed_count": 30, + "v2_failed_count": 0, + "v2_total_count": 30, + "v2_loc": 242, + "v1_claude": { + "input_tokens": 9, + "output_tokens": 2040, + "cache_creation_tokens": 9694, + "cache_read_tokens": 171296, + "cost_usd": 0.19728050000000003, + "num_turns": 8, + "duration_ms": 60325 + }, + "v2_claude": { + "input_tokens": 9, + "output_tokens": 3175, + "cache_creation_tokens": 13452, + "cache_read_tokens": 170340, + "cost_usd": 0.24866500000000002, + "num_turns": 8, + "duration_ms": 126838 + } + }, + { + "language": "julia", + "trial": 3, + "v1_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-3-v1", + "v2_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-3-v2", + "v1_time": 64.6, + "v1_pass": true, + "v1_passed_count": 11, + "v1_failed_count": 0, + "v1_total_count": 11, + "v1_loc": 133, + "v2_time": 118.2, + "v2_pass": true, + "v2_passed_count": 30, + "v2_failed_count": 0, + "v2_total_count": 30, + "v2_loc": 282, + "v1_claude": { + "input_tokens": 9, + "output_tokens": 2097, + "cache_creation_tokens": 9671, + "cache_read_tokens": 171230, + "cost_usd": 0.19852875000000003, + "num_turns": 8, + "duration_ms": 62952 + }, + "v2_claude": { + "input_tokens": 8, + "output_tokens": 2734, + "cache_creation_tokens": 18504, + "cache_read_tokens": 153391, + "cost_usd": 0.2607355, + "num_turns": 7, + "duration_ms": 116552 + } + }, + { + "language": "julia", + "trial": 4, + "v1_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-4-v1", + "v2_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-4-v2", + "v1_time": 50.6, + "v1_pass": true, + "v1_passed_count": 11, + "v1_failed_count": 0, + "v1_total_count": 11, + "v1_loc": 124, + "v2_time": 89.3, + "v2_pass": true, + "v2_passed_count": 30, + "v2_failed_count": 0, + "v2_total_count": 30, + "v2_loc": 271, + "v1_claude": { + "input_tokens": 9, + "output_tokens": 2080, + "cache_creation_tokens": 9438, + "cache_read_tokens": 171197, + "cost_usd": 0.196631, + "num_turns": 8, + "duration_ms": 48790 + }, + "v2_claude": { + "input_tokens": 8, + "output_tokens": 2913, + "cache_creation_tokens": 12681, + "cache_read_tokens": 143049, + "cost_usd": 0.22364575000000003, + "num_turns": 8, + "duration_ms": 86213 + } + }, + { + "language": "julia", + "trial": 5, + "v1_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-5-v1", + "v2_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-5-v2", + "v1_time": 70.2, + "v1_pass": true, + "v1_passed_count": 11, + "v1_failed_count": 0, + "v1_total_count": 11, + "v1_loc": 141, + "v2_time": 174.5, + "v2_pass": true, + "v2_passed_count": 30, + "v2_failed_count": 0, + "v2_total_count": 30, + "v2_loc": 264, + "v1_claude": { + "input_tokens": 9, + "output_tokens": 2202, + "cache_creation_tokens": 9784, + "cache_read_tokens": 171572, + "cost_usd": 0.20203100000000002, + "num_turns": 8, + "duration_ms": 68527 + }, + "v2_claude": { + "input_tokens": 10, + "output_tokens": 2888, + "cache_creation_tokens": 19841, + "cache_read_tokens": 220613, + "cost_usd": 0.30656275, + "num_turns": 10, + "duration_ms": 172701 + } + }, + { + "language": "julia", + "trial": 6, + "v1_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-6-v1", + "v2_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-6-v2", + "v1_time": 66.1, + "v1_pass": true, + "v1_passed_count": 11, + "v1_failed_count": 0, + "v1_total_count": 11, + "v1_loc": 125, + "v2_time": 127.1, + "v2_pass": true, + "v2_passed_count": 30, + "v2_failed_count": 0, + "v2_total_count": 30, + "v2_loc": 269, + "v1_claude": { + "input_tokens": 9, + "output_tokens": 2065, + "cache_creation_tokens": 9653, + "cache_read_tokens": 171176, + "cost_usd": 0.19758925, + "num_turns": 8, + "duration_ms": 64093 + }, + "v2_claude": { + "input_tokens": 10, + "output_tokens": 2676, + "cache_creation_tokens": 20180, + "cache_read_tokens": 222655, + "cost_usd": 0.3044025, + "num_turns": 10, + "duration_ms": 125307 + } + }, + { + "language": "julia", + "trial": 7, + "v1_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-7-v1", + "v2_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-7-v2", + "v1_time": 68.1, + "v1_pass": true, + "v1_passed_count": 11, + "v1_failed_count": 0, + "v1_total_count": 11, + "v1_loc": 125, + "v2_time": 125.0, + "v2_pass": true, + "v2_passed_count": 30, + "v2_failed_count": 0, + "v2_total_count": 30, + "v2_loc": 260, + "v1_claude": { + "input_tokens": 9, + "output_tokens": 2024, + "cache_creation_tokens": 9610, + "cache_read_tokens": 171041, + "cost_usd": 0.196228, + "num_turns": 8, + "duration_ms": 66512 + }, + "v2_claude": { + "input_tokens": 8, + "output_tokens": 2747, + "cache_creation_tokens": 18396, + "cache_read_tokens": 153139, + "cost_usd": 0.26025950000000003, + "num_turns": 7, + "duration_ms": 123299 + } + }, + { + "language": "julia", + "trial": 8, + "v1_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-8-v1", + "v2_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-8-v2", + "v1_time": 63.5, + "v1_pass": true, + "v1_passed_count": 11, + "v1_failed_count": 0, + "v1_total_count": 11, + "v1_loc": 128, + "v2_time": 131.6, + "v2_pass": true, + "v2_passed_count": 30, + "v2_failed_count": 0, + "v2_total_count": 30, + "v2_loc": 285, + "v1_claude": { + "input_tokens": 9, + "output_tokens": 2038, + "cache_creation_tokens": 9693, + "cache_read_tokens": 171290, + "cost_usd": 0.19722125000000004, + "num_turns": 8, + "duration_ms": 61342 + }, + "v2_claude": { + "input_tokens": 8, + "output_tokens": 2929, + "cache_creation_tokens": 18974, + "cache_read_tokens": 154764, + "cost_usd": 0.2692345, + "num_turns": 8, + "duration_ms": 129755 + } + }, + { + "language": "julia", + "trial": 9, + "v1_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-9-v1", + "v2_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-9-v2", + "v1_time": 57.2, + "v1_pass": true, + "v1_passed_count": 11, + "v1_failed_count": 0, + "v1_total_count": 11, + "v1_loc": 145, + "v2_time": 105.2, + "v2_pass": true, + "v2_passed_count": 30, + "v2_failed_count": 0, + "v2_total_count": 30, + "v2_loc": 294, + "v1_claude": { + "input_tokens": 9, + "output_tokens": 2156, + "cache_creation_tokens": 9524, + "cache_read_tokens": 171476, + "cost_usd": 0.19920800000000002, + "num_turns": 8, + "duration_ms": 55412 + }, + "v2_claude": { + "input_tokens": 9, + "output_tokens": 3019, + "cache_creation_tokens": 18224, + "cache_read_tokens": 187650, + "cost_usd": 0.283245, + "num_turns": 8, + "duration_ms": 103696 + } + }, + { + "language": "julia", + "trial": 10, + "v1_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-10-v1", + "v2_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-10-v2", + "v1_time": 69.5, + "v1_pass": true, + "v1_passed_count": 11, + "v1_failed_count": 0, + "v1_total_count": 11, + "v1_loc": 130, + "v2_time": 138.4, + "v2_pass": true, + "v2_passed_count": 30, + "v2_failed_count": 0, + "v2_total_count": 30, + "v2_loc": 272, + "v1_claude": { + "input_tokens": 9, + "output_tokens": 1985, + "cache_creation_tokens": 9638, + "cache_read_tokens": 171140, + "cost_usd": 0.1954775, + "num_turns": 8, + "duration_ms": 67714 + }, + "v2_claude": { + "input_tokens": 8, + "output_tokens": 2779, + "cache_creation_tokens": 18461, + "cache_read_tokens": 153261, + "cost_usd": 0.26152675, + "num_turns": 7, + "duration_ms": 136383 + } + }, + { + "language": "julia", + "trial": 11, + "v1_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-11-v1", + "v2_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-11-v2", + "v1_time": 63.2, + "v1_pass": true, + "v1_passed_count": 11, + "v1_failed_count": 0, + "v1_total_count": 11, + "v1_loc": 122, + "v2_time": 127.2, + "v2_pass": true, + "v2_passed_count": 30, + "v2_failed_count": 0, + "v2_total_count": 30, + "v2_loc": 265, + "v1_claude": { + "input_tokens": 9, + "output_tokens": 1943, + "cache_creation_tokens": 9595, + "cache_read_tokens": 170999, + "cost_usd": 0.19408825000000002, + "num_turns": 8, + "duration_ms": 61584 + }, + "v2_claude": { + "input_tokens": 8, + "output_tokens": 2819, + "cache_creation_tokens": 18515, + "cache_read_tokens": 153475, + "cost_usd": 0.26297125, + "num_turns": 8, + "duration_ms": 124361 + } + }, + { + "language": "julia", + "trial": 12, + "v1_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-12-v1", + "v2_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-12-v2", + "v1_time": 51.0, + "v1_pass": true, + "v1_passed_count": 11, + "v1_failed_count": 0, + "v1_total_count": 11, + "v1_loc": 125, + "v2_time": 90.5, + "v2_pass": true, + "v2_passed_count": 30, + "v2_failed_count": 0, + "v2_total_count": 30, + "v2_loc": 267, + "v1_claude": { + "input_tokens": 9, + "output_tokens": 2034, + "cache_creation_tokens": 9391, + "cache_read_tokens": 171072, + "cost_usd": 0.19512475, + "num_turns": 8, + "duration_ms": 49150 + }, + "v2_claude": { + "input_tokens": 8, + "output_tokens": 2772, + "cache_creation_tokens": 17733, + "cache_read_tokens": 153149, + "cost_usd": 0.25674575, + "num_turns": 7, + "duration_ms": 88818 + } + }, + { + "language": "julia", + "trial": 13, + "v1_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-13-v1", + "v2_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-13-v2", + "v1_time": 57.7, + "v1_pass": true, + "v1_passed_count": 11, + "v1_failed_count": 0, + "v1_total_count": 11, + "v1_loc": 122, + "v2_time": 85.5, + "v2_pass": true, + "v2_passed_count": 30, + "v2_failed_count": 0, + "v2_total_count": 30, + "v2_loc": 269, + "v1_claude": { + "input_tokens": 9, + "output_tokens": 2065, + "cache_creation_tokens": 9428, + "cache_read_tokens": 171186, + "cost_usd": 0.19618799999999997, + "num_turns": 8, + "duration_ms": 56023 + }, + "v2_claude": { + "input_tokens": 9, + "output_tokens": 2534, + "cache_creation_tokens": 17640, + "cache_read_tokens": 185675, + "cost_usd": 0.2664825, + "num_turns": 8, + "duration_ms": 83906 + } + }, + { + "language": "julia", + "trial": 14, + "v1_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-14-v1", + "v2_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-14-v2", + "v1_time": 66.2, + "v1_pass": true, + "v1_passed_count": 11, + "v1_failed_count": 0, + "v1_total_count": 11, + "v1_loc": 126, + "v2_time": 137.6, + "v2_pass": true, + "v2_passed_count": 30, + "v2_failed_count": 0, + "v2_total_count": 30, + "v2_loc": 272, + "v1_claude": { + "input_tokens": 7, + "output_tokens": 1959, + "cache_creation_tokens": 9179, + "cache_read_tokens": 118532, + "cost_usd": 0.16564474999999998, + "num_turns": 6, + "duration_ms": 64467 + }, + "v2_claude": { + "input_tokens": 8, + "output_tokens": 2747, + "cache_creation_tokens": 17714, + "cache_read_tokens": 153308, + "cost_usd": 0.25608149999999996, + "num_turns": 7, + "duration_ms": 135650 + } + }, + { + "language": "julia", + "trial": 15, + "v1_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-15-v1", + "v2_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-15-v2", + "v1_time": 65.7, + "v1_pass": true, + "v1_passed_count": 11, + "v1_failed_count": 0, + "v1_total_count": 11, + "v1_loc": 128, + "v2_time": 129.8, + "v2_pass": true, + "v2_passed_count": 30, + "v2_failed_count": 0, + "v2_total_count": 30, + "v2_loc": 272, + "v1_claude": { + "input_tokens": 9, + "output_tokens": 2084, + "cache_creation_tokens": 9666, + "cache_read_tokens": 171231, + "cost_usd": 0.19817299999999996, + "num_turns": 8, + "duration_ms": 63607 + }, + "v2_claude": { + "input_tokens": 8, + "output_tokens": 2844, + "cache_creation_tokens": 18837, + "cache_read_tokens": 154600, + "cost_usd": 0.26617124999999997, + "num_turns": 8, + "duration_ms": 127290 + } + }, + { + "language": "julia", + "trial": 16, + "v1_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-16-v1", + "v2_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-16-v2", + "v1_time": 66.1, + "v1_pass": true, + "v1_passed_count": 11, + "v1_failed_count": 0, + "v1_total_count": 11, + "v1_loc": 131, + "v2_time": 134.7, + "v2_pass": true, + "v2_passed_count": 30, + "v2_failed_count": 0, + "v2_total_count": 30, + "v2_loc": 278, + "v1_claude": { + "input_tokens": 9, + "output_tokens": 2115, + "cache_creation_tokens": 9673, + "cache_read_tokens": 171255, + "cost_usd": 0.19900374999999998, + "num_turns": 8, + "duration_ms": 64311 + }, + "v2_claude": { + "input_tokens": 8, + "output_tokens": 2851, + "cache_creation_tokens": 18667, + "cache_read_tokens": 153724, + "cost_usd": 0.26484575000000005, + "num_turns": 8, + "duration_ms": 132289 + } + }, + { + "language": "julia", + "trial": 17, + "v1_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-17-v1", + "v2_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-17-v2", + "v1_time": 54.3, + "v1_pass": true, + "v1_passed_count": 11, + "v1_failed_count": 0, + "v1_total_count": 11, + "v1_loc": 124, + "v2_time": 88.7, + "v2_pass": true, + "v2_passed_count": 30, + "v2_failed_count": 0, + "v2_total_count": 30, + "v2_loc": 269, + "v1_claude": { + "input_tokens": 9, + "output_tokens": 2091, + "cache_creation_tokens": 9452, + "cache_read_tokens": 171251, + "cost_usd": 0.19702050000000002, + "num_turns": 8, + "duration_ms": 52427 + }, + "v2_claude": { + "input_tokens": 8, + "output_tokens": 2802, + "cache_creation_tokens": 17867, + "cache_read_tokens": 153488, + "cost_usd": 0.25850275, + "num_turns": 7, + "duration_ms": 86451 + } + }, + { + "language": "julia", + "trial": 18, + "v1_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-18-v1", + "v2_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-18-v2", + "v1_time": 62.5, + "v1_pass": true, + "v1_passed_count": 11, + "v1_failed_count": 0, + "v1_total_count": 11, + "v1_loc": 129, + "v2_time": 209.1, + "v2_pass": true, + "v2_passed_count": 30, + "v2_failed_count": 0, + "v2_total_count": 30, + "v2_loc": 263, + "v1_claude": { + "input_tokens": 9, + "output_tokens": 2085, + "cache_creation_tokens": 9674, + "cache_read_tokens": 171248, + "cost_usd": 0.1982565, + "num_turns": 8, + "duration_ms": 60681 + }, + "v2_claude": { + "input_tokens": 10, + "output_tokens": 3225, + "cache_creation_tokens": 17357, + "cache_read_tokens": 208384, + "cost_usd": 0.29334825, + "num_turns": 10, + "duration_ms": 207413 + } + }, + { + "language": "julia", + "trial": 19, + "v1_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-19-v1", + "v2_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-19-v2", + "v1_time": 50.4, + "v1_pass": true, + "v1_passed_count": 11, + "v1_failed_count": 0, + "v1_total_count": 11, + "v1_loc": 130, + "v2_time": 86.1, + "v2_pass": true, + "v2_passed_count": 30, + "v2_failed_count": 0, + "v2_total_count": 30, + "v2_loc": 280, + "v1_claude": { + "input_tokens": 9, + "output_tokens": 2100, + "cache_creation_tokens": 9449, + "cache_read_tokens": 171245, + "cost_usd": 0.19722375, + "num_turns": 8, + "duration_ms": 48132 + }, + "v2_claude": { + "input_tokens": 7, + "output_tokens": 2677, + "cache_creation_tokens": 12047, + "cache_read_tokens": 116535, + "cost_usd": 0.20052125, + "num_turns": 6, + "duration_ms": 84777 + } + }, + { + "language": "julia", + "trial": 20, + "v1_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-20-v1", + "v2_dir": "/home/ai-coding-lang-bench/generated/minigit-julia-20-v2", + "v1_time": 67.6, + "v1_pass": true, + "v1_passed_count": 11, + "v1_failed_count": 0, + "v1_total_count": 11, + "v1_loc": 134, + "v2_time": 133.0, + "v2_pass": true, + "v2_passed_count": 30, + "v2_failed_count": 0, + "v2_total_count": 30, + "v2_loc": 272, + "v1_claude": { + "input_tokens": 9, + "output_tokens": 2050, + "cache_creation_tokens": 9705, + "cache_read_tokens": 171335, + "cost_usd": 0.19761875, + "num_turns": 8, + "duration_ms": 65954 + }, + "v2_claude": { + "input_tokens": 8, + "output_tokens": 2780, + "cache_creation_tokens": 18526, + "cache_read_tokens": 153467, + "cost_usd": 0.262061, + "num_turns": 7, + "duration_ms": 130855 + } } -] \ No newline at end of file +]