diff --git a/koka_bench/benchmarks.md b/koka_bench/benchmarks.md index c68058b..1e80aa9 100644 --- a/koka_bench/benchmarks.md +++ b/koka_bench/benchmarks.md @@ -2,16 +2,15 @@ ## cfold_table.md -| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| Command | Mean [s] | Min [s] | Max [s] | Relative | |:---|---:|---:|---:|---:| -| `build/kraken/out/bench/kraken-cfold 5` | 26.0 ± 1.0 | 23.9 | 28.2 | 89.45 ± 114.65 | -| `build/kraken/out/bench/kraken-cfold-wavm 5` | 38.2 ± 0.8 | 36.8 | 40.7 | 131.51 ± 168.51 | -| `build/java/out/bench/cfold 5` | 69.4 ± 8.5 | 58.8 | 87.5 | 239.07 ± 307.69 | -| `build/ocaml/ml-cfold 5` | 0.4 ± 0.4 | 0.1 | 2.6 | 1.36 ± 2.23 | -| `build/swift/sw-cfold 5` | 1.8 ± 0.4 | 1.4 | 3.6 | 6.13 ± 7.99 | -| `build/cpp/cpp-cfold 5` | 0.7 ± 0.4 | 0.3 | 3.9 | 2.28 ± 3.27 | -| `build/haskell/hs-cfold 5` | 0.6 ± 0.3 | 0.3 | 3.7 | 1.93 ± 2.72 | -| `build/koka/out/bench/kk-cfold 5` | 0.3 ± 0.4 | 0.0 | 2.5 | 1.00 | +| `build/kraken/out/bench/kraken-cfold-wavm 20` | 1.262 ± 0.009 | 1.255 | 1.285 | 9.52 ± 0.21 | +| `build/java/out/bench/cfold 20` | 0.357 ± 0.006 | 0.346 | 0.364 | 2.69 ± 0.07 | +| `build/ocaml/ml-cfold 20` | 0.539 ± 0.002 | 0.536 | 0.540 | 4.06 ± 0.09 | +| `build/swift/sw-cfold 20` | 0.864 ± 0.004 | 0.855 | 0.871 | 6.52 ± 0.14 | +| `build/cpp/cpp-cfold 20` | 0.306 ± 0.002 | 0.303 | 0.311 | 2.31 ± 0.05 | +| `build/haskell/hs-cfold 20` | 0.504 ± 0.001 | 0.502 | 0.505 | 3.80 ± 0.08 | +| `build/koka/out/bench/kk-cfold 20` | 0.133 ± 0.003 | 0.129 | 0.139 | 1.00 | @@ -108,3 +107,15 @@ +## slow_rbtree_table.md + +| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +|:---|---:|---:|---:|---:| +| `./slow/kraken-rbtree-opt 100` | 24.1 ± 1.0 | 22.2 | 26.7 | 1.88 ± 0.12 | +| `./slow/kraken-rbtree-opt-wavm 100` | 34.7 ± 7.8 | 32.5 | 104.4 | 2.71 ± 0.63 | +| `./slow/newlisp-slow-fexpr-rbtree 100` | 334.7 ± 1.7 | 331.3 | 337.2 | 26.13 ± 1.27 | +| `./slow/newlisp-macro-rbtree 100` | 12.8 ± 0.6 | 12.1 | 15.0 | 1.00 | +| `./slow/kraken-rbtree-slow-wavm 100` | 2501167.5 ± 26231.6 | 2428798.0 | 2514951.4 | 195272.93 ± 9651.02 | + + + diff --git a/koka_bench/cfold_table.csv b/koka_bench/cfold_table.csv index f3b427d..a974084 100644 --- a/koka_bench/cfold_table.csv +++ b/koka_bench/cfold_table.csv @@ -1,9 +1,8 @@ command,mean,stddev,median,user,system,min,max -build/kraken/out/bench/kraken-cfold 5,0.02598189402339623,0.0009827326433493578,0.02583577658,0.01569527433962264,0.011144978490566036,0.02388484608,0.028162415080000003 -build/kraken/out/bench/kraken-cfold-wavm 5,0.03820012912109588,0.0007528298714480669,0.038197756080000006,0.025975090958904108,0.010941174794520546,0.036762960080000004,0.040656301080000005 -build/java/out/bench/cfold 5,0.06944132718869567,0.008515814941967776,0.07077024558,0.04709683217391304,0.02651659826086956,0.05881540608000001,0.08747230608 -build/ocaml/ml-cfold 5,0.0003952098066591666,0.0004022557222530553,0.0002778910799999994,0.0003407772328458943,0.0003637282789651291,0.00007847807999999913,0.0025972300799999995 -build/swift/sw-cfold 5,0.0017816054617663822,0.0004122523773228782,0.0016685800799999995,0.0010956197720797714,0.0007277779487179485,0.001442869079999999,0.00363124608 -build/cpp/cpp-cfold 5,0.0006628962093413166,0.0004239916203779702,0.0005484790799999994,0.0005155858682634732,0.00039921322155688594,0.00033362907999999946,0.0039450690799999995 -build/haskell/hs-cfold 5,0.0005605955675373863,0.0003324826920403133,0.00047954007999999933,0.00042487940179461567,0.0003972196211365898,0.00029374707999999927,0.0037330830799999996 -build/koka/out/bench/kk-cfold 5,0.000290463924008908,0.0003721316109128869,0.00018300457999999927,0.00030117986636971056,0.0003280589755011133,0,0.00252745408 +build/kraken/out/bench/kraken-cfold-wavm 20,1.2621564595800003,0.008640411915661495,1.2593907364799999,1.13313784,0.118044,1.25479187498,1.28487568498 +build/java/out/bench/cfold 20,0.35657448718000007,0.00558577826352044,0.35739406448000005,0.40431914,0.1469187,0.34625631998000006,0.36395965198 +build/ocaml/ml-cfold 20,0.5385355318800001,0.001566527327571848,0.53853815098,0.50224824,0.036017,0.53558768298,0.54036365198 +build/swift/sw-cfold 20,0.8642314439800002,0.0044482895786414275,0.8649492004800001,0.80760954,0.05612069999999999,0.85469952498,0.8705609119800001 +build/cpp/cpp-cfold 20,0.30622296198,0.0024784149826698612,0.30592687648000005,0.21424323999999997,0.0918147,0.30293893898,0.31113143098 +build/haskell/hs-cfold 20,0.5037699929799999,0.001091585403056106,0.50399623548,0.4556685399999999,0.04782399999999999,0.50162550498,0.5051531599800001 +build/koka/out/bench/kk-cfold 20,0.1326358102981818,0.002816643075142655,0.13199419048,0.09420199454545455,0.03824613636363636,0.12888567098,0.13929317298 diff --git a/koka_bench/cfold_table.csvwith_relative.csv b/koka_bench/cfold_table.csvwith_relative.csv index 7ad63be..aae082e 100644 --- a/koka_bench/cfold_table.csvwith_relative.csv +++ b/koka_bench/cfold_table.csvwith_relative.csv @@ -1,9 +1,8 @@ command,mean,stddev,median,user,system,min,max,relative -build/koka/out/bench/kk-cfold 5,0.000290463924008908,0.0003721316109128869,0.00018300457999999927,0.00030117986636971056,0.0003280589755011133,0,0.00252745408,1.0 -build/ocaml/ml-cfold 5,0.0003952098066591666,0.0004022557222530553,0.0002778910799999994,0.0003407772328458943,0.0003637282789651291,0.00007847807999999913,0.0025972300799999995,1.3606158079963357 -build/haskell/hs-cfold 5,0.0005605955675373863,0.0003324826920403133,0.00047954007999999933,0.00042487940179461567,0.0003972196211365898,0.00029374707999999927,0.0037330830799999996,1.9300006685862783 -build/cpp/cpp-cfold 5,0.0006628962093413166,0.0004239916203779702,0.0005484790799999994,0.0005155858682634732,0.00039921322155688594,0.00033362907999999946,0.0039450690799999995,2.2821980788257434 -build/swift/sw-cfold 5,0.0017816054617663822,0.0004122523773228782,0.0016685800799999995,0.0010956197720797714,0.0007277779487179485,0.001442869079999999,0.00363124608,6.133654869001025 -build/kraken/out/bench/kraken-cfold 5,0.02598189402339623,0.0009827326433493578,0.02583577658,0.01569527433962264,0.011144978490566036,0.02388484608,0.028162415080000003,89.44964202370073 -build/kraken/out/bench/kraken-cfold-wavm 5,0.03820012912109588,0.0007528298714480669,0.038197756080000006,0.025975090958904108,0.010941174794520546,0.036762960080000004,0.040656301080000005,131.51419492606027 -build/java/out/bench/cfold 5,0.06944132718869567,0.008515814941967776,0.07077024558,0.04709683217391304,0.02651659826086956,0.05881540608000001,0.08747230608,239.07040237659956 \ No newline at end of file +build/koka/out/bench/kk-cfold 20,0.1326358102981818,0.002816643075142655,0.13199419048,0.09420199454545455,0.03824613636363636,0.12888567098,0.13929317298,1.0 +build/cpp/cpp-cfold 20,0.30622296198,0.0024784149826698612,0.30592687648000005,0.21424323999999997,0.0918147,0.30293893898,0.31113143098,2.3087502635342045 +build/java/out/bench/cfold 20,0.35657448718000007,0.00558577826352044,0.35739406448000005,0.40431914,0.1469187,0.34625631998000006,0.36395965198,2.6883726678215805 +build/haskell/hs-cfold 20,0.5037699929799999,0.001091585403056106,0.50399623548,0.4556685399999999,0.04782399999999999,0.50162550498,0.5051531599800001,3.798144647719664 +build/ocaml/ml-cfold 20,0.5385355318800001,0.001566527327571848,0.53853815098,0.50224824,0.036017,0.53558768298,0.54036365198,4.060257412152158 +build/swift/sw-cfold 20,0.8642314439800002,0.0044482895786414275,0.8649492004800001,0.80760954,0.05612069999999999,0.85469952498,0.8705609119800001,6.515822853851462 +build/kraken/out/bench/kraken-cfold-wavm 20,1.2621564595800003,0.008640411915661495,1.2593907364799999,1.13313784,0.118044,1.25479187498,1.28487568498,9.515955432718476 \ No newline at end of file diff --git a/koka_bench/cfold_table.md b/koka_bench/cfold_table.md index 808764c..f1a1269 100644 --- a/koka_bench/cfold_table.md +++ b/koka_bench/cfold_table.md @@ -1,10 +1,9 @@ -| Command | Mean [ms] | Min [ms] | Max [ms] | Relative | +| Command | Mean [s] | Min [s] | Max [s] | Relative | |:---|---:|---:|---:|---:| -| `build/kraken/out/bench/kraken-cfold 5` | 26.0 ± 1.0 | 23.9 | 28.2 | 89.45 ± 114.65 | -| `build/kraken/out/bench/kraken-cfold-wavm 5` | 38.2 ± 0.8 | 36.8 | 40.7 | 131.51 ± 168.51 | -| `build/java/out/bench/cfold 5` | 69.4 ± 8.5 | 58.8 | 87.5 | 239.07 ± 307.69 | -| `build/ocaml/ml-cfold 5` | 0.4 ± 0.4 | 0.1 | 2.6 | 1.36 ± 2.23 | -| `build/swift/sw-cfold 5` | 1.8 ± 0.4 | 1.4 | 3.6 | 6.13 ± 7.99 | -| `build/cpp/cpp-cfold 5` | 0.7 ± 0.4 | 0.3 | 3.9 | 2.28 ± 3.27 | -| `build/haskell/hs-cfold 5` | 0.6 ± 0.3 | 0.3 | 3.7 | 1.93 ± 2.72 | -| `build/koka/out/bench/kk-cfold 5` | 0.3 ± 0.4 | 0.0 | 2.5 | 1.00 | +| `build/kraken/out/bench/kraken-cfold-wavm 20` | 1.262 ± 0.009 | 1.255 | 1.285 | 9.52 ± 0.21 | +| `build/java/out/bench/cfold 20` | 0.357 ± 0.006 | 0.346 | 0.364 | 2.69 ± 0.07 | +| `build/ocaml/ml-cfold 20` | 0.539 ± 0.002 | 0.536 | 0.540 | 4.06 ± 0.09 | +| `build/swift/sw-cfold 20` | 0.864 ± 0.004 | 0.855 | 0.871 | 6.52 ± 0.14 | +| `build/cpp/cpp-cfold 20` | 0.306 ± 0.002 | 0.303 | 0.311 | 2.31 ± 0.05 | +| `build/haskell/hs-cfold 20` | 0.504 ± 0.001 | 0.502 | 0.505 | 3.80 ± 0.08 | +| `build/koka/out/bench/kk-cfold 20` | 0.133 ± 0.003 | 0.129 | 0.139 | 1.00 | diff --git a/koka_bench/relative.py b/koka_bench/relative.py index ff6a8a4..6482401 100755 --- a/koka_bench/relative.py +++ b/koka_bench/relative.py @@ -19,7 +19,7 @@ with open(sys.argv[1] + "with_relative.csv", "w") as f: f.write(out) print(csv) -csv = [ x for x in csv if 'rbtree' not in x[0] or 'kraken' not in x[0] or 'opt' in x[0] ] +csv = [ x for x in csv if 'slow' in x[0] or 'rbtree' not in x[0] or 'kraken' not in x[0] or 'opt' in x[0] ] print(csv) def make_name(n): @@ -28,7 +28,7 @@ def make_name(n): .split(' ')[0]\ .replace("-", " ").split(" ")\ if word not in {"rbtree"}) - if out == "" and "java" in n: + if "java" in n: out = "Java" print(f"changed {n} to {out}") return out @@ -56,6 +56,8 @@ for do_log in [False, True]: ax.set_facecolor('gainsboro') rects1 = plt.bar(index, a, bar_width, alpha=opacity, color='orange') + for k_index in (i for i in range(len(d)) if 'Kraken' in d[i]): + rects1[k_index].set_color('r') plt.xlabel(f"{out_name} Benchmark" + (" (Log Scale)" if do_log else "")) plt.ylabel('Runtime (s)' + (" (Log Scale)" if do_log else "")) ax.ticklabel_format(useOffset=False, style='plain') diff --git a/koka_bench/slow_rbtree_table.csv b/koka_bench/slow_rbtree_table.csv new file mode 100644 index 0000000..9e2eb37 --- /dev/null +++ b/koka_bench/slow_rbtree_table.csv @@ -0,0 +1,6 @@ +command,mean,stddev,median,user,system,min,max +./slow/kraken-rbtree-opt 100,0.024073812766902663,0.00101568872666385,0.023981108820000003,0.013964058938053093,0.010416836283185843,0.022171882820000002,0.026677836820000002 +./slow/kraken-rbtree-opt-wavm 100,0.034688892393170745,0.007841371606363657,0.033655831319999996,0.02159911268292682,0.01100518292682927,0.03249684882,0.10438634682000002 +./slow/newlisp-slow-fexpr-rbtree 100,0.33467720061999995,0.0017169909863666642,0.33419976582,0.33144182000000005,0.0029782,0.33133876282,0.33724592082 +./slow/newlisp-macro-rbtree 100,0.012808572662105267,0.0006186239065595693,0.01251284132,0.009985556842105265,0.0026657263157894733,0.012097865820000001,0.014971716820000001 +./slow/kraken-rbtree-slow-wavm 100,2501.1674679941198,26.23156008638233,2511.02493639732,2500.37616872,0.5680571000000001,2428.79803291582,2514.95137303882 diff --git a/koka_bench/slow_rbtree_table.csvwith_relative.csv b/koka_bench/slow_rbtree_table.csvwith_relative.csv new file mode 100644 index 0000000..d4374f4 --- /dev/null +++ b/koka_bench/slow_rbtree_table.csvwith_relative.csv @@ -0,0 +1,6 @@ +command,mean,stddev,median,user,system,min,max,relative +./slow/newlisp-macro-rbtree 100,0.012808572662105267,0.0006186239065595693,0.01251284132,0.009985556842105265,0.0026657263157894733,0.012097865820000001,0.014971716820000001,1.0 +./slow/kraken-rbtree-opt 100,0.024073812766902663,0.00101568872666385,0.023981108820000003,0.013964058938053093,0.010416836283185843,0.022171882820000002,0.026677836820000002,1.879507842284887 +./slow/kraken-rbtree-opt-wavm 100,0.034688892393170745,0.007841371606363657,0.033655831319999996,0.02159911268292682,0.01100518292682927,0.03249684882,0.10438634682000002,2.7082558930082334 +./slow/newlisp-slow-fexpr-rbtree 100,0.33467720061999995,0.0017169909863666642,0.33419976582,0.33144182000000005,0.0029782,0.33133876282,0.33724592082,26.129156577309924 +./slow/kraken-rbtree-slow-wavm 100,2501.1674679941198,26.23156008638233,2511.02493639732,2500.37616872,0.5680571000000001,2428.79803291582,2514.95137303882,195272.92649819877 \ No newline at end of file diff --git a/koka_bench/test.sh b/koka_bench/test.sh index 0c712d2..b403e4c 100755 --- a/koka_bench/test.sh +++ b/koka_bench/test.sh @@ -15,14 +15,15 @@ popd mkdir -p slow find build -type f -name \*slow\* -exec mv {} slow \; cp ./build/kraken/out/bench/kraken-* ./slow +mv ./build/kraken/out/bench/kraken-cfold ./slow mv ./build/newlisp/out/bench/* ./slow #nix develop -i -c bash -c 'ulimit -s unlimited && find build -type f -executable -name \*rbtree\* -printf "\"%p 420000\"\n" | xargs hyperfine --ignore-failure --warmup 2 --export-markdown rbtree_table.md --export-csv rbtree_table.csv' #nix develop -i -c bash -c 'ulimit -s unlimited && find build -type f -executable -name \*fib\* -printf "\"%p 30\"\n" | xargs hyperfine --ignore-failure --warmup 2 --export-markdown fib_table.md --export-csv fib_table.csv' #nix develop -i -c bash -c 'ulimit -s unlimited && find build -type f -executable -name \*nqueens\* -printf "\"%p 10\"\n" | xargs hyperfine --ignore-failure --warmup 2 --export-markdown nqueens_table.md --export-csv nqueens_table.csv' -#nix develop -i -c bash -c 'ulimit -s unlimited && find build -type f -executable -name \*cfold\* -printf "\"%p 5\"\n" | xargs hyperfine --ignore-failure --warmup 2 --export-markdown cfold_table.md --export-csv cfold_table.csv' #nix develop -i -c bash -c 'ulimit -s unlimited && find build -type f -executable -name \*deriv\* -printf "\"%p 8\"\n" | xargs hyperfine --ignore-failure --warmup 2 --export-markdown deriv_table.md --export-csv deriv_table.csv' +#nix develop -i -c bash -c 'ulimit -s unlimited && find build -type f -executable -name \*cfold\* -printf "\"%p 20\"\n" | xargs hyperfine --ignore-failure --warmup 2 --export-markdown cfold_table.md --export-csv cfold_table.csv' #nix develop -i -c bash -c 'ulimit -s unlimited && find slow -type f -executable -name \*nqueens\* -printf "\"%p 7\"\n" | xargs hyperfine --ignore-failure --warmup 2 --export-markdown slow_nqueens_table.md --export-csv slow_nqueens_table.csv' @@ -32,10 +33,11 @@ mv ./build/newlisp/out/bench/* ./slow #nix develop -i -c bash -c 'ulimit -s unlimited && find slow -type f -executable -name \*fib\* -printf "\"%p 30\"\n" | xargs hyperfine --ignore-failure --warmup 2 --export-markdown slow_fib_table.md --export-csv slow_fib_table.csv' #nix develop -i -c bash -c 'ulimit -s unlimited && hyperfine --ignore-failure --warmup 2 --export-markdown slow_ish_rbtree_table.md --export-csv slow_ish_rbtree_table.csv "./slow/kraken-rbtree-opt 890" "./slow/kraken-rbtree-opt-wavm 890" "./slow/newlisp-slow-fexpr-rbtree 890" "./slow/newlisp-macro-rbtree 890"' +#nix develop -i -c bash -c 'ulimit -s unlimited && hyperfine --ignore-failure --warmup 2 --export-markdown slow_rbtree_table.md --export-csv slow_rbtree_table.csv "./slow/kraken-rbtree-opt 100" "./slow/kraken-rbtree-opt-wavm 100" "./slow/newlisp-slow-fexpr-rbtree 100" "./slow/newlisp-macro-rbtree 100" "./slow/kraken-rbtree-slow-wavm 100"' for x in *_table.csv do - ./relative.py $x + nix develop -i -c bash -c "./relative.py $x" done printf "# Benchmarks\n\n" > benchmarks.md