benchmark improvements

koka_bench/benchmarks.md

@@ -2,16 +2,15 @@
 
 ## cfold_table.md
 
-| Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
+| Command | Mean [s] | Min [s] | Max [s] | Relative |
 |:---|---:|---:|---:|---:|
-| `build/kraken/out/bench/kraken-cfold 5` | 26.0 ± 1.0 | 23.9 | 28.2 | 89.45 ± 114.65 |
-| `build/kraken/out/bench/kraken-cfold-wavm 5` | 38.2 ± 0.8 | 36.8 | 40.7 | 131.51 ± 168.51 |
-| `build/java/out/bench/cfold 5` | 69.4 ± 8.5 | 58.8 | 87.5 | 239.07 ± 307.69 |
-| `build/ocaml/ml-cfold 5` | 0.4 ± 0.4 | 0.1 | 2.6 | 1.36 ± 2.23 |
-| `build/swift/sw-cfold 5` | 1.8 ± 0.4 | 1.4 | 3.6 | 6.13 ± 7.99 |
-| `build/cpp/cpp-cfold 5` | 0.7 ± 0.4 | 0.3 | 3.9 | 2.28 ± 3.27 |
-| `build/haskell/hs-cfold 5` | 0.6 ± 0.3 | 0.3 | 3.7 | 1.93 ± 2.72 |
-| `build/koka/out/bench/kk-cfold 5` | 0.3 ± 0.4 | 0.0 | 2.5 | 1.00 |
+| `build/kraken/out/bench/kraken-cfold-wavm 20` | 1.262 ± 0.009 | 1.255 | 1.285 | 9.52 ± 0.21 |
+| `build/java/out/bench/cfold 20` | 0.357 ± 0.006 | 0.346 | 0.364 | 2.69 ± 0.07 |
+| `build/ocaml/ml-cfold 20` | 0.539 ± 0.002 | 0.536 | 0.540 | 4.06 ± 0.09 |
+| `build/swift/sw-cfold 20` | 0.864 ± 0.004 | 0.855 | 0.871 | 6.52 ± 0.14 |
+| `build/cpp/cpp-cfold 20` | 0.306 ± 0.002 | 0.303 | 0.311 | 2.31 ± 0.05 |
+| `build/haskell/hs-cfold 20` | 0.504 ± 0.001 | 0.502 | 0.505 | 3.80 ± 0.08 |
+| `build/koka/out/bench/kk-cfold 20` | 0.133 ± 0.003 | 0.129 | 0.139 | 1.00 |
 
 
 
@@ -108,3 +107,15 @@
 
 
 
+## slow_rbtree_table.md
+
+| Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
+|:---|---:|---:|---:|---:|
+| `./slow/kraken-rbtree-opt 100` | 24.1 ± 1.0 | 22.2 | 26.7 | 1.88 ± 0.12 |
+| `./slow/kraken-rbtree-opt-wavm 100` | 34.7 ± 7.8 | 32.5 | 104.4 | 2.71 ± 0.63 |
+| `./slow/newlisp-slow-fexpr-rbtree 100` | 334.7 ± 1.7 | 331.3 | 337.2 | 26.13 ± 1.27 |
+| `./slow/newlisp-macro-rbtree 100` | 12.8 ± 0.6 | 12.1 | 15.0 | 1.00 |
+| `./slow/kraken-rbtree-slow-wavm 100` | 2501167.5 ± 26231.6 | 2428798.0 | 2514951.4 | 195272.93 ± 9651.02 |
+
+
+

koka_bench/cfold_table.csv

@@ -1,9 +1,8 @@
 command,mean,stddev,median,user,system,min,max
-build/kraken/out/bench/kraken-cfold 5,0.02598189402339623,0.0009827326433493578,0.02583577658,0.01569527433962264,0.011144978490566036,0.02388484608,0.028162415080000003
-build/kraken/out/bench/kraken-cfold-wavm 5,0.03820012912109588,0.0007528298714480669,0.038197756080000006,0.025975090958904108,0.010941174794520546,0.036762960080000004,0.040656301080000005
-build/java/out/bench/cfold 5,0.06944132718869567,0.008515814941967776,0.07077024558,0.04709683217391304,0.02651659826086956,0.05881540608000001,0.08747230608
-build/ocaml/ml-cfold 5,0.0003952098066591666,0.0004022557222530553,0.0002778910799999994,0.0003407772328458943,0.0003637282789651291,0.00007847807999999913,0.0025972300799999995
-build/swift/sw-cfold 5,0.0017816054617663822,0.0004122523773228782,0.0016685800799999995,0.0010956197720797714,0.0007277779487179485,0.001442869079999999,0.00363124608
-build/cpp/cpp-cfold 5,0.0006628962093413166,0.0004239916203779702,0.0005484790799999994,0.0005155858682634732,0.00039921322155688594,0.00033362907999999946,0.0039450690799999995
-build/haskell/hs-cfold 5,0.0005605955675373863,0.0003324826920403133,0.00047954007999999933,0.00042487940179461567,0.0003972196211365898,0.00029374707999999927,0.0037330830799999996
-build/koka/out/bench/kk-cfold 5,0.000290463924008908,0.0003721316109128869,0.00018300457999999927,0.00030117986636971056,0.0003280589755011133,0,0.00252745408
+build/kraken/out/bench/kraken-cfold-wavm 20,1.2621564595800003,0.008640411915661495,1.2593907364799999,1.13313784,0.118044,1.25479187498,1.28487568498
+build/java/out/bench/cfold 20,0.35657448718000007,0.00558577826352044,0.35739406448000005,0.40431914,0.1469187,0.34625631998000006,0.36395965198
+build/ocaml/ml-cfold 20,0.5385355318800001,0.001566527327571848,0.53853815098,0.50224824,0.036017,0.53558768298,0.54036365198
+build/swift/sw-cfold 20,0.8642314439800002,0.0044482895786414275,0.8649492004800001,0.80760954,0.05612069999999999,0.85469952498,0.8705609119800001
+build/cpp/cpp-cfold 20,0.30622296198,0.0024784149826698612,0.30592687648000005,0.21424323999999997,0.0918147,0.30293893898,0.31113143098
+build/haskell/hs-cfold 20,0.5037699929799999,0.001091585403056106,0.50399623548,0.4556685399999999,0.04782399999999999,0.50162550498,0.5051531599800001
+build/koka/out/bench/kk-cfold 20,0.1326358102981818,0.002816643075142655,0.13199419048,0.09420199454545455,0.03824613636363636,0.12888567098,0.13929317298

koka_bench/cfold_table.csvwith_relative.csv

@@ -1,9 +1,8 @@
 command,mean,stddev,median,user,system,min,max,relative
-build/koka/out/bench/kk-cfold 5,0.000290463924008908,0.0003721316109128869,0.00018300457999999927,0.00030117986636971056,0.0003280589755011133,0,0.00252745408,1.0
-build/ocaml/ml-cfold 5,0.0003952098066591666,0.0004022557222530553,0.0002778910799999994,0.0003407772328458943,0.0003637282789651291,0.00007847807999999913,0.0025972300799999995,1.3606158079963357
-build/haskell/hs-cfold 5,0.0005605955675373863,0.0003324826920403133,0.00047954007999999933,0.00042487940179461567,0.0003972196211365898,0.00029374707999999927,0.0037330830799999996,1.9300006685862783
-build/cpp/cpp-cfold 5,0.0006628962093413166,0.0004239916203779702,0.0005484790799999994,0.0005155858682634732,0.00039921322155688594,0.00033362907999999946,0.0039450690799999995,2.2821980788257434
-build/swift/sw-cfold 5,0.0017816054617663822,0.0004122523773228782,0.0016685800799999995,0.0010956197720797714,0.0007277779487179485,0.001442869079999999,0.00363124608,6.133654869001025
-build/kraken/out/bench/kraken-cfold 5,0.02598189402339623,0.0009827326433493578,0.02583577658,0.01569527433962264,0.011144978490566036,0.02388484608,0.028162415080000003,89.44964202370073
-build/kraken/out/bench/kraken-cfold-wavm 5,0.03820012912109588,0.0007528298714480669,0.038197756080000006,0.025975090958904108,0.010941174794520546,0.036762960080000004,0.040656301080000005,131.51419492606027
-build/java/out/bench/cfold 5,0.06944132718869567,0.008515814941967776,0.07077024558,0.04709683217391304,0.02651659826086956,0.05881540608000001,0.08747230608,239.07040237659956
+build/koka/out/bench/kk-cfold 20,0.1326358102981818,0.002816643075142655,0.13199419048,0.09420199454545455,0.03824613636363636,0.12888567098,0.13929317298,1.0
+build/cpp/cpp-cfold 20,0.30622296198,0.0024784149826698612,0.30592687648000005,0.21424323999999997,0.0918147,0.30293893898,0.31113143098,2.3087502635342045
+build/java/out/bench/cfold 20,0.35657448718000007,0.00558577826352044,0.35739406448000005,0.40431914,0.1469187,0.34625631998000006,0.36395965198,2.6883726678215805
+build/haskell/hs-cfold 20,0.5037699929799999,0.001091585403056106,0.50399623548,0.4556685399999999,0.04782399999999999,0.50162550498,0.5051531599800001,3.798144647719664
+build/ocaml/ml-cfold 20,0.5385355318800001,0.001566527327571848,0.53853815098,0.50224824,0.036017,0.53558768298,0.54036365198,4.060257412152158
+build/swift/sw-cfold 20,0.8642314439800002,0.0044482895786414275,0.8649492004800001,0.80760954,0.05612069999999999,0.85469952498,0.8705609119800001,6.515822853851462
+build/kraken/out/bench/kraken-cfold-wavm 20,1.2621564595800003,0.008640411915661495,1.2593907364799999,1.13313784,0.118044,1.25479187498,1.28487568498,9.515955432718476

koka_bench/cfold_table.md

@@ -1,10 +1,9 @@
-| Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
+| Command | Mean [s] | Min [s] | Max [s] | Relative |
 |:---|---:|---:|---:|---:|
-| `build/kraken/out/bench/kraken-cfold 5` | 26.0 ± 1.0 | 23.9 | 28.2 | 89.45 ± 114.65 |
-| `build/kraken/out/bench/kraken-cfold-wavm 5` | 38.2 ± 0.8 | 36.8 | 40.7 | 131.51 ± 168.51 |
-| `build/java/out/bench/cfold 5` | 69.4 ± 8.5 | 58.8 | 87.5 | 239.07 ± 307.69 |
-| `build/ocaml/ml-cfold 5` | 0.4 ± 0.4 | 0.1 | 2.6 | 1.36 ± 2.23 |
-| `build/swift/sw-cfold 5` | 1.8 ± 0.4 | 1.4 | 3.6 | 6.13 ± 7.99 |
-| `build/cpp/cpp-cfold 5` | 0.7 ± 0.4 | 0.3 | 3.9 | 2.28 ± 3.27 |
-| `build/haskell/hs-cfold 5` | 0.6 ± 0.3 | 0.3 | 3.7 | 1.93 ± 2.72 |
-| `build/koka/out/bench/kk-cfold 5` | 0.3 ± 0.4 | 0.0 | 2.5 | 1.00 |
+| `build/kraken/out/bench/kraken-cfold-wavm 20` | 1.262 ± 0.009 | 1.255 | 1.285 | 9.52 ± 0.21 |
+| `build/java/out/bench/cfold 20` | 0.357 ± 0.006 | 0.346 | 0.364 | 2.69 ± 0.07 |
+| `build/ocaml/ml-cfold 20` | 0.539 ± 0.002 | 0.536 | 0.540 | 4.06 ± 0.09 |
+| `build/swift/sw-cfold 20` | 0.864 ± 0.004 | 0.855 | 0.871 | 6.52 ± 0.14 |
+| `build/cpp/cpp-cfold 20` | 0.306 ± 0.002 | 0.303 | 0.311 | 2.31 ± 0.05 |
+| `build/haskell/hs-cfold 20` | 0.504 ± 0.001 | 0.502 | 0.505 | 3.80 ± 0.08 |
+| `build/koka/out/bench/kk-cfold 20` | 0.133 ± 0.003 | 0.129 | 0.139 | 1.00 |

koka_bench/relative.py

@@ -19,7 +19,7 @@ with open(sys.argv[1] + "with_relative.csv", "w") as f:
     f.write(out)
 
 print(csv)
-csv = [ x for x in csv if 'rbtree' not in x[0] or 'kraken' not in x[0] or 'opt' in x[0] ]
+csv = [ x for x in csv if 'slow' in x[0] or 'rbtree' not in x[0] or 'kraken' not in x[0] or 'opt' in x[0] ]
 print(csv)
 
 def make_name(n):
@@ -28,7 +28,7 @@ def make_name(n):
                                                               .split(' ')[0]\
                                                               .replace("-", " ").split(" ")\
                     if word not in {"rbtree"})
-    if out == "" and "java" in n:
+    if "java" in n:
         out = "Java"
     print(f"changed {n} to {out}")
     return out
@@ -56,6 +56,8 @@ for do_log in [False, True]:
 
     ax.set_facecolor('gainsboro')
     rects1 = plt.bar(index, a, bar_width, alpha=opacity, color='orange')
+    for k_index  in (i for i in range(len(d)) if 'Kraken' in d[i]):
+        rects1[k_index].set_color('r')
     plt.xlabel(f"{out_name} Benchmark" + (" (Log Scale)" if do_log else ""))
     plt.ylabel('Runtime (s)' + (" (Log Scale)" if do_log else ""))
     ax.ticklabel_format(useOffset=False, style='plain')

koka_bench/slow_rbtree_table.csv

@@ -0,0 +1,6 @@
+command,mean,stddev,median,user,system,min,max
+./slow/kraken-rbtree-opt 100,0.024073812766902663,0.00101568872666385,0.023981108820000003,0.013964058938053093,0.010416836283185843,0.022171882820000002,0.026677836820000002
+./slow/kraken-rbtree-opt-wavm 100,0.034688892393170745,0.007841371606363657,0.033655831319999996,0.02159911268292682,0.01100518292682927,0.03249684882,0.10438634682000002
+./slow/newlisp-slow-fexpr-rbtree 100,0.33467720061999995,0.0017169909863666642,0.33419976582,0.33144182000000005,0.0029782,0.33133876282,0.33724592082
+./slow/newlisp-macro-rbtree 100,0.012808572662105267,0.0006186239065595693,0.01251284132,0.009985556842105265,0.0026657263157894733,0.012097865820000001,0.014971716820000001
+./slow/kraken-rbtree-slow-wavm 100,2501.1674679941198,26.23156008638233,2511.02493639732,2500.37616872,0.5680571000000001,2428.79803291582,2514.95137303882

koka_bench/slow_rbtree_table.csvwith_relative.csv

@@ -0,0 +1,6 @@
+command,mean,stddev,median,user,system,min,max,relative
+./slow/newlisp-macro-rbtree 100,0.012808572662105267,0.0006186239065595693,0.01251284132,0.009985556842105265,0.0026657263157894733,0.012097865820000001,0.014971716820000001,1.0
+./slow/kraken-rbtree-opt 100,0.024073812766902663,0.00101568872666385,0.023981108820000003,0.013964058938053093,0.010416836283185843,0.022171882820000002,0.026677836820000002,1.879507842284887
+./slow/kraken-rbtree-opt-wavm 100,0.034688892393170745,0.007841371606363657,0.033655831319999996,0.02159911268292682,0.01100518292682927,0.03249684882,0.10438634682000002,2.7082558930082334
+./slow/newlisp-slow-fexpr-rbtree 100,0.33467720061999995,0.0017169909863666642,0.33419976582,0.33144182000000005,0.0029782,0.33133876282,0.33724592082,26.129156577309924
+./slow/kraken-rbtree-slow-wavm 100,2501.1674679941198,26.23156008638233,2511.02493639732,2500.37616872,0.5680571000000001,2428.79803291582,2514.95137303882,195272.92649819877

koka_bench/test.sh

@@ -15,14 +15,15 @@ popd
 mkdir -p slow
 find build -type f -name \*slow\* -exec mv {} slow \;
 cp ./build/kraken/out/bench/kraken-* ./slow
+mv ./build/kraken/out/bench/kraken-cfold ./slow
 mv ./build/newlisp/out/bench/* ./slow
  
 
 #nix develop -i -c bash -c 'ulimit -s unlimited && find build -type f -executable -name \*rbtree\* -printf "\"%p 420000\"\n" | xargs hyperfine --ignore-failure --warmup 2 --export-markdown rbtree_table.md --export-csv rbtree_table.csv'
 #nix develop -i -c bash -c 'ulimit -s unlimited && find build -type f -executable -name \*fib\* -printf "\"%p 30\"\n"      | xargs hyperfine --ignore-failure --warmup 2 --export-markdown fib_table.md --export-csv fib_table.csv'
 #nix develop -i -c bash -c 'ulimit -s unlimited && find build -type f -executable -name \*nqueens\* -printf "\"%p 10\"\n"   | xargs hyperfine --ignore-failure --warmup 2 --export-markdown nqueens_table.md --export-csv nqueens_table.csv'
-#nix develop -i -c bash -c 'ulimit -s unlimited && find build -type f -executable -name \*cfold\* -printf "\"%p 5\"\n"      | xargs hyperfine --ignore-failure --warmup 2 --export-markdown cfold_table.md --export-csv cfold_table.csv'
 #nix develop -i -c bash -c 'ulimit -s unlimited && find build -type f -executable -name \*deriv\* -printf "\"%p 8\"\n"      | xargs hyperfine --ignore-failure --warmup 2 --export-markdown deriv_table.md --export-csv deriv_table.csv'
+#nix develop -i -c bash -c 'ulimit -s unlimited && find build -type f -executable -name \*cfold\* -printf "\"%p 20\"\n"      | xargs hyperfine --ignore-failure --warmup 2 --export-markdown cfold_table.md --export-csv cfold_table.csv'
 
 
 #nix develop -i -c bash -c 'ulimit -s unlimited && find slow -type f -executable -name \*nqueens\* -printf "\"%p 7\"\n"  | xargs hyperfine --ignore-failure --warmup 2 --export-markdown slow_nqueens_table.md --export-csv slow_nqueens_table.csv'
@@ -32,10 +33,11 @@ mv ./build/newlisp/out/bench/* ./slow
 
 #nix develop -i -c bash -c 'ulimit -s unlimited && find slow -type f -executable -name \*fib\* -printf "\"%p 30\"\n"  | xargs hyperfine --ignore-failure --warmup 2 --export-markdown slow_fib_table.md --export-csv slow_fib_table.csv'
 #nix develop -i -c bash -c 'ulimit -s unlimited && hyperfine --ignore-failure --warmup 2 --export-markdown slow_ish_rbtree_table.md  --export-csv slow_ish_rbtree_table.csv "./slow/kraken-rbtree-opt 890" "./slow/kraken-rbtree-opt-wavm 890" "./slow/newlisp-slow-fexpr-rbtree 890" "./slow/newlisp-macro-rbtree 890"'
+#nix develop -i -c bash -c 'ulimit -s unlimited && hyperfine --ignore-failure --warmup 2 --export-markdown slow_rbtree_table.md  --export-csv slow_rbtree_table.csv "./slow/kraken-rbtree-opt 100" "./slow/kraken-rbtree-opt-wavm 100" "./slow/newlisp-slow-fexpr-rbtree 100" "./slow/newlisp-macro-rbtree 100" "./slow/kraken-rbtree-slow-wavm 100"'
 
 for x in *_table.csv
 do
-	./relative.py $x
+	nix develop -i -c bash -c "./relative.py $x"
 done
 
 printf "# Benchmarks\n\n" > benchmarks.md