Split benchmark packages

This change allows each group of benchmarks to be run selectively,
as well as running all of them at once.

A new macro `define-coalton-benchmark` provides boilerplate of package definitions.
See README.md for the instructions.

benchmarks/README.md

@@ -1,7 +1,55 @@
-# To run Coalton Benchmarks:
+# To run Coalton benchmarks:
 
-`(ql:quickload :coalton/benchmarks)` or `(asdf:load-system :coalton/benchmarks)`
+- `(ql:quickload :coalton/benchmarks)` or `(asdf:load-system :coalton/benchmarks)`
 
-`(in-package #:coalton-benchmarks)`
+- `(in-package #:coalton-benchmarks)`
 
-`(run-benchmarks)`
+- `(run-benchmarks)` runs all the benchmarks
+
+- `(run-benchmark :BENCHMARK-PACKAGE)` runs the named benchmark package.
+
+# To add a new benchmark
+
+## Add `define-coalton-benchmark` form in `package.lisp`
+
+```
+(define-coalton-benchmark mybenchmark
+   (<extra-package-clause> ...)
+  <native-package-clause>
+  ...)
+```
+
+This expands into two package definitions as follows.
+
+```
+(defpackage :coalton-benchmark-mybenchmark/native
+  <native-package-clause> ...)
+
+(define-benchmark-package :coalton-benchmark-mybenchmark
+  (:local-nicknames (#:native :coalton-benchmark-mybenchmark/native)
+  <extra-package-clause> ...)
+```
+
+The idea is to put coalton code into the `/native` package, and
+benchmarking code that calls the coalton code (`define-benchmark`
+form etc.) in the package without `/native`.
+
+## Add benchmark source files
+
+```
+(cl:in-package :coalton-benchmark-mybenchmark)
+
+(define-benchmark ...)
+
+;; here, you can refer to Coalton procedures with `native:` prefix
+
+
+(cl:in-package :coalton-benchmark-mybenchmark/native
+
+(coalton-toplevel
+  ...
+  )
+
+```
+
+## Add the benchmark source files in `coalton.asd`

benchmarks/big-float.lisp

@@ -2,7 +2,7 @@
 ;;;;
 ;;;; Benchmarks for arbitrary precision floats
 
-(cl:in-package #:coalton-benchmarks)
+(cl:in-package #:coalton-benchmark-big-float)
 
 (cl:defvar *big-float-bench-precision*
   #-coalton-portable-bigfloat 10000
@@ -16,7 +16,7 @@
   (declare (optimize speed))
   (loop :repeat  *big-float-bench-iterations*
         :do (with-benchmark-sampling
-              (coalton-benchmarks/native::big-trig
+              (native::big-trig
                *big-float-bench-precision*
                (* (- (random 2)) (random 100.0d0)))))
   (report trivial-benchmark::*current-timer*))
@@ -26,7 +26,7 @@
   (declare (optimize speed))
   (loop :repeat *big-float-bench-iterations*
         :do (with-benchmark-sampling
-              (coalton-benchmarks/native::big-inv-trig
+              (native::big-inv-trig
                *big-float-bench-precision*
                (* (- (random 2)) (random 1.0d0)))))
   (report trivial-benchmark::*current-timer*))
@@ -36,7 +36,7 @@
   (declare (optimize speed))
   (loop :repeat *big-float-bench-iterations*
         :do (with-benchmark-sampling
-              (coalton-benchmarks/native::big-ln-exp
+              (native::big-ln-exp
                *big-float-bench-precision*
                (* (- (random 2)) (random 100.0d0)))))
   (report trivial-benchmark::*current-timer*))
@@ -46,7 +46,7 @@
   (declare (optimize speed))
   (loop :repeat *big-float-bench-iterations*
         :do (with-benchmark-sampling
-              (coalton-benchmarks/native::big-sqrt
+              (native::big-sqrt
                *big-float-bench-precision*
                (random 100.0d0))))
   (report trivial-benchmark::*current-timer*))
@@ -56,12 +56,12 @@
   (declare (optimize speed))
   (loop :repeat *big-float-bench-iterations*
         :do (with-benchmark-sampling
-              (coalton-benchmarks/native::big-sqrt
+              (native::big-sqrt
                *big-float-bench-precision*
                (* (- (random 2)) (random 100.0d0)))))
   (report trivial-benchmark::*current-timer*))
 
-(cl:in-package #:coalton-benchmarks/native)
+(cl:in-package #:coalton-benchmark-big-float/native)
 
 (cl:declaim (cl:optimize (cl:speed 3) (cl:safety 1)))
 

benchmarks/fibonacci.lisp

@@ -2,20 +2,20 @@
 ;;;;
 ;;;; Benchmarks for different methods of generating fibonacci numbers
 
-(cl:in-package #:coalton-benchmarks)
+(cl:in-package #:coalton-benchmark-fibonacci)
 
 (define-benchmark recursive-fib ()
   (declare (optimize speed))
   (loop :repeat 1000
         :do (with-benchmark-sampling
-              (coalton-benchmarks/native:fib 20)))
+              (native:fib 20)))
   (report trivial-benchmark::*current-timer*))
 
 (define-benchmark recursive-fib-generic ()
   (declare (optimize speed))
   (loop :repeat 1000
         :do (with-benchmark-sampling
-              (coalton-benchmarks/native:fib-generic-wrapped 20)))
+              (native:fib-generic-wrapped 20)))
   (report trivial-benchmark::*current-timer*))
 
 (define-benchmark recursive-fib-lisp ()
@@ -30,7 +30,7 @@
   (declare (optimize speed))
   (loop :repeat 1000
         :do (with-benchmark-sampling
-              (coalton-benchmarks/native:fib-monomorphized 20)))
+              (native:fib-monomorphized 20)))
   (report trivial-benchmark::*current-timer*))
 
 ;;
@@ -43,15 +43,15 @@
   (declare (optimize speed))
   (loop :repeat 1000
         :do (with-benchmark-sampling
-              (coalton-benchmarks/native:fib-generic-optional 10)))
+              (native:fib-generic-optional 10)))
   (report trivial-benchmark::*current-timer*))
 
 #+ignore
 (define-benchmark recursive-fib-monomorphized-optional ()
   (declare (optimize speed))
   (loop :repeat 1000
         :do (with-benchmark-sampling
-              (coalton-benchmarks/native:fib-monomorphized-optional 10)))
+              (native:fib-monomorphized-optional 10)))
   (report trivial-benchmark::*current-timer*))
 
 (defun lisp-fib (n)
@@ -66,7 +66,7 @@
 
   (+ (lisp-fib (- n 1)) (lisp-fib (- n 2))))
 
-(cl:in-package #:coalton-benchmarks/native)
+(cl:in-package #:coalton-benchmark-fibonacci/native)
 
 (cl:declaim (cl:optimize (cl:speed 3) (cl:safety 0)))
 

benchmarks/gabriel-benchmarks/stak.lisp

@@ -2,13 +2,13 @@
 ;;;;
 ;;;;
 
-(in-package #:coalton-benchmarks)
+(in-package #:coalton-benchmark-gabriel)
 
 (define-benchmark stak ()
   (declare (optimize speed))
   (loop :repeat 1000
         :do (with-benchmark-sampling
-              (coalton-benchmarks/native:stak 18 12 6)))
+              (native:stak 18 12 6)))
   (report trivial-benchmark::*current-timer*))
 
 (define-benchmark stak-lisp ()
@@ -53,7 +53,7 @@
 ;;;
 
 
-(cl:in-package #:coalton-benchmarks/native)
+(cl:in-package #:coalton-benchmark-gabriel/native)
 
 (cl:declaim (cl:optimize (cl:speed 3) (cl:safety 0)))
 

benchmarks/gabriel-benchmarks/tak.lisp

@@ -2,13 +2,13 @@
 ;;;;
 ;;;;
 
-(cl:in-package #:coalton-benchmarks)
+(cl:in-package #:coalton-benchmark-gabriel)
 
 (define-benchmark tak ()
   (declare (optimize speed))
   (loop :repeat 1000
         :do (with-benchmark-sampling
-              (coalton-benchmarks/native:tak 18 12 6)))
+              (native:tak 18 12 6)))
   (report trivial-benchmark::*current-timer*))
 
 (define-benchmark tak-lisp ()
@@ -26,7 +26,7 @@
                 (lisp-tak (1- y) z x)
                 (lisp-tak (1- z) x y))))
 
-(cl:in-package #:coalton-benchmarks/native)
+(cl:in-package #:coalton-benchmark-gabriel/native)
 
 (cl:declaim (cl:optimize (cl:speed 3) (cl:safety 0)))
 

benchmarks/gabriel-benchmarks/takl.lisp

@@ -2,13 +2,13 @@
 ;;;;
 ;;;;
 
-(cl:in-package #:coalton-benchmarks)
+(cl:in-package #:coalton-benchmark-gabriel)
 
 (define-benchmark takl ()
   (declare (optimize speed))
   (loop :repeat 1000
         :do (with-benchmark-sampling
-              (coalton-benchmarks/native:takl 18 12 6)))
+              (native:takl 18 12 6)))
   (report trivial-benchmark::*current-timer*))
 
 (define-benchmark takl-lisp ()
@@ -53,7 +53,7 @@
 ;;;
 
 
-(cl:in-package #:coalton-benchmarks/native)
+(cl:in-package #:coalton-benchmark-gabriel/native)
 
 (cl:declaim (cl:optimize (cl:speed 3) (cl:safety 0)))
 

benchmarks/package.lisp

@@ -1,40 +1,103 @@
 ;;;; package.lisp
 ;;;;
 ;;;; Benchmarks packages and common functions
+;;;;
+
+;;; Each bechmark group are in separate packages, benchmark-NAME
+;;; and benchmark-NAME/native.  The latter is used to define
+;;; Coalton functions, and the former calls them with benchmarking macros.
+;;;
+;;; The coalton-benchmarks package defines functions that runs each
+;;; individual benchmark group, or all benchmarks at once.
 
-(benchmark:define-benchmark-package #:coalton-benchmarks
-  (:export #:run-benchmarks
+(cl:defpackage #:coalton-benchmarks
+  (:use #:cl
+        #:trivial-benchmark)
+  (:export #:run-benchmark
+           #:run-benchmarks
            #:run-benchmarks-ci))
 
-(cl:defpackage #:coalton-benchmarks/native
+(cl:in-package #:coalton-benchmarks)
+
+(defparameter *all-benchmarks* '())
+
+(defmacro define-coalton-benchmark (name
+                                    benchmark-clauses
+                                    &rest native-clauses)
+  (let ((benchmark-package (intern (format nil "COALTON-BENCHMARK-~S" name) 'keyword))
+        (native-package (intern (format nil "COALTON-BENCHMARK-~S/NATIVE" name) 'keyword)))
+    `(progn
+       (pushnew ',benchmark-package *all-benchmarks*)
+       (defpackage ,native-package ,@native-clauses)
+       (define-benchmark-package ,benchmark-package
+         (:local-nicknames
+          (#:native ,native-package))
+         ,@benchmark-clauses))))
+
+(define-coalton-benchmark big-float
+    ()
   (:use
    #:coalton
    #:coalton-prelude
    #:coalton-library/big-float
    #:coalton-library/math)
-  (:local-nicknames (#:list #:coalton-library/list))
   (:export
    #:fib
    #:fib-fixnum
    #:fib-generic-wrapped
    #:fib-monomorphized
    #:fib-generic-optional
-   #:fib-monomorphized-optional)
+   #:fib-monomorphized-optional))
 
-  ;; gabriel-benchmarks/
+(define-coalton-benchmark fibonacci
+    ()
+  (:use
+   #:coalton
+   #:coalton-prelude
+   #:coalton-library/math)
+  (:export
+   #:fib
+   #:fib-fixnum
+   #:fib-generic-wrapped
+   #:fib-monomorphized
+   #:fib-generic-optional
+   #:fib-monomorphized-optional))
+
+(define-coalton-benchmark gabriel
+    ()
+  (:use
+   #:coalton
+   #:coalton-prelude
+   #:coalton-library/math)
+  (:local-nicknames (#:list #:coalton-library/list))
   (:export
    #:tak
    #:stak
    #:takl
    #:takr))
 
-(cl:in-package #:coalton-benchmarks)
+;;;
+;;; Running benchmarks
+;;;
+
+(defun run-benchmark (package)
+  "Run a single benchmark package.  Returns a hashtable of benchmark stats."
+  (run-package-benchmarks :package package :verbose t))
 
 (defun run-benchmarks ()
-  (run-package-benchmarks :package '#:coalton-benchmarks :verbose t))
+  "Run all benchmark packages.  Returns a hashtable of benchmark stats."
+  (labels ((merge-hash-tables (a b)
+             (loop :for b-key :being :the :hash-keys :of b
+                   :do (setf (gethash b-key a) (gethash b-key b)))
+             a))
+    (reduce #'merge-hash-tables
+            (mapcar (lambda (package)
+                      (run-package-benchmarks :package package :verbose t))
+                    (reverse *all-benchmarks*)))))
 
 (defun run-benchmarks-ci ()
-  (let ((result (run-package-benchmarks :package '#:coalton-benchmarks :verbose t)))
+  "Run all benchmark packages, and dump the result in bench.json."
+  (let ((result (run-benchmarks)))
     (with-open-file (out "bench.json" :direction :output :if-exists :supersede)
       (yason:encode
        (loop :for name :being :the :hash-keys :of result