Benchmarks for bindings

.github/workflows/benchmarks.yml

@@ -25,7 +25,7 @@ jobs:
         run: make install-sdk
       - name: Run benchmark
         shell: 'bash --noprofile --norc -eo pipefail {0}'
-        run: make report-benchmarks | tee benchmarks.txt
+        run: make performance-report | tee benchmarks.txt
       - name: Store benchmark result
         uses: benchmark-action/github-action-benchmark@v1
         with:

.github/workflows/test.yml

@@ -14,7 +14,7 @@ jobs:
       fail-fast: true
       matrix:
         racket-variant: ['BC', 'CS']
-        racket-version: ['8.3', 'stable']
+        racket-version: ['8.5', 'stable']
         experimental: [false]
         include:
           - racket-version: 'current'

Makefile

@@ -7,12 +7,15 @@ DEPS-FLAGS=--check-pkg-deps --unused-pkg-deps
 
 help:
 	@echo "install - install package along with dependencies"
+	@echo "install-sdk - install the SDK which includes developer tools"
 	@echo "remove - remove package"
+	@echo "remove-sdk - remove SDK; this will not remove SDK dependencies"
 	@echo "build - Compile libraries"
 	@echo "build-docs - Build docs"
 	@echo "build-standalone-docs - Build self-contained docs that could be hosted somewhere"
 	@echo "build-all - Compile libraries, build docs, and check dependencies"
 	@echo "clean - remove all build artifacts"
+	@echo "clean-sdk - remove all build artifacts in SDK paths"
 	@echo "check-deps - check dependencies"
 	@echo "test - run tests"
 	@echo "test-with-errortrace - run tests with error tracing"
@@ -27,6 +30,8 @@ help:
 	@echo "  definitions"
 	@echo "  macro"
 	@echo "  util"
+	@echo "  expander"
+	@echo "  compiler"
 	@echo "  probe"
 	@echo "    Note: As probe is not in qi-lib, it isn't part of"
 	@echo "    the tests run in the 'test' target."
@@ -37,9 +42,14 @@ help:
 	@echo "docs - view docs in a browser"
 	@echo "profile - Run comprehensive performance benchmarks"
 	@echo "profile-competitive - Run competitive benchmarks"
-	@echo "profile-forms - Run benchmarks for individual Qi forms"
+	@echo "profile-local - Run benchmarks for individual Qi forms"
+	@echo "profile-nonlocal - Run nonlocal benchmarks exercising many components at once"
 	@echo "profile-selected-forms - Run benchmarks for Qi forms by name (command only)"
-	@echo "report-benchmarks - Run benchmarks for Qi forms and produce results for use in CI"
+	@echo "performance-report - Run benchmarks for Qi forms and produce results for use in CI and for measuring regression"
+	@echo "  For use in regression: make performance-report > /path/to/before.json"
+	@echo "performance-regression-report - Run benchmarks for Qi forms against a reference report."
+	@echo "  make performance-regression-report REF=/path/to/before.json"
+
 
 # Primarily for use by CI.
 # Installs dependencies as well as linking this as a package.
@@ -82,6 +92,9 @@ build-standalone-docs:
 clean:
 	raco setup --fast-clean --pkgs $(PACKAGE-NAME)-{lib,test,doc,probe}
 
+clean-sdk:
+	raco setup --fast-clean --pkgs $(PACKAGE-NAME)-sdk
+
 # Primarily for use by CI, after make install -- since that already
 # does the equivalent of make setup, this tries to do as little as
 # possible except checking deps.
@@ -93,25 +106,31 @@ test:
 	raco test -exp $(PACKAGE-NAME)-{lib,test,doc,probe}
 
 test-flow:
-	racket $(PACKAGE-NAME)-test/tests/flow.rkt
+	racket -y $(PACKAGE-NAME)-test/tests/flow.rkt
 
 test-on:
-	racket $(PACKAGE-NAME)-test/tests/on.rkt
+	racket -y $(PACKAGE-NAME)-test/tests/on.rkt
 
 test-threading:
-	racket $(PACKAGE-NAME)-test/tests/threading.rkt
+	racket -y $(PACKAGE-NAME)-test/tests/threading.rkt
 
 test-switch:
-	racket $(PACKAGE-NAME)-test/tests/switch.rkt
+	racket -y $(PACKAGE-NAME)-test/tests/switch.rkt
 
 test-definitions:
-	racket $(PACKAGE-NAME)-test/tests/definitions.rkt
+	racket -y $(PACKAGE-NAME)-test/tests/definitions.rkt
 
 test-macro:
-	racket $(PACKAGE-NAME)-test/tests/macro.rkt
+	racket -y $(PACKAGE-NAME)-test/tests/macro.rkt
 
 test-util:
-	racket $(PACKAGE-NAME)-test/tests/util.rkt
+	racket -y $(PACKAGE-NAME)-test/tests/util.rkt
+
+test-expander:
+	racket -y $(PACKAGE-NAME)-test/tests/expander.rkt
+
+test-compiler:
+	racket -y $(PACKAGE-NAME)-test/tests/compiler.rkt
 
 test-probe:
 	raco test -exp $(PACKAGE-NAME)-probe
@@ -159,20 +178,27 @@ cover: coverage-check coverage-report
 cover-coveralls:
 	raco cover -b -f coveralls -p $(PACKAGE-NAME)-{lib,test}
 
-profile-forms:
-	echo "Profiling forms..."
-	racket $(PACKAGE-NAME)-sdk/profile/forms.rkt
+profile-local:
+	racket $(PACKAGE-NAME)-sdk/profile/local/report.rkt
+
+profile-loading:
+	racket $(PACKAGE-NAME)-sdk/profile/loading/report.rkt
 
 profile-selected-forms:
-	@echo "Use 'racket profile/forms.rkt' directly, with -f form-name for each form."
+	@echo "Use 'racket $(PACKAGE-NAME)-sdk/profile/local/report.rkt' directly, with -s form-name for each form."
 
 profile-competitive:
-	echo "Running competitive benchmarks..."
-	racket $(PACKAGE-NAME)-sdk/profile/competitive.rkt
+	cd $(PACKAGE-NAME)-sdk/profile/nonlocal; racket report-competitive.rkt
+
+profile-nonlocal:
+	cd $(PACKAGE-NAME)-sdk/profile/nonlocal; racket report-intrinsic.rkt -l qi
+
+profile: profile-local profile-nonlocal profile-loading
 
-profile: profile-competitive profile-forms
+performance-report:
+	@racket $(PACKAGE-NAME)-sdk/profile/report.rkt -f json
 
-report-benchmarks:
-	@racket $(PACKAGE-NAME)-sdk/profile/report.rkt
+performance-regression-report:
+	@racket $(PACKAGE-NAME)-sdk/profile/report.rkt -r $(REF)
 
-.PHONY:	help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-forms profile-selected-forms profile-competitive profile report-benchmarks
+.PHONY:	help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-expander test-compiler test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile-local profile-loading profile-selected-forms profile-competitive profile-nonlocal profile performance-report performance-regression-report

qi-doc/scribblings/field-guide.scrbl

@@ -8,18 +8,24 @@
                     racket]]
 
 @(define eval-for-docs
-  (parameterize ([sandbox-output 'string]
-                 [sandbox-error-output 'string]
-                 [sandbox-memory-limit #f])
-    (make-evaluator 'racket/base
-                    '(require qi
-                              qi/probe
-                              (only-in racket/list range)
-                              racket/string
-                              (for-syntax syntax/parse
-                                          racket/base))
-                    '(define (sqr x)
-                       (* x x)))))
+  ;; The "trusted" sandbox configuration is needed possibly
+  ;; because of the interaction of binding spaces with
+  ;; sandbox evaluator. For more context, see the Qi wiki
+  ;; "Qi Compiler Sync Sept 2 2022."
+  (call-with-trusted-sandbox-configuration
+   (lambda ()
+     (parameterize ([sandbox-output 'string]
+                    [sandbox-error-output 'string]
+                    [sandbox-memory-limit #f])
+       (make-evaluator 'racket/base
+                       '(require qi
+                                 qi/probe
+                                 (only-in racket/list range)
+                                 racket/string
+                                 (for-syntax syntax/parse
+                                             racket/base))
+                       '(define (sqr x)
+                          (* x x)))))))
 
 @title{Field Guide}
 
@@ -336,6 +342,7 @@ Another way to do it is to simply promote the expression out of the nest:
     (~> (3) (get-f 1))
   ]
 
+@;{TODO: Update this to reflect new partial application behavior}
 Now, you might, once again, expect this to be treated as a partial application template, so that this would be equivalent to @racket[(get-f 3 1)] and would raise an error. But in fact, since the expression @racket[(get-f 1)] happens to be fully qualified with all the arguments it needs, the currying employed under the hood to implement partial application in this case @seclink["Using_Racket_to_Define_Flows"]{evaluates to a function result right away}. This then receives the value @racket[3], and consequently, this expression produces the correct result.
 
 So in sum, it's perhaps best to rely on @racket[esc] in such cases to be as explicit as possible about what you mean, rather than rely on quirks of the implementation that are revealed at this boundary between two languages.

qi-doc/scribblings/forms.scrbl

@@ -7,15 +7,21 @@
                     racket]]
 
 @(define eval-for-docs
-  (parameterize ([sandbox-output 'string]
-                 [sandbox-error-output 'string]
-                 [sandbox-memory-limit #f])
-    (make-evaluator 'racket/base
-                    '(require qi
-                              (only-in racket/list range first rest)
-                              racket/string)
-                    '(define (sqr x)
-                       (* x x)))))
+  ;; The "trusted" sandbox configuration is needed possibly
+  ;; because of the interaction of binding spaces with
+  ;; sandbox evaluator. For more context, see the Qi wiki
+  ;; "Qi Compiler Sync Sept 2 2022."
+  (call-with-trusted-sandbox-configuration
+   (lambda ()
+     (parameterize ([sandbox-output 'string]
+                    [sandbox-error-output 'string]
+                    [sandbox-memory-limit #f])
+       (make-evaluator 'racket/base
+                       '(require qi
+                                 (only-in racket/list range first rest)
+                                 racket/string)
+                       '(define (sqr x)
+                          (* x x)))))))
 
 @(define diagram-eval (make-base-eval))
 @(diagram-eval '(require metapict))
@@ -346,11 +352,15 @@ Note that the symbol form uses Unicode @code{0x2225} corresponding to LaTeX's @c
 @deftogether[(
 @defform[(== flo ...)]
 @defform[(relay flo ...)]
+@defidform[#:link-target? #f ==]
+@defidform[#:link-target? #f relay]
 )]{
   Compose flows in parallel, so that inputs are passed through the corresponding @racket[flo]'s individually. The number of @racket[flo]s must be the same as the number of runtime inputs.
 
   In the common case of @code{1 × 1} @racket[flo]s (i.e. where the flows each accept one input and produce one output), the number of outputs will be the same as the number of inputs, but as @seclink["What_is_a_Flow_"]{flows can be nonlinear}, this is not necessarily the case in general.
 
+  When used in identifier form simply as @racket[==], it behaves identically to @racket[><].
+
   See also the field guide entry on the @seclink["Bindings_are_an_Alternative_to_Nonlinearity"]{relationship between bindings and nonlinearity}.
 
 @examples[
@@ -568,11 +578,14 @@ A form of generalized @racket[sieve], passing all the inputs that satisfy each
            (loop condition-flo map-flo)]
   @defform[#:link-target? #f
            (loop map-flo)]
+  @defidform[#:link-target? #f loop]
 )]{
   A simple loop for structural recursion on the input values, this applies @racket[map-flo] to the first input on each successive iteration and recurses on the remaining inputs, combining these using @racket[combine-flo] to yield the result as long as the inputs satisfy @racket[condition-flo]. When the inputs do not satisfy @racket[condition-flo], @racket[return-flo] is applied to the inputs to yield the result at that terminating step. If the condition is satisfied and there are no further values, the loop terminates naturally.
 
   If unspecified, @racket[condition-flo] defaults to @racket[#t], @racket[combine-flo] defaults to @racket[_], and @racket[return-flo] defaults to @racket[⏚].
 
+  When used in identifier form simply as @racket[loop], this behaves the same as the fully qualified version, except that the flows parametrizing the loop are expected as the initial four inputs (in the same order), and the data inputs being acted upon are expected to follow.
+
 @examples[
     #:eval eval-for-docs
     ((☯ (loop (* 2))) 1 2 3)

qi-doc/scribblings/interface.scrbl

@@ -8,16 +8,22 @@
                     syntax/parse/define]]
 
 @(define eval-for-docs
-  (parameterize ([sandbox-output 'string]
-                 [sandbox-error-output 'string]
-                 [sandbox-memory-limit #f])
-    (make-evaluator 'racket/base
-                    '(require qi
-                              (only-in racket/list range)
-                              racket/string)
-                    '(define ->string number->string)
-                    '(define (sqr x)
-                       (* x x)))))
+  ;; The "trusted" sandbox configuration is needed possibly
+  ;; because of the interaction of binding spaces with
+  ;; sandbox evaluator. For more context, see the Qi wiki
+  ;; "Qi Compiler Sync Sept 2 2022."
+  (call-with-trusted-sandbox-configuration
+   (lambda ()
+     (parameterize ([sandbox-output 'string]
+                    [sandbox-error-output 'string]
+                    [sandbox-memory-limit #f])
+       (make-evaluator 'racket/base
+                       '(require qi
+                                 (only-in racket/list range)
+                                 racket/string)
+                       '(define ->string number->string)
+                       '(define (sqr x)
+                          (* x x)))))))
 
 @title{Language Interface}
 
@@ -354,19 +360,6 @@ The second way is if you want to describe a flow using the host language instead
     (~> (3 5) add-two)
   ]
 
-Finally, note that the following case works:
-
-@examples[
-    #:eval eval-for-docs
-    (define (get-flow v)
-      (☯ (~> sqr (+ v))))
-    (~> (5) (get-flow 3))
-  ]
-
-You might expect here that the expression @racket[(get-flow 3)] would be treated as a @seclink["Templates_and_Partial_Application"]{partial application template}, so that the value @racket[5] would be provided to it as @racket[(get-flow 5 3)], resulting in an error. The reason this isn't what happens is that the partial application behavior in Qi when no argument positions have been indicated is implemented using currying rather than as a template application, and Racket's @racket[curry] and @racket[curryr] functions happen to evaluate to a result immediately if the maximum expected arguments have been provided. Thus, in this case, the @racket[(get-flow 3)] expression is first evaluated to produce a resulting flow which then receives the value @racket[5].
-
-So, function applications where all of the arguments are provided syntactically, and which produce functions as their result, may be used as if they were simple function identifiers, and @racket[esc] may be left out.
-
 @subsection{Using Racket Macros as Flows}
 
 Flows are expected to be @seclink["What_is_a_Flow_"]{functions}, and so you cannot naively use a macro as a flow. But there are many ways in which you can. If you'd just like to use such a macro in a one-off manner, see @secref["Converting_a_Macro_to_a_Flow"] for an ad hoc way to do this. But a simpler and more complete way in many cases is to first register the macro (or any number of such macros) using @racket[define-qi-foreign-syntaxes] prior to use.

qi-doc/scribblings/intro.scrbl

@@ -7,15 +7,21 @@
                     racket]]
 
 @(define eval-for-docs
-  (parameterize ([sandbox-output 'string]
-                 [sandbox-error-output 'string]
-                 [sandbox-memory-limit #f])
-    (make-evaluator 'racket/base
-                    '(require qi
-                              (only-in racket/list range)
-                              racket/string)
-                    '(define (sqr x)
-                       (* x x)))))
+  ;; The "trusted" sandbox configuration is needed possibly
+  ;; because of the interaction of binding spaces with
+  ;; sandbox evaluator. For more context, see the Qi wiki
+  ;; "Qi Compiler Sync Sept 2 2022."
+  (call-with-trusted-sandbox-configuration
+   (lambda ()
+     (parameterize ([sandbox-output 'string]
+                    [sandbox-error-output 'string]
+                    [sandbox-memory-limit #f])
+       (make-evaluator 'racket/base
+                       '(require qi
+                                 (only-in racket/list range)
+                                 racket/string)
+                       '(define (sqr x)
+                          (* x x)))))))
 
 @title{Introduction and Usage}
 

qi-doc/scribblings/macros.scrbl

@@ -9,16 +9,22 @@
                     syntax/parse/define]]
 
 @(define eval-for-docs
-  (parameterize ([sandbox-output 'string]
-                 [sandbox-error-output 'string]
-                 [sandbox-memory-limit #f])
-    (make-evaluator 'racket/base
-                    '(require qi
-                              (only-in racket/list range first rest)
-                              (for-syntax syntax/parse racket/base)
-                              racket/string)
-                    '(define (sqr x)
-                       (* x x)))))
+  ;; The "trusted" sandbox configuration is needed possibly
+  ;; because of the interaction of binding spaces with
+  ;; sandbox evaluator. For more context, see the Qi wiki
+  ;; "Qi Compiler Sync Sept 2 2022."
+  (call-with-trusted-sandbox-configuration
+   (lambda ()
+     (parameterize ([sandbox-output 'string]
+                    [sandbox-error-output 'string]
+                    [sandbox-memory-limit #f])
+       (make-evaluator 'racket/base
+                       '(require qi
+                                 (only-in racket/list range first rest)
+                                 (for-syntax syntax/parse racket/base)
+                                 racket/string)
+                       '(define (sqr x)
+                          (* x x)))))))
 
 @title[#:tag "Qi_Macros"]{Qi Macros}