Initial JMH setup (#280)

* wip

* more engineering

* minor

* working locally

* more

* try now

* retry

* again

* more

* more

* more

* minor

* working now

* more

* again

* fixes

* fixes

* more

* more

* improve instructions

* cleanup

* change profile id to benchmarks

* minor fix

.github/workflows/ci.yaml

@@ -37,6 +37,8 @@ jobs:
           distribution: 'temurin'
           java-version: '${{ matrix.version }}'
           cache: maven
+      - name: Check Style
+        run:  mvn -B spotless:check -Pbenchmarks -Pfuzz
       - name: Test Java
         run: mvn -B install
       - name: Verify that nightly fuzz compiles

.github/workflows/perf.yaml

@@ -0,0 +1,89 @@
+name: Perf
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+
+jobs:
+  perf-main:
+    name: JMH tests on baseline
+    runs-on: ubuntu-latest
+    outputs:
+      result-link: "https://nightly.link/dylibso/chicory/actions/artifacts/${{ steps.artifact-upload-step.outputs.artifact-id }}.zip"
+    steps:
+      - name: Checkout sources for this branch
+        uses: actions/checkout@v4
+        with:
+          ref: main
+      - name: Checkout testsuite
+        uses: actions/checkout@v4
+        with:
+          repository: WebAssembly/testsuite
+          path: testsuite
+          # The ref needs to stay in sync with the default value in test-gen-plugin
+          ref: c2a67a575ddc815ff2212f68301d333e5e30a923
+      - name: Set up Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '21'
+          cache: maven
+
+      - name: Build JMH jars on main
+        run: ./scripts/build-jmh.sh
+      - name: Run JMH
+        run: ./scripts/run-jmh.sh
+      - uses: actions/upload-artifact@v4
+        id: artifact-upload-step
+        with:
+          name: main
+          path: jmh-result.json
+
+  perf-current:
+    name: JMH tests on current branch
+    runs-on: ubuntu-latest
+    outputs:
+      result-link: "https://nightly.link/dylibso/chicory/actions/artifacts/${{ steps.artifact-upload-step.outputs.artifact-id }}.zip"
+    steps:
+      - name: Checkout sources for this branch
+        uses: actions/checkout@v4
+      - name: Checkout testsuite
+        uses: actions/checkout@v4
+        with:
+          repository: WebAssembly/testsuite
+          path: testsuite
+          # The ref needs to stay in sync with the default value in test-gen-plugin
+          ref: c2a67a575ddc815ff2212f68301d333e5e30a923
+      - name: Set up Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '21'
+          cache: maven
+
+      - name: Build JMH jars
+        run: ./scripts/build-jmh.sh
+      - name: Run JMH
+        run: ./scripts/run-jmh.sh
+      - uses: actions/upload-artifact@v4
+        id: artifact-upload-step
+        with:
+          name: current
+          path: jmh-result.json
+
+  perf-show-results:
+    name: Show JMH tests results
+    runs-on: ubuntu-latest
+    needs:
+      - perf-main
+      - perf-current
+    steps:
+      - name: Produce a link to the visualization
+        run: echo "./scripts/show-results.sh ${{ needs.perf-main.outputs.result-link }} ${{ needs.perf-current.outputs.result-link }}"
+      - name: Show the result in the summary
+        run: |
+          echo "Visualize JMH results using this script from the project root:" >> $GITHUB_STEP_SUMMARY
+          echo -e '<pre><code>./scripts/show-results.sh ci ${{ needs.perf-main.outputs.result-link }} ${{ needs.perf-current.outputs.result-link }}</code></pre>' >> $GITHUB_STEP_SUMMARY
+          echo "Or use a pre-built container image:" >> $GITHUB_STEP_SUMMARY
+          echo -e '<pre><code>docker run --rm -it -p 3000:3000 docker.io/andreatp/chicory-show-jmh ci ${{ needs.perf-main.outputs.result-link }} ${{ needs.perf-current.outputs.result-link }}</code></pre>' >> $GITHUB_STEP_SUMMARY

.gitignore

@@ -4,4 +4,7 @@ testsuite/
 .classpath
 .project
 .settings
-
+dependency-reduced-pom.xml
+jmh-result.json
+scripts/jmh-tmp
+main/

cli/pom.xml

@@ -46,7 +46,6 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.5.2</version>
         <executions>
           <execution>
             <goals>

jmh/Dockerfile

@@ -0,0 +1,10 @@
+# Build from the repository root using the command:
+# docker build -f jmh/Dockerfile . -t docker.io/andreatp/chicory-show-jmh
+FROM node
+
+RUN apt update && apt install zip wget -y
+RUN npm install http-server -g
+
+COPY scripts/show-results.sh .
+
+ENTRYPOINT ["./show-results.sh"]

jmh/README.md

@@ -0,0 +1,54 @@
+# jmh
+
+This module is used to run JMH performance tests on Chicory and make it easy to compare results.
+
+## Dependencies
+
+There a few additional tools that are needed to see the benchmark results:
+
+- `wget`
+- `http-server` install with: `npm install http-server -g`
+
+## Locally
+
+### Prepare the baseline
+
+The baseline is produced out of main, prepare it by running:
+
+```bash
+./scripts/build-jmh-main.sh
+./scripts/run-jmh-main.sh
+```
+
+### Run the JMH benchmarks
+
+From the current local folder you can produce the JMH results running:
+
+```bash
+./scripts/build-jmh.sh
+./scripts/run-jmh.sh
+```
+
+### Show the results
+
+```bash
+./scripts/show-results.sh local
+```
+and open the generated link.
+
+## From GH Actions
+
+If you run the JMH tests from a GH Action:
+
+- Open the relevant GH Action Summary view of the Perf run
+- Follow the steps described
+
+```bash
+./scripts/show-results.sh ci <link-to-baseline-results> <link-to-current-results>
+```
+
+Or using a container image:
+
+```bash
+docker run --rm -it -p 3000:3000 docker.io/andreatp/chicory-show-jmh ci <link-to-baseline-results> <link-to-current-results>
+```

jmh/pom.xml

@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>com.dylibso.chicory</groupId>
+    <artifactId>chicory</artifactId>
+    <version>999-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+  <artifactId>jmh</artifactId>
+  <packaging>jar</packaging>
+  <name>Chicory - JMH</name>
+  <description>JMH tests for the Chicory runtime</description>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.dylibso.chicory</groupId>
+      <artifactId>runtime</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.openjdk.jmh</groupId>
+      <artifactId>jmh-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.openjdk.jmh</groupId>
+      <artifactId>jmh-generator-annprocess</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <annotationProcessorPaths>
+            <path>
+              <groupId>org.openjdk.jmh</groupId>
+              <artifactId>jmh-generator-annprocess</artifactId>
+            </path>
+          </annotationProcessorPaths>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <executions>
+          <execution>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <phase>package</phase>
+            <configuration>
+              <finalName>benchmarks</finalName>
+              <transformers>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                  <mainClass>org.openjdk.jmh.Main</mainClass>
+                </transformer>
+              </transformers>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

jmh/src/main/java/com/dylibso/chicory/bench/BenchmarkFactorialExecution.java

@@ -0,0 +1,50 @@
+package com.dylibso.chicory.bench;
+
+import com.dylibso.chicory.runtime.ExportFunction;
+import com.dylibso.chicory.runtime.Module;
+import com.dylibso.chicory.wasm.types.Value;
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+@State(Scope.Benchmark)
+@Warmup(iterations = 2)
+@Measurement(iterations = 5)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@Fork(1)
+public class BenchmarkFactorialExecution {
+
+    @Param({"5", "1000"})
+    private int input;
+
+    ExportFunction iterFact;
+
+    @Setup
+    public void setup() throws IOException {
+        var factorial =
+                Module.builder(
+                                new File(
+                                        "wasm-corpus/src/test/resources/compiled/iterfact.wat.wasm"))
+                        .build()
+                        .instantiate();
+        iterFact = factorial.export("iterFact");
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.Throughput)
+    public void benchmark(Blackhole bh) {
+        bh.consume(iterFact.apply(Value.i32(input)));
+    }
+}

jmh/src/main/java/com/dylibso/chicory/bench/BenchmarkParsing.java

@@ -0,0 +1,46 @@
+package com.dylibso.chicory.bench;
+
+import com.dylibso.chicory.runtime.Module;
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.io.FileUtils;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+@State(Scope.Benchmark)
+@Warmup(iterations = 2)
+@Measurement(iterations = 5)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@Fork(1)
+public class BenchmarkParsing {
+
+    @Param({
+        "wasm-corpus/src/test/resources/compiled/basic.c.wasm",
+        "wasm-corpus/src/test/resources/compiled/javy-demo.js.javy.wasm"
+    })
+    private String fileName;
+
+    byte[] memoryMappedFile;
+
+    @Setup
+    public void setup() throws IOException {
+        memoryMappedFile = FileUtils.readFileToByteArray(new File(fileName));
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.Throughput)
+    public void benchmark(Blackhole bh) {
+        bh.consume(Module.builder(memoryMappedFile).build());
+    }
+}