Add benchmark and binary size tests to CI

.gitlab-ci.yml

@@ -50,3 +50,16 @@ pages:
     - public
   only:
   - master
+
+compare-benchmarks:
+  image: rustlang/rust:nightly
+  stage: test
+  allow_failure: true
+  except: [master]
+  script:
+    - apt-get update && apt-get install -y build-essential libboost-dev jq bc
+    - sh ./ci/run_benchmark.sh
+  artifacts:
+    reports:
+      junit: target/report.xml
+    paths: [target/criterion]

benches/terminator.rs

-#[macro_use]
-extern crate criterion;
-
 use criterion::*;
 use ion_shell::parser::Terminator;
+use std::time::Duration;
 
 const TEXT: &str = include_str!("test.ion");
 const EOF: &str = include_str!("herestring.ion");
@@ -22,6 +20,8 @@ fn criterion_benchmark(c: &mut Criterion) {
             },
             vec![TEXT, EOF],
         )
+        .warm_up_time(Duration::from_secs(10))
+        .measurement_time(Duration::from_secs(300))
         .throughput(|script| Throughput::Bytes(script.len() as u32)),
     );
 }

ci/run_benchmark.sh

+#!/bin/sh
+
+git checkout origin/master
+cargo bench
+cargo build --release
+PREV_SIZE=$(ls -al target/release/ion | cut -d' ' -f5)
+git reset --hard HEAD
+git checkout -
+cargo bench
+cargo build --release
+SIZE=$(ls -al target/release/ion | cut -d' ' -f5)
+
+# if lower_bound*upper_bound > 0, then we consider the benchmark "changed"
+NOISE=0.05
+JQ_FILTER="if .Median.confidence_interval.lower_bound > $NOISE or .Median.confidence_interval.upper_bound < -$NOISE then .Median.point_estimate else \"\" end"
+
+total=0
+total_worse=0
+result=""
+
+for suite in ./target/criterion/*; do
+    name=$(echo $suite | cut -d'/' -f 4)
+    worse=0
+    tests=0
+
+    testcases=""
+
+    for test in $suite/*/*/change/estimates.json; do
+        estimate=$(cat "$test" | jq -r "$JQ_FILTER" -c)
+        case "$estimate" in
+            -*)
+                inner="<failure message=\"Performance Regressed\" type=\"WARNING\">\
+                    Performance regressed by $estimate in $test\
+                </failure>"
+                worse=$((worse+1))
+            ;;
+        esac
+        testcases="$testcases<testcase id=\"$(echo "$test" | cut -d'/' -f 6)\" name=\"$(echo "$test" | cut -d'/' -f 6)\">$inner</testcase>"
+        tests=$((tests+1))
+    done
+
+    result="$result<testsuite id=\"$name\" name=\"$name\" tests=\"$tests\" failures=\"$worse\">$testcases</testsuite>"
+
+    total_worse=$((total_worse + worse))
+    total=$((total + tests))
+done
+
+binary=$(test $(echo "$PREV_SIZE * 105 / 100" | bc) -ge $SIZE; echo $?)
+result="$result\
+<testsuite id=\"size\" name=\"Binary size\" tests=\"1\" failures=\"$binary\">\
+<testcase id=\"size\" name=\"Binary size\">"
+
+total=$((total + 1))
+if [ ! "$binary" -eq "0" ]; then
+    result="$result\
+    <failure message=\"Binary size increased\" type=\"WARNING\">\
+        Binary size increased from $PREV_SIZE to $SIZE.\
+    </failure>"
+    total_worse=$((total_worse + 1))
+fi
+
+result="$result</testcase></testsuite>"
+
+result="<?xml version=\"1.0\" encoding=\"UTF-8\" ?>
+<testsuites id=\"$(date +%s)\" name=\"Performances\" tests=\"$total\" failures=\"$total_worse\">
+$result
+</testsuites>"
+
+echo $result > target/report.xml
+
+exit $(test "$total_worse" -eq "0"; echo $?)