Merge branch 'rusty-tests' into 'master'

Convert test runner to rust

See merge request redox-os/relibc!333

Cargo.toml

@@ -10,7 +10,7 @@ crate-type = ["staticlib"]
 
 [workspace]
 members = ["src/crt0", "src/crti", "src/crtn", "src/ld_so"]
-exclude = ["core_io", "ralloc"]
+exclude = ["core_io", "ralloc", "tests"]
 
 [build-dependencies]
 cbindgen = "0.13.2"

redoxer.sh

@@ -13,5 +13,5 @@ then
 fi
 
 export CARGO_TEST="redoxer"
-export TEST_RUNNER="redoxer exec --folder . -- sh --"
+export TEST_RUNNER="redoxer exec --folder . --"
 redoxer env make "$@"

tests/.gitignore

-/bins_static/
-/bins_dynamic/
+/bins_*/
 /gen/
 /*.out

tests/Cargo.lock

+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+[[package]]
+name = "relibc-tests"
+version = "0.1.0"

tests/Cargo.toml

+[package]
+name = "relibc-tests"
+version = "0.1.0"
+authors = ["Jeremy Soller <jeremy@system76.com>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]

tests/Makefile

@@ -158,7 +158,8 @@ EXPECT_BINS=$(patsubst %,bins_static/%,$(EXPECT_NAMES))
 EXPECT_BINS+=$(patsubst %,bins_dynamic/%,$(EXPECT_NAMES))
 EXPECT_BINS+=$(patsubst %,bins_dynamic/%,$(DYNAMIC_ONLY_NAMES))
 
-TEST_RUNNER?=sh --
+CARGO_TEST?=cargo
+TEST_RUNNER?=
 
 .PHONY: all clean run expected verify
 
@@ -184,8 +185,11 @@ expected: | $(EXPECT_BINS)
 		"$${bin}" test args > "expected/$${bin}.stdout" 2> "expected/$${bin}.stderr" || exit $$?; \
 	done
 
-verify: | $(EXPECT_BINS)
-	$(TEST_RUNNER) ./verify.sh $(EXPECT_BINS)
+bins_verify/relibc-tests: src/main.rs
+	$(CARGO_TEST) build --release --bin relibc-tests --out-dir bins_verify -Z unstable-options
+
+verify: bins_verify/relibc-tests | $(EXPECT_BINS)
+	$(TEST_RUNNER) $< $(EXPECT_BINS)
 
 FLAGS=\
 	-std=c11 \

tests/src/main.rs

+use std::{
+    env, fs,
+    io::{self, Write},
+    path::PathBuf,
+    process::{self, Command, ExitStatus},
+};
+
+fn expected(bin: &str, kind: &str, generated: &[u8], status: ExitStatus) -> Result<(), String> {
+    let mut expected_file = PathBuf::from(format!("expected/{}.{}", bin, kind));
+    if !expected_file.exists() {
+        expected_file = PathBuf::from(format!(
+            "expected/{}.{}",
+            bin.replace("bins_static", "").replace("bins_dynamic", ""),
+            kind
+        ));
+    }
+
+    let expected = match fs::read(&expected_file) {
+        Ok(ok) => ok,
+        Err(err) => {
+            return Err(format!(
+                "{} failed to read {}: {}",
+                bin,
+                expected_file.display(),
+                err
+            ));
+        }
+    };
+
+    if expected != generated {
+        println!("# {}: {}: expected #", bin, kind);
+        io::stdout().write(&expected).unwrap();
+
+        println!("# {}: {}: generated #", bin, kind);
+        io::stdout().write(generated).unwrap();
+
+        return Err(format!(
+            "{} failed - retcode {}, {} mismatch",
+            bin, status, kind
+        ));
+    }
+
+    Ok(())
+}
+
+fn main() {
+    let mut failures = Vec::new();
+
+    for bin in env::args().skip(1) {
+        println!("# {} #", bin);
+
+        match Command::new(&bin).arg("test").arg("args").output() {
+            Ok(output) => {
+                if let Err(failure) = expected(&bin, "stdout", &output.stdout, output.status) {
+                    println!("{}", failure);
+                    failures.push(failure);
+                }
+
+                if let Err(failure) = expected(&bin, "stderr", &output.stderr, output.status) {
+                    println!("{}", failure);
+                    failures.push(failure);
+                }
+            }
+            Err(err) => {
+                let failure = format!("{}: failed to execute: {}", bin, err);
+                println!("{}", failure);
+                failures.push(failure);
+            }
+        }
+    }
+
+    if !failures.is_empty() {
+        println!("# FAILURES #");
+        for failure in failures {
+            println!("{}", failure);
+        }
+        process::exit(1);
+    }
+}

tests/verify.sh

-#!/bin/sh
-
-rm -rf gen || exit 1
-mkdir -p gen || exit 1
-
-summary=""
-
-while [ "$#" -gt 0 ]
-do
-    bin="$1"
-    shift
-
-    echo "# ${bin} #"
-    mkdir -p "gen/$(dirname ${bin})" || exit 1
-
-    "${bin}" test args > "gen/${bin}.stdout" 2> "gen/${bin}.stderr"
-    retcode="$?"
-    status=""
-
-    for output in stdout stderr
-    do
-        gen="$(sha256sum "gen/${bin}.${output}" | cut -d " " -f 1)"
-
-        # look for expected output file that is specific to binary type (either static or dynamic)
-        expected_file="expected/${bin}.${output}"
-        if [ ! -e $expected_file ]
-        then
-            # if unable to find above, the expected output file is the same to both static and dynamic binary
-            name=$(echo $bin | cut -d "/" -f2-)
-            expected_file="expected/${name}.${output}"
-        fi
-        expected="$(sha256sum "${expected_file}" | cut -d " " -f 1)"
-        if [ "${gen}" != "${expected}" ]
-        then
-            echo "# ${bin}: ${output}: expected #"
-            cat "${expected_file}"
-
-            echo "# ${bin}: ${output}: generated #"
-            cat "gen/${bin}.${output}"
-
-            # FIXME: Make diff available on Redox
-            if [ $(uname) != "Redox" ]
-            then
-                echo "# ${bin}: ${output}: diff #"
-                diff --color -u "${expected_file}" "gen/${bin}.${output}"
-            fi
-
-            status="${bin} failed - retcode ${retcode}, ${output} mismatch"
-            summary="${summary}${status}\n"
-        fi
-    done
-
-    if [ -n "${status}" ]
-    then
-        echo "# ${status} #"
-    fi
-done
-
-if [ -n "$summary" ]
-then
-    echo -e "$summary"
-    exit 1
-fi