CI: introduce shell unit-test infra

Introduce some basic infrastructure to allow running per-package unit
test jobs, as well as a library to support unit-testing shell scripts.

Packages can provide their own unit tests by adding a tests/
subdirectory to the package, this should contain a .gitlab-ci.yml file
with jobs that extend the .package-test job template as well as any
supporting data and scripts.

Signed-off-by: Caleb Connolly <caleb@postmarketos.org>
Part-of: https://gitlab.postmarketos.org/postmarketOS/pmaports/-/merge_requests/6182

.ci/lib/testlib.sh

@@ -0,0 +1,194 @@
+#!/bin/sh
+# Unit tests for the postmarketOS initramfs
+# Invoke with the path to a tests/ subdir of a package and the
+# package name as arguments. Or source it with the following
+# variables set:
+# * results_dir - second argument to the test script
+
+set -e
+
+# Colors :D
+RED="\e[31m"
+GREEN="\e[32m"
+YELLOW="\e[33m"
+BLUE="\e[34m"
+BOLDBLUE="\e[1;34m"
+RESET="\e[0m"
+
+testlib_path="$(realpath "$0")"
+
+_test_assert_failed=""
+_test_testn=1
+_test_passedn=0
+_test_failedn=0
+_test_current_test=""
+_test_statefile="$(mktemp)"
+# Set by each test
+results_dir="${results_dir:-}"
+
+test_log() {
+	printf "%s\n" "$*" >&2
+}
+
+test_info() {
+	printf "${BLUE}%s${RESET}\n" "$*" >&2
+}
+
+test_debug() {
+	printf "${YELLOW}%s${RESET}\n" "$*" >&2
+}
+
+assert_strequal() {
+	# $1: first string
+	# $2: second string
+	if [ -z "$1" ] || [ -z "$2" ]; then
+		test_log "ERROR: assert_equal: not enough arguments given"
+		exit 1
+	fi
+	if [ "$1" != "$2" ]; then
+		_test_assert_failed="$1 != $2"
+		test_log "    ❌ $_test_assert_failed"
+		return
+	fi
+	test_log "    ✅ $1 == $2"
+}
+
+assert_equal() {
+	# $1: name of variable
+	# $2: value to compare
+	if [ -z "$1" ] || [ -z "$2" ]; then
+		test_log "ERROR: assert_equal: not enough arguments given"
+		exit 1
+	fi
+	val="$(eval echo "\$$1")"
+	if [ "$val" != "$2" ]; then
+		_test_assert_failed="\$$1 != $2 (got $val)"
+		test_log "    ❌ $_test_assert_failed"
+		return
+	fi
+	test_log "    ✅ \$$1 == $2"
+}
+
+assert_unset() {
+	if [ -z "$1" ]; then
+		test_log "ERROR: assert_unset: not enough arguments given"
+		exit 1
+	fi
+	val="$(eval echo \$"$1")"
+	if [ -n "$val" ]; then
+		_test_assert_failed="\$$1 should be unset (has value '$val')"
+		test_log "    ❌ $_test_assert_failed"
+		return
+	fi
+	test_log "    ✅ \$$1 is unset"
+}
+
+# Log some expression and evaluate it
+logeval() {
+	eval "$*"
+	test_log "${YELLOW}+ $*${RESET}"
+}
+
+# Start a test. Shell scripts involved in the test should be sourced AFTER calling this function!
+start_test() {
+	if [ -n "$_test_current_test" ]; then
+		test_log "ERROR: multiple calls to start_test without calling end_test first!"
+		test_log "Current test: $_test_current_test"
+		exit 1
+	fi
+	test_log "==> Test $_test_testn: $1"
+	_test_current_test="$1"
+	set > "$_test_statefile"
+}
+
+# End a test and reset the environment.
+end_test() {
+	local msg
+	local save_assert_failed
+
+	# Restore environment
+	save_assert_failed="$_test_assert_failed"
+
+	# Splitting is intentional
+	# shellcheck disable=SC2046
+	unset $(set | grep "='" | cut -d= -f1 | grep -vE "(PATH|IFS|TERM|LANG|HOME|PWD|SHELL|USER|save_assert_failed|_test_statefile)" )
+
+	# shellcheck disable=SC1090
+	. "$_test_statefile"
+	_test_assert_failed="$save_assert_failed"
+
+	if [ -n "$_test_assert_failed" ]; then
+		msg="${RED}Failed! ❌ ${_test_assert_failed}${RESET}"
+		echo "Test $_test_testn: $_test_assert_failed" >> "$results_dir/failed"
+		_test_failedn=$((_test_failedn+1))
+		echo "$_test_failedn" > "$results_dir/failedn"
+		unset _test_assert_failed
+	else
+		msg="${GREEN}Passed!${RESET}"
+		_test_passedn=$((_test_passedn+1))
+		echo "$_test_passedn" > "$results_dir/passedn"
+	fi
+	test_log "<== Test $_test_testn: $msg $_test_current_test\n"
+	_test_testn=$((_test_testn+1))
+	_test_current_test=""
+}
+
+# Call at the end of each testsuite shell script
+end_testsuite() {
+	if [ -n "$_test_current_test" ]; then
+		test_log "${RED}ERROR: end_testsuite called without calling end_test first!${RESET}"
+		test_log "${RED}Current test: $_test_current_test${RESET}"
+		exit 1
+	fi
+	if [ -f "$results_dir/failed" ]; then
+		exit 1
+	fi
+
+	exit 0
+}
+
+run_tests() {
+	local failed test_results_base
+	local test name total passedn failedn t_passed t_failed
+
+	test_results_base="$(mktemp -d)"
+	test_log "Running tests for $2"
+
+	# FIXME: yeah this isn't quite the best way to iterate the files
+	# shellcheck disable=SC2010
+	for name in $(ls "$1" | grep "\.sh$"); do
+		test="$1/$name"
+		mkdir "$test_results_base/$name"
+		echo 0 > "$test_results_base/$name/passedn"
+		echo 0 > "$test_results_base/$name/failedn"
+		test_log
+		test_log "${BOLDBLUE}# Test suite ${name}${RESET}"
+		$test "$testlib_path" "$test_results_base/$name" || failed="$failed $name"
+		t_passed=$(cat "$test_results_base/$name/passedn")
+		t_failed=$(cat "$test_results_base/$name/failedn")
+		passedn=$((passedn+t_passed))
+		failedn=$((failedn+t_failed))
+		total=$((total+t_passed+t_failed))
+	done
+	if [ -n "$failed" ]; then
+		test_log
+		test_log "Some tests failed:"
+		for test in $failed; do
+			test_log "❌ $test"
+			test_log "    ${RED}$(cat "$test_results_base/$test/failed")${RESET}"
+		done
+	fi
+
+	if [ $passedn -lt $total ]; then
+		test_log "[${RED}$passedn/$total tests passed${RESET}]"
+	else
+		test_log "[${GREEN}$passedn/$total tests passed!${RESET}]"
+	fi
+	test_log
+
+	if [ -n "$failed" ]; then exit 1; else exit 0; fi
+}
+
+if [ $# -gt 0 ]; then
+	run_tests "$1" "$2"
+fi

.ci/shellcheck.sh

@@ -38,6 +38,7 @@ sh_files="
 	$(find . -path './main/postmarketos-mvcfg/*.sh')
 
 	$(find . -path './.ci/**.sh')
+	$(find . -path '**/tests/*.sh')
 "
 
 for file in $sh_files; do

.ci/unit-tests.sh

@@ -0,0 +1,19 @@
+#!/bin/sh -e
+# Description: run unit-tests for packages
+# https://postmarketos.org/pmb-ci
+#
+# This script only exists to allow running unit-tests locally.
+# In actual CI the package tests/ subdir has it's own YAML
+# fine defining a job which calls testlib for that package.
+# This way we can run all the package unit-tests locally but
+# have them be individual jobs in GitLab.
+
+if [ "$(id -u)" = 0 ]; then
+	set -x
+	exec su "${TESTUSER:-build}" -c "sh -e $0"
+fi
+
+find . -type d -name "tests" | while read -r testdir; do
+	pkg="$(basename "$(dirname "$testdir")")"
+	.ci/lib/testlib.sh "$testdir" "$pkg"
+done

.gitlab-ci.yml

@@ -10,6 +10,7 @@ stages:
   - lint
   - build
   - autoupdate
+  - test
 variables:
   # Default of 20 is pretty small, sometimes MRs have >20 commits in them. 100
   # seems like a reasonable balance. If this is too low, git merge-base (from ci/
@@ -27,6 +28,10 @@ workflow:
     # Run scheduled pipeline for autoupdate or manually triggered pipeline
     - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"
 
+# Some packages have their own yaml files defining tests. All of these should contain
+# only jobs which extend the .package-test job!
+include: "**/.gitlab-ci.yml"
+
 # device documentation
 wiki:
   stage: lint
@@ -196,3 +201,11 @@ auto-update:
       - log.txt
       - log_testsuite_pmaports.txt
       - pmbootstrap.cfg
+
+# Template for package test jobs defined per-package
+.package-test:
+  stage: test
+  rules:
+    - if: *mr_rule
+  before_script:
+    - .ci/lib/gitlab_prepare_ci.sh