diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..630253719 --- /dev/null +++ b/.gitignore @@ -0,0 +1,115 @@ +.*.swp + +# Failed patches +*.rej +*.orig + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# Visual Studio Code +.vscode/ + +# Pytest +.pytest_cache + +# JetBrains IDEs (PyCharm, etc) +.idea diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..19c22f7cd --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,69 @@ +--- + +# global settings +image: alpine:3.8 +after_script: + - .gitlab-ci/move_logs.sh $CI_PROJECT_DIR +stages: + - first + - second + +# device documentation +wiki: + stage: first + before_script: + - apk -q add python3 + script: + - .gitlab-ci/check_devices_in_wiki.py --booting + +# static code analysis (shellcheck is not in Alpine, so we use Debian) +py-sh-static: + stage: first + image: python:3.6-slim-stretch + before_script: + - apt -q update >/dev/null + - apt -y install flake8 shellcheck >/dev/null + script: + - .gitlab-ci/static_code_analysis.sh + +# aports checks (generic) +aports-static: + stage: first + before_script: + - .gitlab-ci/install_pmbootstrap.sh pytest + script: + - su pmos -c ".gitlab-ci/run_testcases.sh + -m 'not pmaports_upstream_compat'" + artifacts: + when: on_failure + paths: + - log.txt + - log_testsuite_pmaports.txt + - pmbootstrap.cfg + expire_in: 1 week + +# aports checks (upstream compatibility) +aports-static-upstream: + stage: second + before_script: + - .gitlab-ci/install_pmbootstrap.sh pytest + script: + - su pmos -c ".gitlab-ci/run_testcases.sh + -m 'pmaports_upstream_compat'" + artifacts: + when: on_failure + paths: + - log.txt + - log_testsuite_pmaports.txt + - pmbootstrap.cfg + expire_in: 1 week + only: + - master + +# build changed aports +aports-build: + stage: second + before_script: + - .gitlab-ci/install_pmbootstrap.sh git + script: + - su pmos -c ".gitlab-ci/build_changed_aports.py" diff --git a/.gitlab-ci/build_changed_aports.py b/.gitlab-ci/build_changed_aports.py new file mode 100755 index 000000000..5beb93eb4 --- /dev/null +++ b/.gitlab-ci/build_changed_aports.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# Copyright 2018 Oliver Smith +# SPDX-License-Identifier: GPL-3.0-or-later + +import os +import subprocess +import sys + + +def get_pmaports_dir(): + return os.path.realpath(os.path.join(os.path.dirname(__file__) + "/..")) + + +def run_git(parameters): + """ Run git in the pmaports folder and return the output """ + cmd = ["git", "-C", get_pmaports_dir()] + parameters + return subprocess.check_output(cmd).decode() + + +def run_pmbootstrap(parameters): + """ Run pmbootstrap with the pmaports folder as --aports """ + cmd = ["pmbootstrap", "--aports", get_pmaports_dir()] + parameters + process = subprocess.Popen(cmd) + process.communicate() + if process.returncode != 0: + print("** Building failed") + exit(1) + + +def get_changed_files(): + """ Get all changed files and print them, as well as the branch and the + commit that was used for the diff. + :returns: list of changed files + """ + # Current branch + branch = run_git(["rev-parse", "--abbrev-ref", "HEAD"])[:-1] + print("branch: " + branch) + + # Commit to diff against + commit = "HEAD~1" + if branch != "master": + commit = run_git(["merge-base", "master", "HEAD"])[:-1] + print("comparing HEAD with: " + commit) + + # Changed files + ret = run_git(["diff", "--name-only", commit, "HEAD"]).splitlines() + print("changed file(s):") + for file in ret: + print(" " + file) + return ret + + +def get_changed_packages(): + files = get_changed_files() + ret = set() + for file in files: + # Skip files in the root folder of pmaports as well as folders + # beginning with a dot (.gitlab-ci/) + if "/" not in file or file.startswith("."): + continue + + # Add to the ret set (removes duplicated automatically) + ret.add(file.split("/")[1]) + + if len(ret) > 10: + print("ERROR: Too many packages have changed!") + print("This is a sanity check, so we don't end up building packages" + " that have not been modified. CI won't run for more than one" + " hour anyway.") + print("If you see this message on your personal fork of the" + " pmbootstrap repository, try to update your fork's master" + " branch to the upstream master branch.") + sys.exit(1) + return ret + + +def check_build(packages): + # Initialize build environment with less logging + run_pmbootstrap(["build_init"]) + run_pmbootstrap(["--details-to-stdout", "build", "--strict"] + + list(packages)) + + +if __name__ == "__main__": + packages = get_changed_packages() + + if len(packages) == 0: + print("no aports changed in this branch") + else: + print("building in strict mode: " + ", ".join(packages)) + check_build(packages) diff --git a/.gitlab-ci/check_devices_in_wiki.py b/.gitlab-ci/check_devices_in_wiki.py new file mode 100755 index 000000000..2c272c086 --- /dev/null +++ b/.gitlab-ci/check_devices_in_wiki.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +# Copyright 2018 Oliver Smith +# SPDX-License-Identifier: GPL-3.0-or-later + +import argparse +import glob +import os +import sys +import urllib.request + + +def get_devices(): + """:returns: list of all devices""" + ret = [] + pmaports = (os.path.realpath(os.path.join(os.path.dirname(__file__) + + "/.."))) + for path in glob.glob(pmaports + "/device/device-*/"): + device = os.path.dirname(path).split("device-", 1)[1] + ret.append(device) + return sorted(ret) + + +def get_wiki_devices_html(path): + """:param path: to a local file with the saved content of the devices wiki + page or None to download a fresh copy + :returns: HTML of the page, split into booting and not booting: + {"booting": "\n\n

These..."}""" + content = "" + if path: + # Read file + with open(path, encoding="utf-8") as handle: + content = handle.read() + else: + # Download wiki page + url = "http://wiki.postmarketos.org/wiki/Devices" + content = urllib.request.urlopen(url).read().decode("utf-8") + + # Split into booting and not booting + split = content.split("") + + if len(split) != 2: + print("*** Failed to parse wiki page") + sys.exit(2) + return {"booting": split[0], "not_booting": split[1]} + + +def check_device(device, html, is_booting): + """:param is_booting: require the device to be in the booting section, not + just anywhere in the page (i.e. in the not booting + table). + :returns: True when the device is in the appropriate section.""" + if device in html["booting"]: + return True + if device in html["not_booting"]: + if is_booting: + print(device + ": still in 'not booting' section (if this is a" + " merge request, your device should be in the booting" + " section already)") + return False + return True + + print(device + ": not in the wiki yet.") + return False + + +def main(): + # Parse arguments + parser = argparse.ArgumentParser() + parser.add_argument("--booting", help="devices must be in the upper table," + " being in the 'not booting' table below is not" + " enough (all devices in pmaports master and stable" + " branches and should be in the upper table)", + action="store_true") + parser.add_argument("--path", help="instead of downloading the devices" + " page from the wiki, use a local HTML file", + default=None) + args = parser.parse_args() + + # Check all devices + html = get_wiki_devices_html(args.path) + error = False + for device in get_devices(): + if not check_device(device, html, args.booting): + error = True + + # Ask to adjust the wiki + if error: + print("*** Wiki check failed!") + print("Thank you for porting postmarketOS to a new device! \o/") + print("") + print("Now it's time to add some documentation:") + print("1) Create a device specific wiki page as described here:") + print(" ") + print("2) Add your device to the overview matrix:") + print(" ") + print("3) Run these tests again with an empty commit in your MR:") + print(" $ git commit --allow-empty -m 'run tests again'") + print("") + print("Please take the time to do these steps. It will make your") + print("precious porting efforts visible for others, and allow them") + print("not only to use what you have created, but also to build upon") + print("it more easily. Many times one person did a port with basic") + print("functionallity, and then someone else jumped in and") + print("contributed major new features.") + return 1 + else: + print("*** Wiki check successful!") + return 0 + + +sys.exit(main()) diff --git a/.gitlab-ci/install_pmbootstrap.sh b/.gitlab-ci/install_pmbootstrap.sh new file mode 100755 index 000000000..38657640e --- /dev/null +++ b/.gitlab-ci/install_pmbootstrap.sh @@ -0,0 +1,36 @@ +#!/bin/sh -e +# Copyright 2018 Oliver Smith +# SPDX-License-Identifier: GPL-3.0-or-later +# usage: install_pmbootstrap.sh [ADDITIONAL_PACKAGE, ...] + +# Config: pmbootstrap tag (or branch) +tag="feature/split-aports" + +# Get download URL and pmaports path +url="https://gitlab.com/postmarketOS/pmbootstrap/-/archive/$tag/pmbootstrap-$tag.tar.bz2" +pmaports="$(cd $(dirname $0)/..; pwd -P)" + +# Set up depends and binfmt_misc +depends="coreutils openssl python3 sudo $@" +echo "Installing dependencies: $depends" +apk -q add $depends +mount -t binfmt_misc none /proc/sys/fs/binfmt_misc + +# Create pmos user +echo "Creating pmos user" +adduser -D pmos +chown -R pmos:pmos . +echo 'pmos ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers + +# Download pmbootstrap (to /tmp/pmbootstrap) +echo "Downloading pmbootstrap ($tag): $url" +cd /tmp +wget -q -O "pmb.tar.bz2" "$url" +tar -xf "pmb.tar.bz2" +mv pmbootstrap-* pmbootstrap + +# Install to $PATH and init +ln -s /tmp/pmbootstrap/pmbootstrap.py /usr/local/bin/pmbootstrap +echo "Initializing pmbootstrap" +su pmos -c "yes '' | pmbootstrap -q --aports '$pmaports' init" +echo "" diff --git a/.gitlab-ci/move_logs.sh b/.gitlab-ci/move_logs.sh new file mode 100755 index 000000000..cc02fe4ff --- /dev/null +++ b/.gitlab-ci/move_logs.sh @@ -0,0 +1,16 @@ +#!/bin/sh -e +# Copyright 2018 Oliver Smith +# SPDX-License-Identifier: GPL-3.0-or-later + +if [ -z "$1" ]; then + echo "usage: $(basename $0) \$CI_PROJECT_DIR" + exit 1 +fi + +for log in \ + /home/pmos/.local/var/pmbootstrap/log.txt \ + /home/pmos/.local/var/pmbootstrap/log_testsuite_pmaports.txt \ + /home/pmos/.config/pmbootstrap.cfg \ +; do + [ -e "$log" ] && mv "$log" "$1" +done diff --git a/.gitlab-ci/run_testcases.sh b/.gitlab-ci/run_testcases.sh new file mode 100755 index 000000000..21da31376 --- /dev/null +++ b/.gitlab-ci/run_testcases.sh @@ -0,0 +1,35 @@ +#!/bin/sh -e +# Copyright 2018 Oliver Smith +# SPDX-License-Identifier: GPL-3.0-or-later + +# Require pmbootstrap +if ! command -v pmbootstrap > /dev/null; then + echo "ERROR: pmbootstrap needs to be installed." + exit 1 +fi + +# Wrap pmbootstrap to use this repository for --aports +pmaports="$(cd $(dirname $0)/..; pwd -P)" +_pmbootstrap="$(command -v pmbootstrap)" +pmbootstrap() { + "$_pmbootstrap" --aports="$pmaports" "$@" +} + +# Make sure that the work folder format is up to date, and that there are no +# mounts from aborted test cases (pmbootstrap#1595) +pmbootstrap work_migrate +pmbootstrap -q shutdown + +# Make sure we have a valid device (pmbootstrap#1128) +device="$(pmbootstrap config device)" +deviceinfo="$pmaports/device/device-$device/deviceinfo" +if ! [ -e "$deviceinfo" ]; then + echo "ERROR: Could not find deviceinfo file for selected device '$device'." + echo "Expected path: $deviceinfo" + echo "Maybe you have switched to a branch where your device does not exist?" + echo "Use 'pmbootstrap config device qemu-amd64' to switch to a valid device." + exit 1 +fi + +# Run testcases +pytest -vv -x --tb=native "$pmaports/.gitlab-ci/testcases" "$@" diff --git a/.gitlab-ci/static_code_analysis.sh b/.gitlab-ci/static_code_analysis.sh new file mode 100755 index 000000000..734c3b071 --- /dev/null +++ b/.gitlab-ci/static_code_analysis.sh @@ -0,0 +1,58 @@ +#!/bin/sh -e +# Copyright 2018 Oliver Smith +# SPDX-License-Identifier: GPL-3.0-or-later + +set -e +DIR="$(cd "$(dirname "$0")" && pwd -P)" +cd "$DIR/.." + +# Find CHANGEMEs in APKBUILDs +if grep -qr '(CHANGEME!)' device; then + echo "ERROR: Please replace '(CHANGEME!)' in the following files:" + grep --color=always -r '(CHANGEME!)' device + exit 1 +fi + +# Shell: shellcheck +sh_files=" + ./main/postmarketos-base/firmwareload.sh + ./main/postmarketos-mkinitfs/init.sh.in + ./main/postmarketos-mkinitfs/init_functions.sh + ./main/postmarketos-mkinitfs-hook-debug-shell/20-debug-shell.sh + ./main/postmarketos-update-kernel/update-kernel.sh + ./main/postmarketos-android-recovery-installer/build_zip.sh + ./main/postmarketos-android-recovery-installer/pmos_chroot + ./main/postmarketos-android-recovery-installer/pmos_install + ./main/postmarketos-android-recovery-installer/pmos_install_functions + ./main/postmarketos-android-recovery-installer/pmos_setpw + ./main/postmarketos-android-recovery-installer/update-binary + ./main/mdss-fb-init-hack/mdss-fb-init-hack.sh + ./main/postmarketos-ui-hildon/postmarketos-ui-hildon.post-install + $(find . -path './main/postmarketos-ui-hildon/*.sh') + $(find . -name '*.trigger') + $(find . -path './main/devicepkg-dev/*.sh') + + $(find . -path '.gitlab-ci/*.sh') +" +for file in ${sh_files}; do + echo "Test with shellcheck: $file" + cd "$DIR/../$(dirname "$file")" + shellcheck -e SC1008 -x "$(basename "$file")" +done + +# Python: flake8 +# E501: max line length +# F401: imported, but not used (false positive: add_pmbootstrap_to_import_path) +# E722: do not use bare except +cd "$DIR/.." + +echo "Test with flake8: testcases" +# shellcheck disable=SC2086 +flake8 --ignore E501,F401,E722,W504,W605 $(find . -path "./.gitlab-ci/testcases/*.py") + +echo "Test with flake8: all other Python files" +# shellcheck disable=SC2086 +flake8 --ignore E501,E722,W504,W605 $(find . -not -path './.gitlab-ci/testcases/*' -a -name '*.py') + +# Done +echo "Success!" diff --git a/.gitlab-ci/testcases/add_pmbootstrap_to_import_path/__init__.py b/.gitlab-ci/testcases/add_pmbootstrap_to_import_path/__init__.py new file mode 100755 index 000000000..4a69d12aa --- /dev/null +++ b/.gitlab-ci/testcases/add_pmbootstrap_to_import_path/__init__.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# Copyright 2018 Oliver Smith +# SPDX-License-Identifier: GPL-3.0-or-later + +import shutil +import sys +import os + + +def path_pmbootstrap(): + """ Find the pmbootstrap installation folder, so we can import the Python + code from there. + returns: pmbootstrap installation folder + """ + # Find 'pmbootstrap' executable + bin = shutil.which("pmbootstrap") + if not bin: + print("ERROR: 'pmbootstrap' not found in $PATH") + sys.exit(1) + + # Resolve the symlink and verify the folder + dir = os.path.dirname(os.path.realpath(bin)) + if os.path.exists(dir + "/pmb/__init__.py"): + return dir + + # Symlink not set up properly + print("ERROR: 'pmbootstrap' is not a symlink to pmbootstrap.py") + sys.exit(1) + + +# Add pmbootstrap dir to import path +sys.path.append(os.path.realpath(path_pmbootstrap())) diff --git a/.gitlab-ci/testcases/test_aportgen.py b/.gitlab-ci/testcases/test_aportgen.py new file mode 100644 index 000000000..7ec2bdf5c --- /dev/null +++ b/.gitlab-ci/testcases/test_aportgen.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# Copyright 2018 Oliver Smith +# SPDX-License-Identifier: GPL-3.0-or-later + +import filecmp +import pytest +import sys +import os + +import add_pmbootstrap_to_import_path +import pmb.aportgen +import pmb.config +import pmb.helpers.logging +import pmb.parse + + +@pytest.fixture +def args(request): + # Initialize args + sys.argv = ["pmbootstrap", + "--aports", os.path.dirname(__file__) + "/../..", + "--log", "$WORK/log_testsuite_pmaports.txt" + "chroot"] + args = pmb.parse.arguments() + + # Initialize logging + pmb.helpers.logging.init(args) + request.addfinalizer(args.logfd.close) + return args + + +def test_aportgen(args, tmpdir): + # Fake aports folder in tmpdir + aports_real = args.aports + args.aports = str(tmpdir) + pmb.helpers.run.user(args, ["mkdir", "-p", str(tmpdir) + "/cross"]) + + # Create aportgen folder -> code path where it still exists + pmb.helpers.run.user(args, ["mkdir", "-p", args.work + "/aportgen"]) + + # Generate all valid packages (gcc twice -> different code path) + pkgnames = ["binutils-armhf", "musl-armhf", "busybox-static-armhf", + "gcc-armhf", "gcc-armhf"] + for pkgname in pkgnames: + pmb.aportgen.generate(args, pkgname) + path_new = args.aports + "/cross/" + pkgname + "/APKBUILD" + path_old = aports_real + "/cross/" + pkgname + "/APKBUILD" + assert os.path.exists(path_new) + assert filecmp.cmp(path_new, path_old, False) + + +def test_aportgen_invalid_generator(args): + with pytest.raises(ValueError) as e: + pmb.aportgen.generate(args, "pkgname-with-no-generator") + assert "No generator available" in str(e.value) + + +def test_aportgen_get_upstream_aport(args, monkeypatch): + + # Fake pmb.parse.apkbuild() + def fake_apkbuild(*args, **kwargs): + return apkbuild + monkeypatch.setattr(pmb.parse, "apkbuild", fake_apkbuild) + + # Fake pmb.parse.apkindex.package() + def fake_package(*args, **kwargs): + return package + monkeypatch.setattr(pmb.parse.apkindex, "package", fake_package) + + # Equal version + func = pmb.aportgen.core.get_upstream_aport + upstream = "main/gcc" + upstream_full = args.work + "/cache_git/aports_upstream/" + upstream + apkbuild = {"pkgver": "2.0", "pkgrel": "0"} + package = {"version": "2.0-r0"} + assert func(args, upstream) == upstream_full + + # APKBUILD < binary + apkbuild = {"pkgver": "1.0", "pkgrel": "0"} + package = {"version": "2.0-r0"} + with pytest.raises(RuntimeError) as e: + func(args, upstream) + assert str(e.value).startswith("You can update your local checkout with") + + # APKBUILD > binary + apkbuild = {"pkgver": "3.0", "pkgrel": "0"} + package = {"version": "2.0-r0"} + with pytest.raises(RuntimeError) as e: + func(args, upstream) + assert str(e.value).startswith("You can force an update of your binary") diff --git a/.gitlab-ci/testcases/test_aports.py b/.gitlab-ci/testcases/test_aports.py new file mode 100644 index 000000000..eb60e3acd --- /dev/null +++ b/.gitlab-ci/testcases/test_aports.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# Copyright 2018 Oliver Smith +# SPDX-License-Identifier: GPL-3.0-or-later + +import glob +import pytest +import sys +import os + +import add_pmbootstrap_to_import_path +import pmb.parse +import pmb.parse._apkbuild + + +@pytest.fixture +def args(request): + # Initialize args + sys.argv = ["pmbootstrap", + "--aports", os.path.dirname(__file__) + "/../..", + "--log", "$WORK/log_testsuite_pmaports.txt" + "chroot"] + args = pmb.parse.arguments() + + # Initialize logging + pmb.helpers.logging.init(args) + request.addfinalizer(args.logfd.close) + return args + + +def test_deviceinfo(args): + """ + Parse all deviceinfo files successfully and run checks on the parsed data. + """ + # Iterate over all devices + last_exception = None + count = 0 + for folder in glob.glob(args.aports + "/device/device-*"): + device = folder[len(args.aports):].split("-", 1)[1] + + try: + # Check for successful deviceinfo parsing + info = pmb.parse.deviceinfo(args, device) + + # deviceinfo_name must start with manufacturer + name = info["name"] + manufacturer = info["manufacturer"] + if not name.startswith(manufacturer) and \ + not name.startswith("Google"): + raise RuntimeError("Please add the manufacturer in front of" + " the deviceinfo_name, e.g.: '" + + manufacturer + " " + name + "'") + + # Don't abort on first error + except Exception as e: + last_exception = e + count += 1 + print(device + ": " + str(e)) + + # Raise the last exception + if last_exception: + print("deviceinfo error count: " + str(count)) + raise last_exception + + +def test_aports_device(args): + """ + Various tests performed on the /device/device-* aports. + """ + for path in glob.glob(args.aports + "/device/device-*/APKBUILD"): + apkbuild = pmb.parse.apkbuild(args, path) + + # Depends: Require "postmarketos-base" + if "postmarketos-base" not in apkbuild["depends"]: + raise RuntimeError("Missing 'postmarketos-base' in depends of " + + path) + + # Depends: Must not have firmware packages + for depend in apkbuild["depends"]: + if (depend.startswith("firmware-") or + depend.startswith("linux-firmware")): + raise RuntimeError("Firmware package '" + depend + "' found in" + " depends of " + path + ". These go into" + " subpackages now, see" + " .") + + +def test_aports_device_kernel(args): + """ + Verify the kernels specified in the device packages: + * Kernel must not be in depends when kernels are in subpackages + * Check if only one kernel is defined in depends + * Validate kernel subpackage names + """ + # Generate list of valid subpackages + valid_subpackages = ["downstream", "rpi", "rpi2"] + for path in glob.glob(args.aports + "/main/linux-postmarketos-*"): + suffix = os.path.basename(path)[len("linux-postmarketos-"):] + valid_subpackages.append(suffix) + + # Iterate over device aports + for path in glob.glob(args.aports + "/device/device-*/APKBUILD"): + # Parse apkbuild and kernels from subpackages + apkbuild = pmb.parse.apkbuild(args, path) + device = apkbuild["pkgname"][len("device-"):] + kernels_subpackages = pmb.parse._apkbuild.kernels(args, device) + + # Parse kernels from depends + kernels_depends = [] + for depend in apkbuild["depends"]: + if not depend.startswith("linux-"): + continue + kernels_depends.append(depend) + + # Kernel in subpackages *and* depends + if kernels_subpackages: + raise RuntimeError("Kernel package '" + depend + "' needs to" + " be removed when using kernel" + + " subpackages: " + path) + + # No kernel + if not kernels_depends and not kernels_subpackages: + raise RuntimeError("Device doesn't have a kernel in depends or" + " subpackages: " + path) + + # Multiple kernels in depends + if len(kernels_depends) > 1: + raise RuntimeError("Please use kernel subpackages instead of" + " multiple kernels in depends (see" + " ): " + + path) + + # Verify subpackages + if kernels_subpackages: + for subpackage in kernels_subpackages: + if subpackage not in valid_subpackages: + raise RuntimeError("Invalid kernel subpackage name '" + + subpackage + "', valid: " + + str(valid_subpackages)) diff --git a/.gitlab-ci/testcases/test_aports_kde.py b/.gitlab-ci/testcases/test_aports_kde.py new file mode 100644 index 000000000..a1a11761b --- /dev/null +++ b/.gitlab-ci/testcases/test_aports_kde.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# Copyright 2018 Oliver Smith +# SPDX-License-Identifier: GPL-3.0-or-later + +import glob +import logging +import os +import pytest +import sys + +import add_pmbootstrap_to_import_path +import pmb.config +import pmb.parse + + +@pytest.fixture +def args(request): + # Initialize args + sys.argv = ["pmbootstrap", + "--aports", os.path.dirname(__file__) + "/../..", + "--log", "$WORK/log_testsuite_pmaports.txt" + "chroot"] + args = pmb.parse.arguments() + + # Initialize logging + pmb.helpers.logging.init(args) + request.addfinalizer(args.logfd.close) + return args + + +def get_categorized_kde_packages(args): + """ + Parse all aports in the kde folder, and categorize them. + + :returns: {"plasma": {"kwin": "5.13.3", ...}, + "kde": {"kcrash": "5.48.0", ...}, + "other": {"konsole": "1234", ...}} + """ + ret = {"plasma": {}, "kde": {}, "other": {}} + + for path in glob.glob(args.aports + "/kde/*/APKBUILD"): + # Parse APKBUILD + apkbuild = pmb.parse.apkbuild(args, path) + url = apkbuild["url"] + pkgname = apkbuild["pkgname"] + pkgver = apkbuild["pkgver"] + + # Categorize by URL + category = "other" + if "https://www.kde.org/workspaces/plasmadesktop" in url: + category = "plasma" + elif "https://community.kde.org/Frameworks" in url: + category = "kde" + + # Save result + ret[category][pkgname] = pkgver + return ret + + +def check_categories(categories): + """ + Make sure that all packages in one framework (kde, plasma) have the same + package version (and that there is at least one package in each category). + + :param categories: see return of get_categorized_kde_packages() + :returns: True when the check passed, False otherwise + """ + ret = True + for category, packages in categories.items(): + reference = None + for pkgname, pkgver in packages.items(): + + # Use the first package as reference and print a summary + if not reference: + logging.info("---") + logging.info("KDE package category: " + category) + logging.info("Packages (" + str(len(packages)) + "): " + + ", ".join(sorted(packages.keys()))) + reference = {"pkgname": pkgname, "pkgver": pkgver} + + # Category "other": done after printing the summary, no need to + # compare the package versions + if category == "other": + break + + # Print the reference and skip checking it against itself + logging.info("Reference pkgver: " + pkgver + " (from '" + + pkgname + "')") + continue + + # Check version against reference + if pkgver != reference["pkgver"]: + logging.info("ERROR: " + pkgname + " has version " + pkgver) + ret = False + + # Each category must at least have one package + if not reference: + logging.info("ERROR: could not find any packages in category: " + + category) + ret = False + return ret + + +def test_kde_versions(args): + """ + Make sure that KDE packages of the same framework have the same version. + """ + categories = get_categorized_kde_packages(args) + if not check_categories(categories): + raise RuntimeError("KDE version check failed!") diff --git a/.gitlab-ci/testcases/test_soname_bump.py b/.gitlab-ci/testcases/test_soname_bump.py new file mode 100644 index 000000000..a9bf28b9b --- /dev/null +++ b/.gitlab-ci/testcases/test_soname_bump.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# Copyright 2018 Oliver Smith +# SPDX-License-Identifier: GPL-3.0-or-later + +""" +This file uses pmb.helper.pkgrel_bump to check if the aports need a pkgrel bump +for any package, caused by a soname bump. Example: A new libressl/openssl +version was released, which increased the soname version, and now all packages +that link against it, need to be rebuilt. +""" + +import os +import pytest +import sys + +import add_pmbootstrap_to_import_path +import pmb.helpers.pkgrel_bump +import pmb.helpers.logging +import pmb.parse + + +@pytest.fixture +def args(request): + # Initialize args + sys.argv = ["pmbootstrap", + "--aports", os.path.dirname(__file__) + "/../..", + "--log", "$WORK/log_testsuite_pmaports.txt" + "chroot"] + args = pmb.parse.arguments() + + # Initialize logging + pmb.helpers.logging.init(args) + request.addfinalizer(args.logfd.close) + return args + + +@pytest.mark.pmaports_upstream_compat +def test_soname_bump(args): + if pmb.helpers.pkgrel_bump.auto(args, True): + raise RuntimeError("One or more packages need to be rebuilt, because" + " a library they link against had an incompatible" + " upgrade (soname bump). Run 'pmbootstrap" + " pkgrel_bump --auto' to automatically increase the" + " pkgrel in order to trigger a rebuild. If this" + " test case failed during a pull request, the issue" + " needs to be fixed on the 'master' branch first," + " then rebase your PR on 'master' afterwards.") diff --git a/.gitlab-ci/testcases/test_upstream_compatibility.py b/.gitlab-ci/testcases/test_upstream_compatibility.py new file mode 100644 index 000000000..1a7bb2155 --- /dev/null +++ b/.gitlab-ci/testcases/test_upstream_compatibility.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# Copyright 2018 Oliver Smith +# SPDX-License-Identifier: GPL-3.0-or-later + +import os +import sys +import glob +import pytest + +import add_pmbootstrap_to_import_path +import pmb.helpers.logging +import pmb.parse.apkindex +import pmb.parse + + +@pytest.fixture +def args(request): + # Initialize args + sys.argv = ["pmbootstrap", + "--aports", os.path.dirname(__file__) + "/../..", + "--log", "$WORK/log_testsuite_pmaports.txt" + "chroot"] + args = pmb.parse.arguments() + + # Initialize logging + pmb.helpers.logging.init(args) + request.addfinalizer(args.logfd.close) + return args + + +@pytest.mark.pmaports_upstream_compat +def test_qt_versions(args): + """ + Verify, that all postmarketOS qt5- package versions match with Alpine's + qt5-qtbase version. + """ + # Upstream version + index = pmb.helpers.repo.alpine_apkindex_path(args, "community", "x86_64") + index_data = pmb.parse.apkindex.package(args, "qt5-qtbase", + indexes=[index]) + pkgver_upstream = index_data["version"].split("-r")[0] + + # Iterate over our packages + failed = [] + for path in glob.glob(args.aports + "/*/qt5-*/APKBUILD"): + # Read the pkgver + apkbuild = pmb.parse.apkbuild(args, path) + pkgname = apkbuild["pkgname"] + pkgver = apkbuild["pkgver"] + + # When we temporarily override packages from Alpine, we set the pkgver + # to 9999 and _pkgver contains the real version (see #994). + if pkgver == "9999": + pkgver = apkbuild["_pkgver"] + + # Compare + if pkgver == pkgver_upstream: + continue + failed.append(pkgname + ": " + pkgver + " != " + + pkgver_upstream) + + assert [] == failed + + +@pytest.mark.pmaports_upstream_compat +def test_aportgen_versions(args): + """ + Verify that the packages generated by 'pmbootstrap aportgen' have + the same version (pkgver *and* pkgrel!) as the upstream packages + they are based on. + """ + # Get Alpine's "main" repository APKINDEX path + index = pmb.helpers.repo.alpine_apkindex_path(args, "main", "x86_64") + + # Alpine packages and patterns for our derivatives + map = {"binutils": "binutils-*", + "busybox": "busybox-static-*", + "gcc": "gcc-*", + "musl": "musl-*"} + + # Iterate over Alpine packages + failed = [] + generated = "# Automatically generated aport, do not edit!" + for pkgname, pattern in map.items(): + # Upstream version + index_data = pmb.parse.apkindex.package(args, pkgname, + indexes=[index]) + version_upstream = index_data["version"] + + # Iterate over our packages + for path in glob.glob(args.aports + "/*/" + pattern + "/APKBUILD"): + # Skip non-aportgen APKBUILDs + with open(path) as handle: + if generated not in handle.read(): + continue + + # Compare the version + print("Checking " + path) + apkbuild = pmb.parse.apkbuild(args, path) + version = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"] + if version != version_upstream: + failed.append(apkbuild["pkgname"] + ": " + version + + " != " + version_upstream + + " (from " + pkgname + ")") + continue + + assert [] == failed