From e5b8b8631ff0b63bb6e28a474eb1bb6b6430f0bc Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Mon, 24 Aug 2020 15:21:37 +0200 Subject: [PATCH] CI: build packages on all arches (MR 802) Split the aports-build job into arch-specific build-x86_64, build-aarch64, ... jobs. Extend build_changed_aports.py to accept the architecture as argument, and to build all packages for that arch where possible. Import and use pmbootstrap code for APKBUILD parsing etc, as we do this already in various testcases running in this repository. Co-Authored-By: Bart Ribbers --- .gitlab-ci.yml | 19 +++++++- .gitlab-ci/build.sh | 8 +++ .gitlab-ci/build_changed_aports.py | 78 +++++++++++++++++++++++------- 3 files changed, 85 insertions(+), 20 deletions(-) create mode 100755 .gitlab-ci/build.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index de16e7bfc..71c9df488 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -133,16 +133,31 @@ mr-settings: - python3 ./check_mr_settings.py # build changed aports -aports-build: +.build: stage: second <<: *only-default before_script: - wget "https://gitlab.com/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh" - sh ./install_pmbootstrap.sh script: - - PYTHONUNBUFFERED=1 su pmos -c ".gitlab-ci/build_changed_aports.py" + - .gitlab-ci/build.sh - cp -r /home/pmos/.local/var/pmbootstrap/packages/ packages/ || true artifacts: expire_in: 1 week paths: - packages/ + +build-x86_64: + extends: .build + +build-x86: + extends: .build + +build-aarch64: + extends: .build + +build-armv7: + extends: .build + +build-armhf: + extends: .build diff --git a/.gitlab-ci/build.sh b/.gitlab-ci/build.sh new file mode 100755 index 000000000..0e05857d4 --- /dev/null +++ b/.gitlab-ci/build.sh @@ -0,0 +1,8 @@ +#!/bin/sh -e +# Convenience wrapper for short arch-specific build jobs in .gitlab-ci.yml + +export PYTHONUNBUFFERED=1 +JOB_ARCH="${CI_JOB_NAME#build-}" + +set -x +su pmos -c ".gitlab-ci/build_changed_aports.py $JOB_ARCH" diff --git a/.gitlab-ci/build_changed_aports.py b/.gitlab-ci/build_changed_aports.py index 4fe0a255f..25b520fe5 100755 --- a/.gitlab-ci/build_changed_aports.py +++ b/.gitlab-ci/build_changed_aports.py @@ -1,37 +1,79 @@ #!/usr/bin/env python3 # Copyright 2020 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later +import sys # Same dir import common +# pmbootstrap +import testcases.add_pmbootstrap_to_import_path +import pmb.parse +import pmb.parse._apkbuild +import pmb.helpers.pmaports -def check_build(packages, verify_only=False): - # Initialize build environment with less logging + +def build_strict(packages, arch): common.run_pmbootstrap(["build_init"]) + common.run_pmbootstrap(["--details-to-stdout", "build", "--strict", + "--force", "--arch", arch] + list(packages)) - if verify_only: - common.run_pmbootstrap(["--details-to-stdout", "checksum", - "--verify"] + list(packages)) - else: - common.run_pmbootstrap(["--details-to-stdout", "build", "--strict", - "--force"] + list(packages)) + +def verify_checksums(packages, arch): + # Only do this with one build-{arch} job + arch_verify = "x86_64" + if arch != arch_verify: + print(f"NOTE: doing checksum verification in build-{arch_verify} job," + " not here.") + return + + common.run_pmbootstrap(["build_init"]) + common.run_pmbootstrap(["--details-to-stdout", "checksum", "--verify"] + + list(packages)) if __name__ == "__main__": + # Architecture to build for (as in build-{arch}) + if len(sys.argv) != 2: + print("usage: build_changed_aports.py ARCH") + sys.exit(1) + arch = sys.argv[1] + # Get and print modified packages common.add_upstream_git_remote() packages = common.get_changed_packages() - # Build changed packages + # Package count sanity check common.get_changed_packages_sanity_check(len(packages)) + + # [ci:skip-build]: verify checksums and stop + verify_only = common.commit_message_has_string("[ci:skip-build]") + if verify_only: + print("WARNING: not building changed packages ([ci:skip-build])!") + print("verifying checksums: " + ", ".join(packages)) + verify_checksums(packages, arch) + sys.exit(0) + + # Prepare "args" to use pmbootstrap code + sys.argv = ["pmbootstrap", "chroot"] + args = pmb.parse.arguments() + + # Filter out packages that can't be built for given arch + # (Iterate over copy of packages, because we modify it in this loop) + for package in packages.copy(): + apkbuild_path = pmb.helpers.pmaports.find(args, package) + apkbuild = pmb.parse._apkbuild.apkbuild(args, + f"{apkbuild_path}/APKBUILD") + + if not pmb.helpers.pmaports.check_arches(apkbuild["arch"], arch): + print(f"{package}: not enabled for {arch}, skipping") + packages.remove(package) + + # No packages: skip build if len(packages) == 0: - print("no aports changed in this branch") - else: - verify_only = common.commit_message_has_string("[ci:skip-build]") - if verify_only: - print("WARNING: not building changed packages ([ci:skip-build])!") - print("verifying checksums: " + ", ".join(packages)) - else: - print("building in strict mode: " + ", ".join(packages)) - check_build(packages, verify_only) + print(f"no packages changed, which can be built for {arch}") + sys.exit(0) + + # Build packages + print(f"building in strict mode for {arch}: {', '.join(packages)}") + build_strict(packages, arch)