From a0fd5e421cbdd73604969e07b1deb8b6dc09260d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Roukala=20=28n=C3=A9=20Peres=29?= Date: Fri, 27 Jun 2025 08:44:23 +0300 Subject: [PATCH] ci: add support for building and testing device boot images Part-of: https://gitlab.postmarketos.org/postmarketOS/pmaports/-/merge_requests/6696 --- .ci/build-jobs.yaml.j2 | 84 +++++++++++++++++ .ci/lib/generate_build_jobs.py | 165 +++++++++++++++++++++++++++++++-- .gitlab-ci.yml | 6 +- 3 files changed, 247 insertions(+), 8 deletions(-) diff --git a/.ci/build-jobs.yaml.j2 b/.ci/build-jobs.yaml.j2 index 19b7c3f92..ce60e84cc 100644 --- a/.ci/build-jobs.yaml.j2 +++ b/.ci/build-jobs.yaml.j2 @@ -1,6 +1,17 @@ image: alpine:latest stages: - build + - hardware tests + +variables: + CI_TRON_TEMPLATE_PROJECT: &ci-tron-template-project postmarketOS/ci-common + CI_TRON_JOB_TEMPLATE_PROJECT_URL: $CI_SERVER_URL/$CI_TRON_TEMPLATE_PROJECT + CI_TRON_JOB_TEMPLATE_COMMIT: &ci-tron-template-commit 7c95b5f2d53533e8722abf57c73e558168e811f3 + +include: + - project: *ci-tron-template-project + ref: *ci-tron-template-commit + file: '/ci-tron/common.yml' workflow: rules: @@ -91,3 +102,76 @@ build-riscv64: tags: [qemu] script: - .ci/build-riscv64.sh + +# Hardware testing + +# Define here all the gitlab-ci snippets that could be common between the +# different packages + +# Extends from this job if the the build job generates a kernel and initramfs +# artifact for your target device. This will not work if your device generates +# boot images. +.device-boot-flow-separate-artifacts: + variables: + CI_TRON_KERNEL__URL: "glartifact://${CI_TRON__PMB_BUILD_JOB}/${CI_TRON__PMB_EXPORT_PATH}/${KERNEL_NAME}" + CI_TRON_INITRAMFS__INITRAMFS__URL: "glartifact://${CI_TRON__PMB_BUILD_JOB}/${CI_TRON__PMB_EXPORT_PATH}/initramfs" + +# NOTE: For fastboot devices, we will need to use something like this +# .ci-tron-device-boot-flow-fastboot-image: +# variables: +# CI_TRON_FASTBOOT__BOOT_IMAGE: "glartifact://${CI_TRON__PMB_BUILD_JOB}/${CI_TRON__PMB_EXPORT_PATH}/bootimg" + +# Include all the devices that need and can to be tested + +{% for device in devices_under_test if device.is_present_in_ci %} +prepare-{{ device.name }}: + stage: hardware tests + extends: + .pmos-ci-tron-build-boot-artifacts + {%- if device.has_kernel_variants %} + # Let's build all the boot artifacts for all the supported kernels in + # different jobs, but hide them behind a single job name so as not to pollute + # the pipeline. + parallel: + matrix: + - KERNEL_VARIANT: + {%- for kernel in device.kernels %} + - {{ kernel }} + {%- endfor %} + {%- endif %} + needs: ["build-{{ device.arch }}"] + variables: + DEVICE_NAME: {{ device.name }} + # NOTE: All the packages that may influence testing should have an + # `install-if postmarketos-mkinitfs-hook-ci`, so no need to add them here + INSTALL_PACKAGES: {{ device.pkgname }} {{ device.pkgname }}-kernel-${KERNEL_VARIANT} postmarketos-mkinitfs-hook-ci + +.test-{{ device.name }}: + stage: hardware tests + extends: + - .pmos-ci-tron-initramfs-test + {%- if device.has_kernel_variants %} + # Let's test all the kernels in different jobs, but hide them behind a single + # job name so as not to pollute the pipeline. + parallel: + matrix: + - KERNEL_VARIANT: + {%- for kernel in device.kernels %} + - {{ kernel }} + {%- endfor %} + {%- endif %} + dependencies: [] + needs: + - job: 'prepare-{{ device.name }}' + artifacts: false + # NOTE: We can't wait for a particular job in the matrix until GitLab implements support for it: + # https://gitlab.com/gitlab-org/gitlab/-/issues/423553 + variables: + CI_TRON__PMB_BUILD_JOB: "prepare-{{ device.name }}{% if device.has_kernel_variants %}: [$KERNEL_VARIANT]{% endif %}" + CI_TRON_KERNEL_CMDLINE__DEVICEINFO: '{{ device.deviceinfo.kernel_cmdline }}' + +# Include the gitlab ci yml fragment, as found in +# `device/$category/device-$devicename/gitlab-ci.yml.j2` which defines the jobs +{{ device.gitlab_ci_fragment }} + +{% endfor %} diff --git a/.ci/lib/generate_build_jobs.py b/.ci/lib/generate_build_jobs.py index 37384b4b8..6f32c0b9b 100755 --- a/.ci/lib/generate_build_jobs.py +++ b/.ci/lib/generate_build_jobs.py @@ -2,19 +2,145 @@ # Copyright 2025 Pablo Correa Gomez # SPDX-License-Identifier: GPL-3.0-or-later +from functools import cached_property from pathlib import Path -import shutil +from typing import Self import sys +import traceback from jinja2 import Template -import add_pmbootstrap_to_import_path import pmb.parse +from pmb.parse.deviceinfo import Deviceinfo +from pmb.types import Apkbuild +import pmb.helpers.devices import pmb.helpers.logging +import pmb.helpers.package from pmb.core.arch import Arch # Same dir import common + +class Device: + def __init__(self, codename: str): + self.codename = codename + self.full_path = pmb.helpers.devices.find_path(codename) + + def __repr__(self): + return f"Device({self.codename})" + + @property + def name(self): + return self.full_path.name.removeprefix("device-") + + @cached_property + def gitlab_ci_fragment(self) -> str | None: + fragment_path = self.full_path / "gitlab-ci.yml.j2" + try: + fragment_tmpl = fragment_path.read_text() + except Exception: + return None + + return Template(fragment_tmpl).render(device=self) + + @property + def is_present_in_ci(self): + return bool(self.gitlab_ci_fragment) + + @cached_property + def pmaports_path(self) -> Path: + # Find the root of pmaports + pmaports_root = self.full_path.parent + while (not (pmaports_root / "pmaports.cfg").exists() and + pmaports_root != pmaports_root.root): + pmaports_root = pmaports_root.parent + assert pmaports_root != pmaports_root.root + + return self.full_path.relative_to(pmaports_root) + + @cached_property + def apkbuild(self) -> Apkbuild: + return pmb.parse.apkbuild(self.full_path) + + @property + def pkgname(self) -> str: + return self.apkbuild['pkgname'] + + @property + def arch(self) -> Arch: + return Arch(self.apkbuild['arch'][0]) + + @property + def testing_dependencies(self) -> set[str]: + # TODO: Source this from the APKGBUILD + return {"postmarketos-mkinitfs-hook-ci"} + + @cached_property + def package_dependencies(self) -> list[str]: + # HACK: Work around a bug in depends_recurse which prevents listing all the dependencies + # Bug: https://gitlab.postmarketos.org/postmarketOS/pmbootstrap/-/issues/2623 + return {"postmarketos-initramfs"} + # return pmb.helpers.package.depends_recurse(pkgname=self.pkgname, + # arch=self.arch) + + @cached_property + def dependencies(self) -> set[str]: + return self.package_dependencies | self.testing_dependencies + + @cached_property + def kernels(self) -> list[str] | None: + kernels = [] + + subpackage_prefix = f"device-{self.codename}-kernel-" + for subpkgname in self.apkbuild.get('subpackages', []): + if not subpkgname.startswith(subpackage_prefix): + continue + kernel_name = subpkgname.removeprefix(subpackage_prefix) + + # Ignore `none` kernels as they are not bootable + if kernel_name != "none": + kernels.append(kernel_name) + + if kernels: + return kernels + else: + return None + + @cached_property + def deviceinfo(self) -> dict[str, Deviceinfo] | Deviceinfo: + # NOTE: deviceinfo files that have different values per kernel are + # currently unsupported due to a gitlab-ci issue which prevents us + # from having a per-kernel list of variables that would be selected + # using `.extends`: + # https://gitlab.com/gitlab-org/gitlab/-/issues/299423 + return Deviceinfo(self.full_path / "deviceinfo", None) + + @property + def has_kernel_variants(self): + return len(self.kernels) > 0 + + @classmethod + def supported_devices(cls) -> dict[Path, Self]: + # NOTE: Not sure I like relying on pmboostrap just to avoid listing + # devices ourselves, but since we need it to parse the APKBUILD, then + # let's avoid duplicating the logic? + supported_devices = {} + for vendor in pmb.helpers.devices.list_vendors(): + for device in pmb.helpers.devices.list_codenames(vendor): + try: + dev = cls(device.codename) + + # Ignore devices that do not have a gitlab ci fragment, + # since we won't be able to make use of them + if not dev.gitlab_ci_fragment: + continue + + supported_devices[dev.pmaports_path] = dev + except Exception: + traceback.print_exc() + return supported_devices + + if __name__ == "__main__": # Needs input to output if we should create the jobs if len(sys.argv) != 3: @@ -29,28 +155,53 @@ if __name__ == "__main__": # be changed to a file and added as a CI artifact if we need to debug # something. pmb.logging.init(Path("/dev/null"), False) + + # Load context + sys.argv = ["pmbootstrap.py", "chroot"] + args = pmb.parse.arguments() + + # Get the list of supported devices + supported_devices = Device.supported_devices() + archs = set() + devices_under_test = set() + packages_modified = set() # Get and print modified packages common.add_upstream_git_remote() for file in common.get_changed_files(): path = Path(file) + + # Check if the modified's file parent folder is one of a supported dev + if device := supported_devices.get(path.parent): + devices_under_test.add(device) + if path.name != "APKBUILD": continue elif not path.exists(): continue # APKBUILD was deleted apkbuild = pmb.parse.apkbuild(path) + packages_modified.add(apkbuild['pkgname']) archs.update(apkbuild["arch"]) - if common.commit_message_has_string("[ci:skip-build]"): - print("User requested skipping build, not creating child pipeline file") - archs = set() + # Add all the devices found in CI that depend on the package that got + # modified + for device in supported_devices.values(): + if apkbuild['pkgname'] in device.dependencies: + devices_under_test.add(device) # This ignores things like !armv7, that could be a follow-up optimization if 'noarch' in archs or 'all' in archs: archs = set([str(arch) for arch in Arch.supported()]) - print(archs) + if common.commit_message_has_string("[ci:skip-build]"): + print("User requested skipping build, not creating child pipeline file") + archs = set() + devices_under_test = set() + + print(f"Architectures to build: {archs}") + print(f"Devices under test: {devices_under_test}") + with open(template) as f: - rendered = Template(f.read()).render(archs=archs) + rendered = Template(f.read()).render(archs=archs, devices_under_test=devices_under_test, packages_modified=packages_modified) with open(child_pipeline, "w") as fw: fw.write(rendered) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bf0ad2f66..59107e824 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -155,7 +155,11 @@ generate-build-jobs: script: - wget "https://gitlab.postmarketos.org/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh" - sh ./install_pmbootstrap.sh py3-jinja2 - - .ci/lib/generate_build_jobs.py .ci/build-jobs.yaml.j2 .ci/build-jobs.yaml + + # HACK: Make the pmb library happy to use the currently-cloned pmaports repo + - mkdir -p /root/.local/var/pmbootstrap/cache_git/ && ln -s $PWD /root/.local/var/pmbootstrap/cache_git/pmaports + + - PYTHONPATH=/tmp/pmbootstrap/ .ci/lib/generate_build_jobs.py .ci/build-jobs.yaml.j2 .ci/build-jobs.yaml artifacts: paths: