ci: add support for building and testing device boot images

Part-of: https://gitlab.postmarketos.org/postmarketOS/pmaports/-/merge_requests/6696

.ci/build-jobs.yaml.j2

@@ -1,6 +1,17 @@
 image: alpine:latest
 stages:
   - build
+  - hardware tests
+
+variables:
+  CI_TRON_TEMPLATE_PROJECT: &ci-tron-template-project postmarketOS/ci-common
+  CI_TRON_JOB_TEMPLATE_PROJECT_URL: $CI_SERVER_URL/$CI_TRON_TEMPLATE_PROJECT
+  CI_TRON_JOB_TEMPLATE_COMMIT: &ci-tron-template-commit 7c95b5f2d53533e8722abf57c73e558168e811f3
+
+include:
+  - project: *ci-tron-template-project
+    ref: *ci-tron-template-commit
+    file: '/ci-tron/common.yml'
 
 workflow:
   rules:
@@ -91,3 +102,76 @@ build-riscv64:
   tags: [qemu]
   script:
     - .ci/build-riscv64.sh
+
+# Hardware testing
+
+# Define here all the gitlab-ci snippets that could be common between the
+# different packages
+
+# Extends from this job if the the build job generates a kernel and initramfs
+# artifact for your target device. This will not work if your device generates
+# boot images.
+.device-boot-flow-separate-artifacts:
+  variables:
+    CI_TRON_KERNEL__URL: "glartifact://${CI_TRON__PMB_BUILD_JOB}/${CI_TRON__PMB_EXPORT_PATH}/${KERNEL_NAME}"
+    CI_TRON_INITRAMFS__INITRAMFS__URL: "glartifact://${CI_TRON__PMB_BUILD_JOB}/${CI_TRON__PMB_EXPORT_PATH}/initramfs"
+
+# NOTE: For fastboot devices, we will need to use something like this
+# .ci-tron-device-boot-flow-fastboot-image:
+#   variables:
+#     CI_TRON_FASTBOOT__BOOT_IMAGE: "glartifact://${CI_TRON__PMB_BUILD_JOB}/${CI_TRON__PMB_EXPORT_PATH}/bootimg"
+
+# Include all the devices that need and can to be tested
+
+{% for device in devices_under_test if device.is_present_in_ci %}
+prepare-{{ device.name }}:
+  stage: hardware tests
+  extends:
+    .pmos-ci-tron-build-boot-artifacts
+  {%- if device.has_kernel_variants %}
+  # Let's build all the boot artifacts for all the supported kernels in
+  # different jobs, but hide them behind a single job name so as not to pollute
+  # the pipeline.
+  parallel:
+    matrix:
+      - KERNEL_VARIANT:
+      {%- for kernel in device.kernels %}
+        - {{ kernel }}
+      {%- endfor %}
+  {%- endif %}
+  needs: ["build-{{ device.arch }}"]
+  variables:
+    DEVICE_NAME: {{ device.name }}
+    # NOTE: All the packages that may influence testing should have an
+    # `install-if postmarketos-mkinitfs-hook-ci`, so no need to add them here
+    INSTALL_PACKAGES: {{ device.pkgname }} {{ device.pkgname }}-kernel-${KERNEL_VARIANT} postmarketos-mkinitfs-hook-ci
+
+.test-{{ device.name }}:
+  stage: hardware tests
+  extends:
+    - .pmos-ci-tron-initramfs-test
+  {%- if device.has_kernel_variants %}
+  # Let's test all the kernels in different jobs, but hide them behind a single
+  # job name so as not to pollute the pipeline.
+  parallel:
+    matrix:
+      - KERNEL_VARIANT:
+      {%- for kernel in device.kernels %}
+        - {{ kernel }}
+      {%- endfor %}
+  {%- endif %}
+  dependencies: []
+  needs:
+    - job: 'prepare-{{ device.name }}'
+      artifacts: false
+      # NOTE: We can't wait for a particular job in the matrix until GitLab implements support for it:
+      # https://gitlab.com/gitlab-org/gitlab/-/issues/423553
+  variables:
+    CI_TRON__PMB_BUILD_JOB: "prepare-{{ device.name }}{% if device.has_kernel_variants %}: [$KERNEL_VARIANT]{% endif %}"
+    CI_TRON_KERNEL_CMDLINE__DEVICEINFO: '{{ device.deviceinfo.kernel_cmdline }}'
+
+# Include the gitlab ci yml fragment, as found in
+# `device/$category/device-$devicename/gitlab-ci.yml.j2` which defines the jobs
+{{ device.gitlab_ci_fragment }}
+
+{% endfor %}

.ci/lib/generate_build_jobs.py

@@ -2,19 +2,145 @@
 # Copyright 2025 Pablo Correa Gomez
 # SPDX-License-Identifier: GPL-3.0-or-later
 
+from functools import cached_property
 from pathlib import Path
-import shutil
+from typing import Self
 import sys
+import traceback
 
 from jinja2 import Template
-import add_pmbootstrap_to_import_path
 import pmb.parse
+from pmb.parse.deviceinfo import Deviceinfo
+from pmb.types import Apkbuild
+import pmb.helpers.devices
 import pmb.helpers.logging
+import pmb.helpers.package
 from pmb.core.arch import Arch
 
 # Same dir
 import common
 
+
+class Device:
+    def __init__(self, codename: str):
+        self.codename = codename
+        self.full_path = pmb.helpers.devices.find_path(codename)
+
+    def __repr__(self):
+        return f"Device({self.codename})"
+
+    @property
+    def name(self):
+        return self.full_path.name.removeprefix("device-")
+
+    @cached_property
+    def gitlab_ci_fragment(self) -> str | None:
+        fragment_path = self.full_path / "gitlab-ci.yml.j2"
+        try:
+            fragment_tmpl = fragment_path.read_text()
+        except Exception:
+            return None
+
+        return Template(fragment_tmpl).render(device=self)
+
+    @property
+    def is_present_in_ci(self):
+        return bool(self.gitlab_ci_fragment)
+
+    @cached_property
+    def pmaports_path(self) -> Path:
+        # Find the root of pmaports
+        pmaports_root = self.full_path.parent
+        while (not (pmaports_root / "pmaports.cfg").exists() and
+               pmaports_root != pmaports_root.root):
+            pmaports_root = pmaports_root.parent
+        assert pmaports_root != pmaports_root.root
+
+        return self.full_path.relative_to(pmaports_root)
+
+    @cached_property
+    def apkbuild(self) -> Apkbuild:
+        return pmb.parse.apkbuild(self.full_path)
+
+    @property
+    def pkgname(self) -> str:
+        return self.apkbuild['pkgname']
+
+    @property
+    def arch(self) -> Arch:
+        return Arch(self.apkbuild['arch'][0])
+
+    @property
+    def testing_dependencies(self) -> set[str]:
+        # TODO: Source this from the APKGBUILD
+        return {"postmarketos-mkinitfs-hook-ci"}
+
+    @cached_property
+    def package_dependencies(self) -> list[str]:
+        # HACK: Work around a bug in depends_recurse which prevents listing all the dependencies
+        # Bug: https://gitlab.postmarketos.org/postmarketOS/pmbootstrap/-/issues/2623
+        return {"postmarketos-initramfs"}
+        # return pmb.helpers.package.depends_recurse(pkgname=self.pkgname,
+        #                                            arch=self.arch)
+
+    @cached_property
+    def dependencies(self) -> set[str]:
+        return self.package_dependencies | self.testing_dependencies
+
+    @cached_property
+    def kernels(self) -> list[str] | None:
+        kernels = []
+
+        subpackage_prefix = f"device-{self.codename}-kernel-"
+        for subpkgname in self.apkbuild.get('subpackages', []):
+            if not subpkgname.startswith(subpackage_prefix):
+                continue
+            kernel_name = subpkgname.removeprefix(subpackage_prefix)
+
+            # Ignore `none` kernels as they are not bootable
+            if kernel_name != "none":
+                kernels.append(kernel_name)
+
+        if kernels:
+            return kernels
+        else:
+            return None
+
+    @cached_property
+    def deviceinfo(self) -> dict[str, Deviceinfo] | Deviceinfo:
+        # NOTE: deviceinfo files that have different values per kernel are
+        # currently unsupported due to a gitlab-ci issue which prevents us
+        # from having a per-kernel list of variables that would be selected
+        # using `.extends`:
+        # https://gitlab.com/gitlab-org/gitlab/-/issues/299423
+        return Deviceinfo(self.full_path / "deviceinfo", None)
+
+    @property
+    def has_kernel_variants(self):
+        return len(self.kernels) > 0
+
+    @classmethod
+    def supported_devices(cls) -> dict[Path, Self]:
+        # NOTE: Not sure I like relying on pmboostrap just to avoid listing
+        # devices ourselves, but since we need it to parse the APKBUILD, then
+        # let's avoid duplicating the logic?
+        supported_devices = {}
+        for vendor in pmb.helpers.devices.list_vendors():
+            for device in pmb.helpers.devices.list_codenames(vendor):
+                try:
+                    dev = cls(device.codename)
+
+                    # Ignore devices that do not have a gitlab ci fragment,
+                    # since we won't be able to make use of them
+                    if not dev.gitlab_ci_fragment:
+                        continue
+
+                    supported_devices[dev.pmaports_path] = dev
+                except Exception:
+                    traceback.print_exc()
+        return supported_devices
+
+
 if __name__ == "__main__":
     # Needs input to output if we should create the jobs
     if len(sys.argv) != 3:
@@ -29,28 +155,53 @@ if __name__ == "__main__":
     # be changed to a file and added as a CI artifact if we need to debug
     # something.
     pmb.logging.init(Path("/dev/null"), False)
+
+    # Load context
+    sys.argv = ["pmbootstrap.py", "chroot"]
+    args = pmb.parse.arguments()
+
+    # Get the list of supported devices
+    supported_devices = Device.supported_devices()
+
     archs = set()
+    devices_under_test = set()
+    packages_modified = set()
     # Get and print modified packages
     common.add_upstream_git_remote()
     for file in common.get_changed_files():
         path = Path(file)
+
+        # Check if the modified's file parent folder is one of a supported dev
+        if device := supported_devices.get(path.parent):
+            devices_under_test.add(device)
+
         if path.name != "APKBUILD":
             continue
         elif not path.exists():
             continue  # APKBUILD was deleted
         apkbuild = pmb.parse.apkbuild(path)
+        packages_modified.add(apkbuild['pkgname'])
         archs.update(apkbuild["arch"])
 
-    if common.commit_message_has_string("[ci:skip-build]"):
-        print("User requested skipping build, not creating child pipeline file")
-        archs = set()
+        # Add all the devices found in CI that depend on the package that got
+        # modified
+        for device in supported_devices.values():
+            if apkbuild['pkgname'] in device.dependencies:
+                devices_under_test.add(device)
 
     # This ignores things like !armv7, that could be a follow-up optimization
     if 'noarch' in archs or 'all' in archs:
         archs = set([str(arch) for arch in Arch.supported()])
 
-    print(archs)
+    if common.commit_message_has_string("[ci:skip-build]"):
+        print("User requested skipping build, not creating child pipeline file")
+        archs = set()
+        devices_under_test = set()
+
+    print(f"Architectures to build: {archs}")
+    print(f"Devices under test: {devices_under_test}")
+
     with open(template) as f:
-        rendered = Template(f.read()).render(archs=archs)
+        rendered = Template(f.read()).render(archs=archs, devices_under_test=devices_under_test, packages_modified=packages_modified)
         with open(child_pipeline, "w") as fw:
             fw.write(rendered)

.gitlab-ci.yml

@@ -155,7 +155,11 @@ generate-build-jobs:
   script:
     - wget "https://gitlab.postmarketos.org/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh"
     - sh ./install_pmbootstrap.sh py3-jinja2
-    - .ci/lib/generate_build_jobs.py .ci/build-jobs.yaml.j2 .ci/build-jobs.yaml
+
+    # HACK: Make the pmb library happy to use the currently-cloned pmaports repo
+    - mkdir -p /root/.local/var/pmbootstrap/cache_git/ && ln -s $PWD /root/.local/var/pmbootstrap/cache_git/pmaports
+
+    - PYTHONPATH=/tmp/pmbootstrap/ .ci/lib/generate_build_jobs.py .ci/build-jobs.yaml.j2 .ci/build-jobs.yaml
 
   artifacts:
     paths: