From 16d63d95cb93865999d134d890540be746285678 Mon Sep 17 00:00:00 2001 From: Bo Lorentsen Date: Sat, 24 Oct 2026 19:39:31 +0200 Subject: [PATCH] Refactor CI into composite actions + repo variables The workflow was mostly inline bash. Moves the logic into reusable composite actions under .gitea/actions/, keeps host paths in the YOCTO_HOME repo variable, adds a workflow_dispatch skip-build option and publishes step outputs (build dir, image list, kas path). --- .gitea/actions/build-image/action.yml | 117 ++++++++++++++++ .gitea/actions/check-build-host/action.yml | 37 +++++ .gitea/actions/setup-kas/action.yml | 34 +++++ .gitea/workflows/build.yaml | 153 ++++++--------------- 4 files changed, 233 insertions(+), 108 deletions(-) create mode 100644 .gitea/actions/build-image/action.yml create mode 100644 .gitea/actions/check-build-host/action.yml create mode 100644 .gitea/actions/setup-kas/action.yml diff --git a/.gitea/actions/build-image/action.yml b/.gitea/actions/build-image/action.yml new file mode 100644 index 0000000..78488af --- /dev/null +++ b/.gitea/actions/build-image/action.yml @@ -0,0 +1,117 @@ +name: build-image +description: >- + Build a kas project in a persistent directory and expose the flashable + images. The workspace is synced into the build dir instead of built in + place, because checkout cleans ignored files (build/, extern/) and bitbake + reuse depends on sstate-cache surviving between runs. +inputs: + project: + description: kas configuration file, relative to the repository + required: false + default: .config.yaml + machine: + description: Machine the images are deployed for + required: false + default: raspberrypi4-64 + image-prefix: + description: Artifact filename prefix in the deploy dir + required: false + default: rpilinux-image + skip-build: + description: Set to true to only collect images from an existing build dir + required: false + default: "false" +outputs: + build-dir: + description: Persistent project directory on the runner + value: ${{ steps.locate.outputs.build-dir }} + image-dir: + description: Deploy dir bitbake wrote to + value: ${{ steps.locate.outputs.image-dir }} + images: + description: Newline separated artifact filenames + value: ${{ steps.collect.outputs.images }} +runs: + using: composite + steps: + - name: Sync workspace into persistent build dir + if: ${{ inputs.skip-build != 'true' }} + shell: bash + run: | + set -euo pipefail + SRC="$(pwd)" + DST="${YOCTO_HOME:-$HOME/yocto-build}/$(basename "$GITHUB_REPOSITORY")" + # checkout deletes ignored files, so the build tree must live elsewhere + if [ "$SRC" = "$DST" ]; then + echo "::error::workspace == build dir ($SRC); point YOCTO_HOME somewhere else" + exit 1 + fi + echo "syncing $SRC -> $DST" + mkdir -p "$DST" + # .git stays (kas requires a repo); extern/ + build/ only exist in DST, + # so --delete cannot reach them + rsync -a --delete --exclude /build/ --exclude /extern/ ./ "$DST/" + + - name: kas build + if: ${{ inputs.skip-build != 'true' }} + shell: bash + run: | + set -euo pipefail + cd "${YOCTO_HOME:-$HOME/yocto-build}/$(basename "$GITHUB_REPOSITORY")" + git config --global --add safe.directory "$PWD" || true + "${YOCTO_HOME:-$HOME/yocto-build}/venv/bin/kas" build + + - name: Locate deploy dir + id: locate + shell: bash + run: | + set -euo pipefail + BUILD_DIR="${YOCTO_HOME:-$HOME/yocto-build}/$(basename "$GITHUB_REPOSITORY")" + IMAGE_DIR="$(find "$BUILD_DIR/build" -type d -path "*/tmp/deploy/images/${{ inputs.machine }}" | head -1)" + if [ -z "$IMAGE_DIR" ]; then + echo "::error::no deploy dir for ${{ inputs.machine }} under $BUILD_DIR/build" + exit 1 + fi + echo "build-dir=$BUILD_DIR" >> "$GITHUB_OUTPUT" + echo "image-dir=$IMAGE_DIR" >> "$GITHUB_OUTPUT" + + - name: Collect artifacts + id: collect + shell: bash + run: | + set -euo pipefail + IMAGE_DIR="${{ steps.locate.outputs.image-dir }}" + shopt -s nullglob + # only the timestamped names: the short ones are hardlinks to the same + # inode and would upload every image twice + FILES=("${IMAGE_DIR}/${{ inputs.image-prefix }}."*.rootfs-[0-9]*.wic.* \ + "${IMAGE_DIR}/${{ inputs.image-prefix }}-*.rootfs-[0-9]*.wic.*") + if [ "${#FILES[@]}" -eq 0 ]; then + echo "::error::no flashable .wic image for ${{ inputs.machine }} in $IMAGE_DIR" + exit 1 + fi + mkdir -p dist + cp -v "${FILES[@]}" dist/ + ( cd dist && sha256sum * > SHA256SUMS ) + ls -lh dist + printf 'images=%s\n' "$(ls dist | tr '\n' ' ')" >> "$GITHUB_OUTPUT" + + - name: Summarise + shell: bash + run: | + set -euo pipefail + SUMMARY="${GITHUB_STEP_SUMMARY:-}" + [ -n "$SUMMARY" ] || exit 0 + { + echo "## ${{ inputs.image-prefix }} for ${{ inputs.machine }}" + echo + echo "| artifact | sha256 |" + echo "| --- | --- |" + ( cd dist && while read -r sum file; do + [ "$file" = SHA256SUMS ] || echo "| \`$file\` | \`${sum:0:16}\` |" + done < SHA256SUMS ) + echo + echo "Flash with \`bzcat | sudo dd of=/dev/sdX bs=4M status=progress\`." + echo + echo "Incremental build tree: \`${{ steps.locate.outputs.build-dir }}\`" + } >> "$SUMMARY" diff --git a/.gitea/actions/check-build-host/action.yml b/.gitea/actions/check-build-host/action.yml new file mode 100644 index 0000000..db54030 --- /dev/null +++ b/.gitea/actions/check-build-host/action.yml @@ -0,0 +1,37 @@ +name: check-build-host +description: >- + Report the build host and fail fast when executables, CPU or disk that the + build needs are missing - a mistake should cost seconds, not hours. +inputs: + tools: + description: Space separated executables that must be on PATH + required: false + default: git python3 + min-disk-gb: + description: Warn when the build filesystem has less than this much free + required: false + default: "100" +runs: + using: composite + steps: + - name: Host report + shell: bash + run: | + set -euo pipefail + echo "os : $(. /etc/os-release && echo "$PRETTY_NAME") ($(uname -m), $(uname -r))" + echo "user : $(id -un) home=$HOME" + echo "cpu : $(nproc) cores / $(awk '/MemTotal/{printf "%.0fG", $2/1048576}' /proc/meminfo) ram" + df -h . | tail -1 | sed 's/^/disk : /' + missing="" + for tool in ${{ inputs.tools }}; do + command -v "$tool" >/dev/null 2>&1 || missing="$missing $tool" + done + if [ -n "$missing" ]; then + echo "::error::Host is missing:$missing" + echo "Install them on the runner host, e.g. sudo apt-get install -y $missing" + exit 1 + fi + free_gb="$(df --output=avail -BG . | tail -1 | tr -d ' ')" + if [ "$free_gb" -lt "${{ inputs.min-disk-gb }}" ]; then + echo "::warning::only ${free_gb}G free on $(df --output=target . | tail -1); expected >= ${{ inputs.min-disk-gb }}G" + fi diff --git a/.gitea/actions/setup-kas/action.yml b/.gitea/actions/setup-kas/action.yml new file mode 100644 index 0000000..1859f92 --- /dev/null +++ b/.gitea/actions/setup-kas/action.yml @@ -0,0 +1,34 @@ +name: setup-kas +description: Create or reuse the persistent python venv that holds kas. +inputs: + requirement: + description: pip requirement for kas (pinned, e.g. kas==5.5) + required: false + default: kas +outputs: + kas: + description: Absolute path of the kas executable + value: ${{ steps.venv.outputs.kas }} + reused: + description: "true when the venv already existed (warm run)" + value: ${{ steps.venv.outputs.reused }} +runs: + using: composite + steps: + - name: Prepare venv + id: venv + shell: bash + run: | + set -euo pipefail + BUILD_ROOT="${YOCTO_HOME:-$HOME/yocto-build}" + VENV="$BUILD_ROOT/venv" + mkdir -p "$BUILD_ROOT" + if [ -d "$VENV" ]; then + echo "reused=true" >> "$GITHUB_OUTPUT" + else + echo "reused=false" >> "$GITHUB_OUTPUT" + python3 -m venv "$VENV" + fi + "$VENV/bin/pip" install --quiet --upgrade "${{ inputs.requirement }}" + echo "kas=$VENV/bin/kas" >> "$GITHUB_OUTPUT" + echo "kas $($VENV/bin/kas --version) in $VENV" diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml index 23b3c94..a8d566f 100644 --- a/.gitea/workflows/build.yaml +++ b/.gitea/workflows/build.yaml @@ -1,28 +1,29 @@ name: Build RPi image -# Yocto (kas) build of rpilinux-image for raspberrypi4-64. +# Yocto (kas) build of rpilinux-image. Steps live in .gitea/actions/* so this +# file stays a description of the pipeline, not a shell script. # -# Runs on the dedicated host runner (label "yocto:host", Ryzen 9 box "ada"). -# Everything persists in $YOCTO_HOME: extern/ (layer clones) + build/ (sstate) -# survive between runs -> incremental builds after the first. -# $YOCTO_HOME comes from the runner config (runner.envs.YOCTO_HOME = -# /home/bl/yocto-build, on the 1.8T disk); falls back to $HOME/yocto-build. -# -# First build takes hours (poky fetch + full image) -> runner.timeout >= 8h. -# -# First build takes hours (poky fetch + full image) -> runner.timeout >= 8h. +# Host-specific paths come from repo settings (Settings -> Actions -> +# Variables), so the workflow is not tied to one machine: +# YOCTO_HOME persistent build dir on the runner +# Order of precedence for the build dir: this variable, then the runner's +# own YOCTO_HOME (runner.envs), then $HOME/yocto-build. on: push: - # main is release-protected; ci/* branches are free scratch space + # main is release protected, ci/* branches are free scratch space branches: [main, 'ci/*'] pull_request: - # gates merges into main, and provides the commit status that a - # "require status checks" branch-protection rule waits for + # gates merges into main and posts the commit status that branch + # protection waits for branches: [main] workflow_dispatch: + inputs: + skip-build: + description: Collect images from the existing build dir instead of rebuilding + type: boolean + default: false -# Never run two builds at once on the same (single) runner. concurrency: group: yocto-rpi-build cancel-in-progress: false @@ -30,110 +31,46 @@ concurrency: permissions: contents: read +env: + YOCTO_HOME: ${{ vars.YOCTO_HOME }} + MACHINE: raspberrypi4-64 + IMAGE_PREFIX: rpilinux-image + +defaults: + run: + shell: bash + jobs: build: runs-on: yocto timeout-minutes: 600 steps: - - uses: builtin:checkout + - name: Check out sources + uses: builtin:checkout - - name: Probe environment and host dependencies - run: | - echo "=== runner environment ===" - uname -a - [ -r /etc/os-release ] && . /etc/os-release && echo "$PRETTY_NAME" - echo "HOME=$HOME workspace=$(pwd) nproc=$(nproc)" - df -h "$HOME" | tail -1 - free -h 2>/dev/null | sed -n 2p || true - command -v docker >/dev/null && docker info --format 'docker: {{.ServerVersion}}' 2>/dev/null || echo "no docker daemon" - echo "=== dependency check ===" - set -e - missing="" - for t in git python3 curl wget rsync gawk xz diffstat chrpath node npm; do - command -v "$t" >/dev/null 2>&1 || missing="$missing $t" - done - if [ -n "$missing" ]; then - echo "::error::Missing host tools:$missing" - echo "Install on the runner host, e.g.:" - echo ' sudo apt-get install -y --no-install-recommends build-essential \ - bison flex chrpath curl diffstat file gawk git libpython3-dev libssl-dev \ - make python3 python3-venv python3-pip python3-dev rsync swig unzip \ - wget xz-utils zstd bzip2 nodejs npm' - exit 1 - fi + - name: Verify build host + uses: $/.gitea/actions/check-build-host + with: + tools: git python3 curl wget rsync gawk xz diffstat chrpath bison flex node + min-disk-gb: "200" - - name: Set up kas (persistent venv) - run: | - set -e - export YOCTO_HOME="${YOCTO_HOME:-$HOME/yocto-build}" - mkdir -p "$YOCTO_HOME" - [ -d "$YOCTO_HOME/venv" ] || python3 -m venv "$YOCTO_HOME/venv" - "$YOCTO_HOME/venv/bin/pip" -q install -U kas - "$YOCTO_HOME/venv/bin/kas" --version + - name: Set up kas + id: kas + uses: $/.gitea/actions/setup-kas + with: + requirement: kas==5.5 - - name: Sync to persistent build dir and build - run: | - set -euo pipefail - export YOCTO_HOME="${YOCTO_HOME:-$HOME/yocto-build}" - SRC="$(pwd)" - DST="$YOCTO_HOME/yocto_rpi" - # Safety: builtin:checkout cleans ignored files (build/, extern/), - # so the build tree must NOT be the ephemeral workspace. - if [ "$SRC" = "$DST" ]; then - echo "::error::workspace == persistent dir ($SRC); keep the runner WorkingDirectory outside YOCTO_HOME" - exit 1 - fi - echo "syncing $SRC -> $DST" - mkdir -p "$DST" - # .git is copied (kas expects a git repo); extern/ and build/ exist - # only in DST and are excluded, so --delete cannot touch them. - rsync -a --delete --exclude /build/ --exclude /extern/ ./ "$DST/" - cd "$DST" - git config --global --add safe.directory "$DST" || true - "$YOCTO_HOME/venv/bin/kas" build + - name: Build image + id: image + uses: $/.gitea/actions/build-image + with: + machine: ${{ env.MACHINE }} + image-prefix: ${{ env.IMAGE_PREFIX }} + skip-build: ${{ github.event.inputs.skip-build }} - - name: Collect images - run: | - set -euo pipefail - export YOCTO_HOME="${YOCTO_HOME:-$HOME/yocto-build}" - WS="$(pwd)" # workspace; upload-artifact's path is relative to it - cd "$YOCTO_HOME/yocto_rpi" - IMG_DIR="$(find build -type d -path '*/tmp/deploy/images/raspberrypi4-64' | head -1)" - if [ -z "$IMG_DIR" ]; then - echo "::error::no deploy dir found under build/" - exit 1 - fi - echo "images in: $IMG_DIR" - mkdir -p "$WS/dist" - shopt -s nullglob - # only the timestamped canonical names - the short "...rootfs.wic.bz2" - # entries are hardlinks to the same inode (would double the upload) - FILES=("$IMG_DIR"/rpilinux-image-*.rootfs-[0-9]*.wic.bz2 "$IMG_DIR"/rpilinux-image-*.rootfs-[0-9]*.wic.bmap) - if [ ${#FILES[@]} -eq 0 ]; then - echo "::error::no flashable .wic image found in $IMG_DIR" - exit 1 - fi - cp -v "${FILES[@]}" "$WS/dist/" - ( cd "$WS/dist" && sha256sum * > SHA256SUMS ) - ls -lh "$WS/dist" - SUMMARY="${GITEA_STEP_SUMMARY:-${GITHUB_STEP_SUMMARY:-}}" - if [ -n "$SUMMARY" ]; then - { - echo "## RPi4 image built" - echo - echo "| file | sha256 |" - echo "|---|---|" - ( cd "$WS/dist" && while read -r sum f; do echo "| \`$f\` | \`${sum:0:16}\` |"; done < SHA256SUMS ) - echo - echo "Flash: \`bzcat | sudo dd of=/dev/sdX bs=4M status=progress\`" - echo - echo "Build tree (incremental) on runner host: \`$YOCTO_HOME/yocto_rpi\`" - } >> "$SUMMARY" - fi - - - name: Upload images + - name: Upload image uses: actions/upload-artifact@v4 with: - name: rpilinux-image-rpi4 + name: ${{ env.IMAGE_PREFIX }}-${{ env.MACHINE }} path: dist/ retention-days: 14