From 32e52d366cf12fded654c80cbd02200039e80050 Mon Sep 17 00:00:00 2001 From: Koen Kooi Date: Thu, 16 Jul 2026 13:42:14 +0200 Subject: [PATCH] Add qemu-only build CI mirroring meta-qcom's pipeline Add a kas build matrix (nodistro x qemuarm/qemuarm64/qemux86-64) that runs on every PR and push, giving meta-ai automated build-verification signal for the first time. This repo previously had only the generic qcom-preflight-checks/stale-issues hygiene workflows and relied on manual `bitbake` runs to catch build breakage. - ci/base.yml + ci/.yml: CI-only kas fragments mirroring meta-qcom's ci/ layout so the reused compile action's hardcoded paths (ci/${machine}.yml, ci/kas-container-shell-helper.sh, etc.) work unmodified. Kept separate from kas/base.yml (the existing hand-run developer path), which has different conventions (branch pin, disk-monitor thresholds). - kas/qemuarm.yml, kas/qemux86-64.yml: matching machine fragments for the developer-facing kas/ path, alongside the existing kas/qemuarm64.yml. - .github/workflows/build-yocto.yml: reusable workflow (kas-setup -> patchreview/check-layer -> compile matrix -> summary), calling qualcomm-linux/meta-qcom's compile action directly rather than vendoring a local copy. Runs on the same self-hosted qcom-u2404 runner pool as meta-qcom, and reuses its CACHE_DIR (/efs/qli/meta-qcom) rather than provisioning new shared storage. - .github/workflows/pr.yml, push.yml: trigger the reusable workflow on PRs to main and pushes to main/wrynose. - .github/workflows/lava-test.yml: disabled workflow_dispatch-only placeholder documenting future on-device LAVA testing (litert, onnxruntime, tensorflow-lite, llama-cpp on rb1-core-kit, rb3gen2-core-kit, uno-q, ventuno-q) without wiring it up yet. Verified locally via `kas-container dump --skip repos_checkout` for every new kas fragment combination; no CI dry-run possible from this environment (self-hosted runners and S3 credentials are not reachable here). Signed-off-by: Koen Kooi --- .github/workflows/Readme.md | 3 + .github/workflows/build-yocto.yml | 205 ++++++++++++++++++++++++++++++ .github/workflows/lava-test.yml | 46 +++++++ .github/workflows/pr.yml | 34 +++++ .github/workflows/push.yml | 18 +++ ci/base.yml | 50 ++++++++ ci/ci.yml | 8 ++ ci/kas-container-shell-helper.sh | 20 +++ ci/qemuarm.yml | 8 ++ ci/qemuarm64.yml | 8 ++ ci/qemux86-64.yml | 8 ++ ci/world.yml | 12 ++ ci/yocto-check-layer.sh | 43 +++++++ ci/yocto-patchreview.sh | 29 +++++ ci/yocto-pybootchartgui.sh | 41 ++++++ kas/qemuarm.yml | 6 + kas/qemux86-64.yml | 6 + 17 files changed, 545 insertions(+) create mode 100644 .github/workflows/build-yocto.yml create mode 100644 .github/workflows/lava-test.yml create mode 100644 .github/workflows/pr.yml create mode 100644 .github/workflows/push.yml create mode 100644 ci/base.yml create mode 100644 ci/ci.yml create mode 100755 ci/kas-container-shell-helper.sh create mode 100644 ci/qemuarm.yml create mode 100644 ci/qemuarm64.yml create mode 100644 ci/qemux86-64.yml create mode 100644 ci/world.yml create mode 100755 ci/yocto-check-layer.sh create mode 100755 ci/yocto-patchreview.sh create mode 100755 ci/yocto-pybootchartgui.sh create mode 100644 kas/qemuarm.yml create mode 100644 kas/qemux86-64.yml diff --git a/.github/workflows/Readme.md b/.github/workflows/Readme.md index 8dc5e2c..bf439dc 100644 --- a/.github/workflows/Readme.md +++ b/.github/workflows/Readme.md @@ -4,3 +4,6 @@ This folder contains workflows that are helpful for maintaining a smooth and sec Workflows: 1. `qcom-preflight-checks.yml` - This workflow runs several preflight checks, including copyight, email, repolinter, and security checks. See [qualcomm/qcom-actions](https://github.com/qualcomm/qcom-actions) 2. `stale-issues.yaml` - This workflow will periodically run every 30 days to check for stalled issues and PRs. If the workflow detects any stalled issues and/or PRs, it will automatically leave just a comment to draw attention. +3. `build-yocto.yml` - Reusable build workflow. Runs a kas build matrix (`nodistro` × `qemuarm`/`qemuarm64`/`qemux86-64`) on the shared `qcom-u2404` self-hosted runner pool, reusing `qualcomm-linux/meta-qcom`'s `compile` action directly rather than a local copy. Called by `pr.yml` and `push.yml`. +4. `pr.yml` / `push.yml` - Trigger `build-yocto.yml` on pull requests to `main` and on pushes to `main`/`wrynose`. +5. `lava-test.yml` - Disabled placeholder for future on-device LAVA testing (see the TODO block in that file). `workflow_dispatch`-only; not wired into `pr.yml`/`push.yml` yet. diff --git a/.github/workflows/build-yocto.yml b/.github/workflows/build-yocto.yml new file mode 100644 index 0000000..ef7fabc --- /dev/null +++ b/.github/workflows/build-yocto.yml @@ -0,0 +1,205 @@ +name: Build Yocto + +on: + workflow_call: + inputs: + profile: + required: false + type: string + default: "full" # values: pr | full + git_ref: + required: false + type: string + default: "" + outputs: + artifacts_url: + description: "URL to retrieve build artifacts" + value: ${{ jobs.create-output.outputs.url }} + +# Concurrency is used to prevent multiple workflows from running for the same PR +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || format('push-{0}', github.sha) }} + cancel-in-progress: true + +env: + CACHE_DIR: /efs/qli/meta-qcom + KAS_CLONE_DEPTH: 1 + +jobs: + kas-setup: + if: github.repository_owner == 'qualcomm-linux' + runs-on: [self-hosted, qcom-u2404, amd64] + steps: + - name: Setting up kas-container + run: | + KAS_CONTAINER=$RUNNER_TEMP/kas-container + echo "KAS_CONTAINER=$KAS_CONTAINER" >> $GITHUB_ENV + LATEST=$(git ls-remote --tags --refs --sort="v:refname" https://github.com/siemens/kas | tail -n1 | sed 's/.*\///') + wget -qO ${KAS_CONTAINER} https://raw.githubusercontent.com/siemens/kas/refs/tags/$LATEST/kas-container + chmod +x ${KAS_CONTAINER} + + - uses: actions/checkout@v6 + with: + persist-credentials: false + ref: ${{ inputs.git_ref }} + + - name: Run kas lock + if: ${{ hashFiles('ci/base.lock.yml') == '' }} + run: | + ${KAS_CONTAINER} lock --update ci/base.yml + + - name: Upload kas lockfile + uses: actions/upload-artifact@v6 + with: + name: kas-lockfile + path: ci/*.lock.yml + + - name: Upload kas-container + uses: actions/upload-artifact@v6 + with: + name: kas-container + path: ${{ env.KAS_CONTAINER }} + + yocto-run-checks: + needs: kas-setup + if: github.repository_owner == 'qualcomm-linux' + runs-on: [self-hosted, qcom-u2404, amd64] + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + ref: ${{ inputs.git_ref }} + + - name: Download kas lockfile + uses: actions/download-artifact@v7 + with: + name: kas-lockfile + path: ci + + - name: Download kas-container + uses: actions/download-artifact@v7 + with: + name: kas-container + path: ${{ runner.temp }} + + - name: Setting up kas-container + run: | + KAS_CONTAINER=$RUNNER_TEMP/kas-container + echo "KAS_CONTAINER=$KAS_CONTAINER" >> $GITHUB_ENV + chmod +x $KAS_CONTAINER + + - name: Run Yocto patchreview + run: | + ci/kas-container-shell-helper.sh ci/yocto-patchreview.sh + + - name: Run yocto-check-layer + run: | + ci/kas-container-shell-helper.sh ci/yocto-check-layer.sh + + compile: + needs: kas-setup + if: github.repository_owner == 'qualcomm-linux' + runs-on: [self-hosted, qcom-u2404, amd64] + outputs: + url: ${{ steps.compile_kas.outputs.url }} + strategy: + fail-fast: false + matrix: + machine: + - qemuarm + - qemuarm64 + - qemux86-64 + distro: + - name: nodistro + yamlfile: "" + kernel: + - type: default + dirname: "" + yamlfile: "" + name: ${{ matrix.machine }}/${{ matrix.distro.name }}${{ matrix.kernel.dirname }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + ref: ${{ inputs.git_ref }} + + - name: Run kas build + uses: qualcomm-linux/meta-qcom/.github/actions/compile@master + id: compile_kas + with: + machine: ${{matrix.machine}} + distro_yaml: ${{matrix.distro.yamlfile}} + distro_name: ${{matrix.distro.name}} + kernel_yaml: ${{matrix.kernel.yamlfile}} + kernel_dirname: ${{matrix.kernel.dirname}} + cache_dir: ${{env.CACHE_DIR}} + + publish_summary: + needs: compile + if: ${{ always() && github.repository_owner == 'qualcomm-linux' }} + runs-on: ubuntu-latest + steps: + - name: 'Download build URLs' + uses: actions/download-artifact@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + pattern: build-url* + path: urlfiles + merge-multiple: true + + - name: "Print output" + shell: python + id: print-output + run: | + import os + ftable = {} + oslist = set() + machinelist = set() + for fname in os.listdir("./urlfiles"): + if fname.startswith("build-url"): + b, m, o = fname.split("_", 2) + oslist.add(o) + machinelist.add(m) + url = "" + with open(f"./urlfiles/{fname}", "r") as urlfile: + url = urlfile.read() + if not o in ftable: + ftable.update({o:{m: url}}) + else: + ftable[o].update({m: url}) + + table_str = "| |" + + for m in sorted(machinelist): + table_str += f" {m} |" + + table_str += "\n|" + for i in range(len(machinelist) + 1): + table_str += " ---- |" + + table_str += "\n" + + for o in sorted(ftable.keys()): + table_str += f"| {o} |" + for m in sorted(machinelist): + url = ftable[o].get(m) + if url: + url = url.strip() + table_str += f" [Files]({url}/{o}/{m}/) |" + else: + table_str += " |" + table_str += "\n" + summary_file_name = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_file_name: + with open(summary_file_name, "a") as summaryfile: + summaryfile.write("## Download URLs\n") + summaryfile.write(table_str) + print(table_str) + + build_successful: + if: github.repository_owner == 'qualcomm-linux' + needs: compile + runs-on: ubuntu-latest + steps: + - name: Build completed successfully + run: echo "All compile matrix jobs succeeded and artifacts uploaded to S3 bucket" diff --git a/.github/workflows/lava-test.yml b/.github/workflows/lava-test.yml new file mode 100644 index 0000000..c0d34c0 --- /dev/null +++ b/.github/workflows/lava-test.yml @@ -0,0 +1,46 @@ +# DISABLED PLACEHOLDER — on-device LAVA testing is not yet implemented for meta-ai. +# +# This workflow only runs via manual `workflow_dispatch` (no `push`/ +# `pull_request` triggers), so it never fires automatically. It exists to +# document the intended shape of on-device testing before it is built. +# +# TODO: enable on-device testing of the AI runtimes (litert, onnxruntime, +# tensorflow-lite, llama-cpp) against simple models, on real hardware, via +# LAVA. See meta-qcom's test.yml / test-distro.yml and the +# `qualcomm-linux/meta-qcom/.github/actions/lava-test-plans` action for the +# pattern to follow (job definitions generated via the +# `qualcomm-linux/lava-test-plans` tool, submitted via +# `foundriesio/lava-action`). +# +# Target machines: +# - rb1-core-kit (qualcomm-linux/meta-qcom) +# - rb3gen2-core-kit (qualcomm-linux/meta-qcom) +# - uno-q (qualcomm-linux/meta-qcom-3rdparty, merged) +# - ventuno-q (qualcomm-linux/meta-qcom-3rdparty, NOT yet merged — +# see PR #16 "Add ventuno-q support"; blocked on that +# landing upstream before this machine can be added) +# +# To enable: add `push`/`pull_request` triggers (mirroring push.yml/pr.yml), +# wire this into push.yml/pr.yml as a needs: [build] job, remove this TODO +# block, and fill in the lava-test-plans invocation per target machine. + +name: LAVA on-device tests (disabled placeholder) + +on: + workflow_dispatch: + inputs: + build_id: + required: true + type: string + description: "Run ID of the build-yocto.yml workflow run to test" + +jobs: + test-nodistro: + name: "Test nodistro (placeholder — not implemented)" + runs-on: ubuntu-latest + steps: + - name: Not implemented + run: | + echo "On-device LAVA testing is not yet implemented for meta-ai." + echo "See the TODO block at the top of this workflow file." + exit 1 diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 0000000..346dc68 --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,34 @@ +name: Build on PR +run-name: "${{ github.base_ref }}: ${{ github.event.pull_request.title }}" + +on: + pull_request: + branches: + - main + paths-ignore: + - 'README.md' + - 'README' + - 'SECURITY.md' + +permissions: + checks: write + pull-requests: write + contents: read + packages: read + actions: read + +jobs: + event-file: + name: "Upload event file" + runs-on: ubuntu-latest + steps: + - name: Upload + uses: actions/upload-artifact@v6 + with: + name: Event File + path: ${{ github.event_path }} + + build-pr: + uses: ./.github/workflows/build-yocto.yml + with: + profile: pr diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..85ad301 --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,18 @@ +name: Build on push +run-name: "${{ github.ref_name }}: ${{ github.event.head_commit.message }}" + +on: + push: + branches: + - main + - wrynose + +permissions: + checks: write + pull-requests: write + contents: read + packages: read + +jobs: + build: + uses: ./.github/workflows/build-yocto.yml diff --git a/ci/base.yml b/ci/base.yml new file mode 100644 index 0000000..34308a3 --- /dev/null +++ b/ci/base.yml @@ -0,0 +1,50 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/siemens/kas/master/kas/schema-kas.json + +header: + version: 14 + +distro: nodistro + +defaults: + repos: + branch: main + +repos: + meta-ai: + + openembedded-core: + url: https://github.com/openembedded/openembedded-core + layers: + meta: + + meta-openembedded: + url: https://github.com/openembedded/meta-openembedded + layers: + meta-oe: + + bitbake: + url: https://github.com/openembedded/bitbake + layers: + .: disabled + +local_conf_header: + base: | + INHERIT += "buildstats buildstats-summary" + INHERIT += "buildhistory" + INHERIT += "rm_work" + diskmon: | + BB_DISKMON_DIRS ??= "\ + STOPTASKS,${TMPDIR},1G,100K \ + STOPTASKS,${DL_DIR},1G,100K \ + STOPTASKS,${SSTATE_DIR},10G,100K \ + STOPTASKS,/tmp,100M,100K \ + HALT,${TMPDIR},100M,1K \ + HALT,${DL_DIR},100M,1K \ + HALT,${SSTATE_DIR},1G,1K \ + HALT,/tmp,10M,1K" + +machine: unset + +target: + - llama-cpp + - tensorflow-lite diff --git a/ci/ci.yml b/ci/ci.yml new file mode 100644 index 0000000..7fc3f32 --- /dev/null +++ b/ci/ci.yml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/siemens/kas/master/kas/schema-kas.json + +header: + version: 14 + +local_conf_header: + ci: | + BB_SIGNATURE_HANDLER = "OEBasicHash" diff --git a/ci/kas-container-shell-helper.sh b/ci/kas-container-shell-helper.sh new file mode 100755 index 0000000..39e4be6 --- /dev/null +++ b/ci/kas-container-shell-helper.sh @@ -0,0 +1,20 @@ +#!/bin/sh -e +# Copyright (c) 2025 Qualcomm Innovation Center, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +TOPDIR=$(realpath $(dirname $(readlink -f $0))/..) +SCRIPT=$(realpath $1) + +if ! [ -f $SCRIPT ]; then + echo "The script path argument is missing, please run it with:" + echo " $0 /path/to/script" + exit 1 +fi + +# make it relative to the TOPDIR +SCRIPT=${SCRIPT#$TOPDIR/} + +# on ci the kas-container is not on the default path +KAS_CONTAINER=${KAS_CONTAINER:-$(which kas-container)} + +exec $KAS_CONTAINER shell $TOPDIR/ci/base.yml --command "/repo/$SCRIPT /repo /work" diff --git a/ci/qemuarm.yml b/ci/qemuarm.yml new file mode 100644 index 0000000..dc82ed1 --- /dev/null +++ b/ci/qemuarm.yml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/siemens/kas/master/kas/schema-kas.json + +header: + version: 14 + includes: + - ci/base.yml + +machine: qemuarm diff --git a/ci/qemuarm64.yml b/ci/qemuarm64.yml new file mode 100644 index 0000000..b39720f --- /dev/null +++ b/ci/qemuarm64.yml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/siemens/kas/master/kas/schema-kas.json + +header: + version: 14 + includes: + - ci/base.yml + +machine: qemuarm64 diff --git a/ci/qemux86-64.yml b/ci/qemux86-64.yml new file mode 100644 index 0000000..b7e1b35 --- /dev/null +++ b/ci/qemux86-64.yml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/siemens/kas/master/kas/schema-kas.json + +header: + version: 14 + includes: + - ci/base.yml + +machine: qemux86-64 diff --git a/ci/world.yml b/ci/world.yml new file mode 100644 index 0000000..544cbf7 --- /dev/null +++ b/ci/world.yml @@ -0,0 +1,12 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/siemens/kas/master/kas/schema-kas.json + +header: + version: 14 + +local_conf_header: + world_build: | + EXCLUDE_FROM_WORLD = "1" + EXCLUDE_FROM_WORLD:layer-ai = "0" + +target: + - world diff --git a/ci/yocto-check-layer.sh b/ci/yocto-check-layer.sh new file mode 100755 index 0000000..9a01af9 --- /dev/null +++ b/ci/yocto-check-layer.sh @@ -0,0 +1,43 @@ +#!/bin/bash -e +# Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +if [ -z $1 ] || [ -z $2 ] ; then + echo "The REPO_DIR or WORK_DIR is empty and it needs to point to the corresponding directories." + echo "Please run it with:" + echo " $0 REPO_DIR WORK_DIR" + exit 1 +fi + +REPO_DIR="$1" +WORK_DIR="$2" + +_is_dir(){ + test -d "$1" && return + echo "The '$1' is not a directory." + exit 1 +} + +_is_dir "$REPO_DIR" +_is_dir "$WORK_DIR" + +# Creates a temporary build directory to run the yocto-check-layer +# script to avoid a contaminated environment. +BUILDDIR="$(mktemp -p $WORK_DIR -d -t build-yocto-check-layer-XXXX)" +source $WORK_DIR/oe-core/oe-init-build-env $BUILDDIR +git -c advice.detachedHead=false -c init.defaultBranch=main clone --quiet --shared $REPO_DIR meta-ai + +# Yocto Project layer checking tool +CMD="yocto-check-layer" +# Layer to check +CMD="$CMD meta-ai" +# Disable auto layer discovery +CMD="$CMD --no-auto" +# Layers to process for dependencies +CMD="$CMD --dependency $WORK_DIR/oe-core/meta" +# Disable automatic testing of dependencies +CMD="$CMD --no-auto-dependency" +# Set machines to all machines defined in this BSP layer +CMD="$CMD --machines $(echo $(find meta-ai/conf/machine/ -maxdepth 1 -name *.conf -exec basename {} .conf \; ))" + +exec $CMD diff --git a/ci/yocto-patchreview.sh b/ci/yocto-patchreview.sh new file mode 100755 index 0000000..033c87e --- /dev/null +++ b/ci/yocto-patchreview.sh @@ -0,0 +1,29 @@ +#!/bin/sh -e +# Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +if [ -z $1 ] || [ -z $2 ] ; then + echo "The REPO_DIR or WORK_DIR is empty and it needs to point to the corresponding directories." + echo "Please run it with:" + echo " $0 REPO_DIR WORK_DIR" + exit 1 +fi + +REPO_DIR="$1" +WORK_DIR="$2" + +_is_dir(){ + test -d "$1" && return + echo "The '$1' is not a directory." + exit 1 +} + +_is_dir "$REPO_DIR" +_is_dir "$WORK_DIR" + + +$WORK_DIR/oe-core/scripts/contrib/patchreview.py -v -b -j status.json $REPO_DIR + +# return an error if any malformed patch is found +cat $WORK_DIR/build/status.json | + python3 -c "import json,sys;obj=json.load(sys.stdin); sys.exit(1) if 'malformed-sob' in obj[0] or 'malformed-upstream-status' in obj[0] else sys.exit(0)" diff --git a/ci/yocto-pybootchartgui.sh b/ci/yocto-pybootchartgui.sh new file mode 100755 index 0000000..5f42919 --- /dev/null +++ b/ci/yocto-pybootchartgui.sh @@ -0,0 +1,41 @@ +#!/bin/sh -e +# Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +if [ -z $1 ] || [ -z $2 ] ; then + echo "The REPO_DIR or WORK_DIR is empty and it needs to point to the corresponding directories." + echo "Please run it with:" + echo " $0 REPO_DIR WORK_DIR" + exit 1 +fi + +REPO_DIR="$1" +WORK_DIR="$2" + +_is_dir(){ + test -d "$1" && return + echo "The '$1' is not a directory." + exit 1 +} + +_is_dir "$REPO_DIR" +_is_dir "$WORK_DIR" + +# latest buildstats folder +BUILDSTATS="$WORK_DIR/build/tmp/buildstats" +BUILDSTATS="$BUILDSTATS/$(ls $BUILDSTATS | tail -1)" + +# pybootchartgui tool +CMD="$CMD $WORK_DIR/oe-core/scripts/pybootchartgui/pybootchartgui.py" +# display time in minutes instead of seconds +CMD="$CMD --minutes" +# display the full time regardless of which processes are currently shown +CMD="$CMD --full-time" +# image format (png, svg, pdf); default format png +CMD="$CMD --format=svg" +# output path (file or directory) where charts are stored +CMD="$CMD --output=buildchart" +# /path/to/tmp/buildstats/// +CMD="$CMD $BUILDSTATS" + +exec $CMD diff --git a/kas/qemuarm.yml b/kas/qemuarm.yml new file mode 100644 index 0000000..17d8177 --- /dev/null +++ b/kas/qemuarm.yml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/siemens/kas/master/kas/schema-kas.json + +header: + version: 14 + +machine: qemuarm diff --git a/kas/qemux86-64.yml b/kas/qemux86-64.yml new file mode 100644 index 0000000..715e25c --- /dev/null +++ b/kas/qemux86-64.yml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/siemens/kas/master/kas/schema-kas.json + +header: + version: 14 + +machine: qemux86-64