diff --git a/.github/actions/setup-bink/action.yml b/.github/actions/setup-bink/action.yml new file mode 100644 index 0000000..cc25818 --- /dev/null +++ b/.github/actions/setup-bink/action.yml @@ -0,0 +1,119 @@ +name: Setup Bink +description: Configure the runner, build bink, and cache container images + +inputs: + cache-key-prefix: + description: Prefix for the image cache key + required: true + +runs: + using: composite + steps: + - name: Configure kernel for nested containers + shell: bash + run: | + sudo aa-teardown 2>/dev/null || true + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 + + - name: Enable KSM (Kernel Same-page Merging) + shell: bash + run: | + sudo sh -c 'echo 1 > /sys/kernel/mm/ksm/run' + sudo sh -c 'echo 5000 > /sys/kernel/mm/ksm/pages_to_scan' + cat /sys/kernel/mm/ksm/run + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + + - name: Install system dependencies + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y \ + podman \ + libgpgme-dev \ + libbtrfs-dev \ + libdevmapper-dev \ + pkg-config + + - name: Set up KVM + shell: bash + run: | + sudo chmod 666 /dev/kvm + ls -la /dev/kvm + + - name: Configure Podman + shell: bash + run: | + podman --version + sudo mkdir -p /etc/containers + echo '{"defaultAction":"SCMP_ACT_ALLOW"}' | sudo tee /etc/containers/seccomp.json + printf '[containers]\napparmor_profile = "unconfined"\nseccomp_profile = "/etc/containers/seccomp.json"\n' | sudo tee /etc/containers/containers.conf + grep -q '^root:' /etc/subuid || echo 'root:100000:65536' | sudo tee -a /etc/subuid + grep -q '^root:' /etc/subgid || echo 'root:100000:65536' | sudo tee -a /etc/subgid + sudo systemctl start podman.socket + sudo podman info --format '{{.Store.GraphRoot}}' + + - name: Build bink binary + shell: bash + run: sudo make build-bink + + - name: Verify prerequisites + shell: bash + run: | + test -f ./bink + sudo podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" + df -h / + free -h + + - name: Get image digests + id: digests + shell: bash + run: | + ALL_DIGESTS="" + for img in $BINK_IMAGES $EXTERNAL_IMAGES; do + digest=$(skopeo inspect --no-creds "docker://${img}" --format '{{.Digest}}') + echo "${img}: ${digest}" + ALL_DIGESTS="${ALL_DIGESTS}${digest}" + done + echo "hash=$(echo -n "${ALL_DIGESTS}" | sha256sum | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + + - name: Restore cached images + id: image-cache + uses: actions/cache/restore@v6 + with: + path: /tmp/podman-image-cache + key: podman-images-v2-${{ inputs.cache-key-prefix }}-${{ steps.digests.outputs.hash }} + + - name: Load cached images + if: steps.image-cache.outputs.cache-hit == 'true' + shell: bash + run: | + for f in /tmp/podman-image-cache/*.tar; do + sudo podman load -i "$f" + done + sudo podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" + + - name: Pre-pull container images + if: steps.image-cache.outputs.cache-hit != 'true' + shell: bash + run: | + mkdir -p /tmp/podman-image-cache + for img in $BINK_IMAGES $EXTERNAL_IMAGES; do + sudo podman pull "$img" + name=$(echo "$img" | sed 's|[/:]|_|g') + sudo podman save -o "/tmp/podman-image-cache/${name}.tar" "$img" + done + sudo chown -R $(id -u):$(id -g) /tmp/podman-image-cache + + - name: Save image cache + if: steps.image-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + with: + path: /tmp/podman-image-cache + key: podman-images-v2-${{ inputs.cache-key-prefix }}-${{ steps.digests.outputs.hash }} diff --git a/.github/workflows/integration-tests-composefs.yml b/.github/workflows/integration-tests-composefs.yml index ee7dec0..be2c541 100644 --- a/.github/workflows/integration-tests-composefs.yml +++ b/.github/workflows/integration-tests-composefs.yml @@ -25,106 +25,12 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 - - name: Configure kernel for nested containers - run: | - sudo aa-teardown 2>/dev/null || true - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 - sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 - - - name: Enable KSM (Kernel Same-page Merging) - run: | - sudo sh -c 'echo 1 > /sys/kernel/mm/ksm/run' - sudo sh -c 'echo 5000 > /sys/kernel/mm/ksm/pages_to_scan' - cat /sys/kernel/mm/ksm/run - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - cache: true - - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - podman \ - libgpgme-dev \ - libbtrfs-dev \ - libdevmapper-dev \ - pkg-config - - - name: Set up KVM - run: | - sudo chmod 666 /dev/kvm - ls -la /dev/kvm - - - name: Configure Podman - run: | - podman --version - sudo mkdir -p /etc/containers - echo '{"defaultAction":"SCMP_ACT_ALLOW"}' | sudo tee /etc/containers/seccomp.json - printf '[containers]\napparmor_profile = "unconfined"\nseccomp_profile = "/etc/containers/seccomp.json"\n' | sudo tee /etc/containers/containers.conf - grep -q '^root:' /etc/subuid || echo 'root:100000:65536' | sudo tee -a /etc/subuid - grep -q '^root:' /etc/subgid || echo 'root:100000:65536' | sudo tee -a /etc/subgid - sudo systemctl start podman.socket - sudo podman info --format '{{.Store.GraphRoot}}' - - - name: Build bink binary - run: sudo make build-bink - - - name: Verify prerequisites - run: | - test -f ./bink - sudo podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" - df -h / - free -h - - - name: Get image digests - id: digests - run: | - ALL_DIGESTS="" - for img in $BINK_IMAGES $EXTERNAL_IMAGES; do - digest=$(skopeo inspect --no-creds "docker://${img}" --format '{{.Digest}}') - echo "${img}: ${digest}" - ALL_DIGESTS="${ALL_DIGESTS}${digest}" - done - echo "hash=$(echo -n "${ALL_DIGESTS}" | sha256sum | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" - - - name: Restore cached images - id: image-cache - uses: actions/cache/restore@v4 - with: - path: /tmp/podman-image-cache - key: podman-images-v2-composefs-${{ steps.digests.outputs.hash }} - - - name: Load cached images - if: steps.image-cache.outputs.cache-hit == 'true' - run: | - for f in /tmp/podman-image-cache/*.tar; do - sudo podman load -i "$f" - done - sudo podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" - - - name: Pre-pull container images - if: steps.image-cache.outputs.cache-hit != 'true' - run: | - mkdir -p /tmp/podman-image-cache - for img in $BINK_IMAGES $EXTERNAL_IMAGES; do - sudo podman pull "$img" - name=$(echo "$img" | sed 's|[/:]|_|g') - sudo podman save -o "/tmp/podman-image-cache/${name}.tar" "$img" - done - sudo chown -R $(id -u):$(id -g) /tmp/podman-image-cache - - - name: Save image cache - if: steps.image-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 + - name: Setup bink + uses: ./.github/actions/setup-bink with: - path: /tmp/podman-image-cache - key: podman-images-v2-composefs-${{ steps.digests.outputs.hash }} + cache-key-prefix: composefs - name: Run composefs integration tests run: sudo make test-integration-composefs @@ -139,7 +45,7 @@ jobs: - name: Upload logs if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: test-logs-composefs path: /tmp/bink-logs/ diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index c47614e..4601135 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -21,6 +21,21 @@ env: quay.io/libpod/busybox:latest jobs: + supported-versions: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Get supported Kubernetes versions + id: set-matrix + run: | + DEFAULT="1.35" + # Exclude the default version — the integration-tests job already runs the full suite for it + MATRIX=$(curl -s https://endoflife.date/api/kubernetes.json | \ + jq -c --arg default "$DEFAULT" \ + '[.[] | select(.eol > (now | strftime("%Y-%m-%d"))) | .cycle][:3] | map(select(. != $default))') + echo "matrix=${MATRIX}" >> "$GITHUB_OUTPUT" + integration-tests: runs-on: ubuntu-latest timeout-minutes: 120 @@ -29,117 +44,66 @@ jobs: - name: Checkout code uses: actions/checkout@v7 - - name: Configure kernel for nested containers - run: | - # Unload all AppArmor profiles from the kernel. - # Stopping the service alone doesn't unload them - the passt profile - # blocks remount operations needed for passt's self-sandboxing. - sudo aa-teardown 2>/dev/null || true - # Allow unprivileged user namespace creation (needed by passt inside containers) - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 - sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 - - - name: Enable KSM (Kernel Same-page Merging) - run: | - sudo sh -c 'echo 1 > /sys/kernel/mm/ksm/run' - sudo sh -c 'echo 5000 > /sys/kernel/mm/ksm/pages_to_scan' - cat /sys/kernel/mm/ksm/run - - - name: Set up Go - uses: actions/setup-go@v6 + - name: Setup bink + uses: ./.github/actions/setup-bink with: - go-version-file: go.mod - cache: true + cache-key-prefix: integration - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - podman \ - libgpgme-dev \ - libbtrfs-dev \ - libdevmapper-dev \ - pkg-config - - - name: Set up KVM - run: | - # Re-apply after apt-get which may upgrade udev and reset device permissions - sudo chmod 666 /dev/kvm - ls -la /dev/kvm + - name: Run integration tests + run: sudo make test-integration TEST_PROCS=2 + timeout-minutes: 90 + env: + CONTAINER_HOST: unix:///run/podman/podman.sock - - name: Configure Podman - run: | - podman --version - # Configure containers.conf BEFORE starting the podman socket - sudo mkdir -p /etc/containers - echo '{"defaultAction":"SCMP_ACT_ALLOW"}' | sudo tee /etc/containers/seccomp.json - printf '[containers]\napparmor_profile = "unconfined"\nseccomp_profile = "/etc/containers/seccomp.json"\n' | sudo tee /etc/containers/containers.conf - # Ensure subuid/subgid for root (needed for userns=auto) - grep -q '^root:' /etc/subuid || echo 'root:100000:65536' | sudo tee -a /etc/subuid - grep -q '^root:' /etc/subgid || echo 'root:100000:65536' | sudo tee -a /etc/subgid - sudo systemctl start podman.socket - sudo podman info --format '{{.Store.GraphRoot}}' - - - name: Build bink binary - run: sudo make build-bink - - - name: Verify prerequisites - run: | - test -f ./bink - sudo podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" - df -h / - free -h + - name: Collect logs + if: failure() + run: .github/collect-logs.sh - - name: Get image digests - id: digests - run: | - ALL_DIGESTS="" - for img in $BINK_IMAGES $EXTERNAL_IMAGES; do - digest=$(skopeo inspect --no-creds "docker://${img}" --format '{{.Digest}}') - echo "${img}: ${digest}" - ALL_DIGESTS="${ALL_DIGESTS}${digest}" - done - echo "hash=$(echo -n "${ALL_DIGESTS}" | sha256sum | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" - - - name: Restore cached images - id: image-cache - uses: actions/cache/restore@v6 + - name: Upload logs + if: failure() + uses: actions/upload-artifact@v7 with: - path: /tmp/podman-image-cache - key: podman-images-v2-integration-${{ steps.digests.outputs.hash }} + name: test-logs + path: /tmp/bink-logs/ - - name: Load cached images - if: steps.image-cache.outputs.cache-hit == 'true' + - name: Cleanup test clusters + if: always() run: | - for f in /tmp/podman-image-cache/*.tar; do - sudo podman load -i "$f" - done - sudo podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" + sudo podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \ + xargs -r sudo podman rm -f 2>/dev/null || true + sudo podman volume prune -f 2>/dev/null || true - - name: Pre-pull container images - if: steps.image-cache.outputs.cache-hit != 'true' - run: | - mkdir -p /tmp/podman-image-cache - for img in $BINK_IMAGES $EXTERNAL_IMAGES; do - sudo podman pull "$img" - name=$(echo "$img" | sed 's|[/:]|_|g') - sudo podman save -o "/tmp/podman-image-cache/${name}.tar" "$img" - done - sudo chown -R $(id -u):$(id -g) /tmp/podman-image-cache - - - name: Save image cache - if: steps.image-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v6 + integration-tests-k8s-versions: + needs: supported-versions + runs-on: ubuntu-latest + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + kube-minor: ${{ fromJson(needs.supported-versions.outputs.matrix) }} + + env: + BINK_NODE_IMAGE: ghcr.io/bootc-dev/bink/node:v${{ matrix.kube-minor }}-fedora-44-disk + BINK_IMAGES: >- + ghcr.io/bootc-dev/bink/cluster:latest + ghcr.io/bootc-dev/bink/node:v${{ matrix.kube-minor }}-fedora-44-disk + ghcr.io/bootc-dev/bink/dns:latest + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Setup bink + uses: ./.github/actions/setup-bink with: - path: /tmp/podman-image-cache - key: podman-images-v2-integration-${{ steps.digests.outputs.hash }} + cache-key-prefix: k8s-${{ matrix.kube-minor }} - - name: Run integration tests - run: sudo make test-integration TEST_PROCS=2 + - name: Run integration tests (K8s ${{ matrix.kube-minor }}) + run: sudo make test-integration GINKGO_FOCUS="should create and initialize a complete Kubernetes cluster" timeout-minutes: 90 env: CONTAINER_HOST: unix:///run/podman/podman.sock + BINK_NODE_IMAGE: ${{ env.BINK_NODE_IMAGE }} - name: Collect logs if: failure() @@ -149,7 +113,7 @@ jobs: if: failure() uses: actions/upload-artifact@v7 with: - name: test-logs + name: test-logs-k8s-${{ matrix.kube-minor }} path: /tmp/bink-logs/ - name: Cleanup test clusters diff --git a/test/integration/cluster_test.go b/test/integration/cluster_test.go index d9b537b..8b8c3c8 100644 --- a/test/integration/cluster_test.go +++ b/test/integration/cluster_test.go @@ -41,11 +41,7 @@ var _ = Describe("Cluster Lifecycle", func() { targetImgRef := "registry.cluster.local:5000/node:latest" By("Creating cluster with --expose, custom node name, memory ballooning, and target-imgref") - args := []string{"cluster", "start", "--cluster-name", clusterName, "--api-port", "0", "--memory", "1900", "--max-memory", "4096", "--node-name", customNodeName, "--expose", kubeconfigPath, "--target-imgref", targetImgRef} - if nodeImage := os.Getenv("BINK_NODE_IMAGE"); nodeImage != "" { - args = append(args, "--node-image", nodeImage) - } - cmd := helpers.BinkCmd(args...) + cmd := helpers.BinkCmd("cluster", "start", "--cluster-name", clusterName, "--api-port", "0", "--memory", "1900", "--max-memory", "4096", "--node-name", customNodeName, "--expose", kubeconfigPath, "--target-imgref", targetImgRef, "--node-image", helpers.NodeImage()) session := helpers.RunCommand(cmd) By("Verifying cluster creation command succeeded") diff --git a/test/integration/helpers/cluster.go b/test/integration/helpers/cluster.go index 291d94e..7a7e206 100644 --- a/test/integration/helpers/cluster.go +++ b/test/integration/helpers/cluster.go @@ -5,15 +5,26 @@ package helpers import ( "fmt" + "os" "os/exec" "time" + "github.com/bootc-dev/bink/internal/config" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/onsi/gomega/gexec" ) +// NodeImage returns the node image to use for tests. +// It reads BINK_NODE_IMAGE env var, falling back to config.DefaultNodeImage. +func NodeImage() string { + if img := os.Getenv("BINK_NODE_IMAGE"); img != "" { + return img + } + return config.DefaultNodeImage +} + // GenerateTestClusterName creates a unique cluster name for testing func GenerateTestClusterName() string { return fmt.Sprintf("test-bink-%s", uuid.New().String()[:8]) @@ -45,7 +56,7 @@ func RunCommand(cmd *exec.Cmd, timeout ...time.Duration) *gexec.Session { // Uses auto-assigned ports (--api-port 0) to avoid port conflicts in tests func CreateCluster(name string) { GinkgoWriter.Printf("Creating cluster: %s (with auto-assigned API port)\n", name) - cmd := BinkCmd("cluster", "start", "--cluster-name", name, "--api-port", "0", "--memory", "1900", "--max-memory", "4096") + cmd := BinkCmd("cluster", "start", "--cluster-name", name, "--api-port", "0", "--memory", "1900", "--max-memory", "4096", "--node-image", NodeImage()) session := RunCommand(cmd, 10*time.Minute) Expect(session.ExitCode()).To(Equal(0), "Failed to create cluster: %s", string(session.Err.Contents())) } @@ -53,7 +64,7 @@ func CreateCluster(name string) { // CreateClusterWithNodeName creates a cluster with a custom control-plane node name func CreateClusterWithNodeName(name, nodeName string) { GinkgoWriter.Printf("Creating cluster: %s with node name: %s (with auto-assigned API port)\n", name, nodeName) - cmd := BinkCmd("cluster", "start", "--cluster-name", name, "--node-name", nodeName, "--api-port", "0", "--memory", "1900", "--max-memory", "4096") + cmd := BinkCmd("cluster", "start", "--cluster-name", name, "--node-name", nodeName, "--api-port", "0", "--memory", "1900", "--max-memory", "4096", "--node-image", NodeImage()) session := RunCommand(cmd, 10*time.Minute) Expect(session.ExitCode()).To(Equal(0), "Failed to create cluster: %s", string(session.Err.Contents())) }