Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions experiments/debian12-13/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
out/
logs/
15 changes: 15 additions & 0 deletions experiments/debian12-13/Dockerfile.debian12
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
FROM debian:bookworm-slim

ENV DEBIAN_FRONTEND=noninteractive

RUN apt-get update && apt-get install -y --no-install-recommends \
bash \
make \
gcc-arm-none-eabi \
binutils-arm-none-eabi \
libnewlib-arm-none-eabi \
ca-certificates \
&& rm -rf /var/lib/apt/lists/*

WORKDIR /workspace

15 changes: 15 additions & 0 deletions experiments/debian12-13/Dockerfile.debian13
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
FROM debian:13-slim

ENV DEBIAN_FRONTEND=noninteractive

RUN apt-get update && apt-get install -y --no-install-recommends \
bash \
make \
gcc-arm-none-eabi \
binutils-arm-none-eabi \
libnewlib-arm-none-eabi \
ca-certificates \
&& rm -rf /var/lib/apt/lists/*

WORKDIR /workspace

19 changes: 19 additions & 0 deletions experiments/debian12-13/Dockerfile.debian13-remediated
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
FROM debian:13-slim

ENV DEBIAN_FRONTEND=noninteractive

RUN apt-get update && apt-get install -y --no-install-recommends \
bash \
make \
ca-certificates \
&& echo "deb http://deb.debian.org/debian bookworm main" > /etc/apt/sources.list.d/bookworm.list \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
gcc-arm-none-eabi=15:12.2.rel1-1 \
binutils-arm-none-eabi=2.40-2+18+b1 \
libnewlib-arm-none-eabi=3.3.0-1.3+deb12u1 \
libnewlib-dev=3.3.0-1.3+deb12u1 \
&& rm -rf /var/lib/apt/lists/*

WORKDIR /workspace

43 changes: 43 additions & 0 deletions experiments/debian12-13/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# Debian 12 vs 13 Experiment

This experiment runs a fair, per-image comparison.

- All images run the same build workflow: `make binaries`
- Output is reported as per-image `PASS/FAIL`
- `debian13-remediated` can be included as an extra case

## Run

```bash
./experiments/debian12-13/run-compare.sh
```

Optional modes:

```bash
# Skip remediation path
RUN_REMEDIATION=0 ./experiments/debian12-13/run-compare.sh

# Return non-zero if any image fails
FAIL_ON_ANY=1 ./experiments/debian12-13/run-compare.sh
```

## What it does

1. Builds images:
- `experiments/debian12-13/Dockerfile.debian12`
- `experiments/debian12-13/Dockerfile.debian13`
2. Optionally builds remediation image:
- `experiments/debian12-13/Dockerfile.debian13-remediated`
3. Captures toolchain versions (`arm-none-eabi-gcc`, `arm-none-eabi-ld`)
4. Runs the same build command for each image:
- `make binaries BUILD=<case-output-dir>`
5. Uses isolated output dirs under this folder:
- `experiments/debian12-13/out/build-debian12`
- `experiments/debian12-13/out/build-debian13`
- `experiments/debian12-13/out/build-debian13-remediated`
6. Prints a result matrix with:
- case name
- `PASS/FAIL`
- exit code
- gcc/ld version lines
124 changes: 124 additions & 0 deletions experiments/debian12-13/run-compare.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
#!/usr/bin/env bash
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
cd "$ROOT_DIR"

RUN_REMEDIATION="${RUN_REMEDIATION:-1}"
FAIL_ON_ANY="${FAIL_ON_ANY:-0}"

if [[ -t 1 ]]; then
COLOR_PASS=$'\033[32m'
COLOR_FAIL=$'\033[31m'
COLOR_RESET=$'\033[0m'
else
COLOR_PASS=""
COLOR_FAIL=""
COLOR_RESET=""
fi

declare -a NAMES
declare -a IMAGES
declare -a DOCKERFILES
declare -a OUTDIRS
declare -a GCC_LINES
declare -a LD_LINES
declare -a EXIT_CODES
declare -a RESULTS

add_case() {
NAMES+=("$1")
IMAGES+=("$2")
DOCKERFILES+=("$3")
OUTDIRS+=("$4")
}

add_case \
"debian12" \
"${IMAGE12:-airbreak-exp:debian12}" \
"$SCRIPT_DIR/Dockerfile.debian12" \
"experiments/debian12-13/out/build-debian12"

add_case \
"debian13" \
"${IMAGE13:-airbreak-exp:debian13}" \
"$SCRIPT_DIR/Dockerfile.debian13" \
"experiments/debian12-13/out/build-debian13"

if [[ "$RUN_REMEDIATION" == "1" ]]; then
add_case \
"debian13-remediated" \
"${IMAGE13_REMEDIATED:-airbreak-exp:debian13-remediated}" \
"$SCRIPT_DIR/Dockerfile.debian13-remediated" \
"experiments/debian12-13/out/build-debian13-remediated"
fi

run_in_image() {
local image="$1"
shift
docker run --rm \
--mount "type=bind,source=$ROOT_DIR,target=/workspace" \
-w /workspace \
"$image" \
bash -lc "$*"
}

echo "[0/4] Clean experiment output directory"
rm -rf "$SCRIPT_DIR/out"
mkdir -p "$SCRIPT_DIR/out"

echo "[1/4] Build images"
for i in "${!NAMES[@]}"; do
echo " - ${NAMES[$i]} (${IMAGES[$i]})"
docker build -f "${DOCKERFILES[$i]}" -t "${IMAGES[$i]}" .
done

echo "[2/4] Capture toolchain versions"
for i in "${!NAMES[@]}"; do
gcc_line="$(run_in_image "${IMAGES[$i]}" "arm-none-eabi-gcc --version | head -n 1")"
ld_line="$(run_in_image "${IMAGES[$i]}" "arm-none-eabi-ld --version | head -n 1")"
GCC_LINES+=("$gcc_line")
LD_LINES+=("$ld_line")
echo " - ${NAMES[$i]} gcc: $gcc_line"
echo " - ${NAMES[$i]} ld : $ld_line"
done

echo "[3/4] Run same build workflow for each image (fair comparison)"
for i in "${!NAMES[@]}"; do
outdir="${OUTDIRS[$i]}"
set +e
run_in_image "${IMAGES[$i]}" "rm -rf $outdir; mkdir -p $outdir; make binaries BUILD=$outdir"
code=$?
set -e
EXIT_CODES+=("$code")
if [[ "$code" -eq 0 ]]; then
RESULTS+=("PASS")
echo " - ${NAMES[$i]}: ${COLOR_PASS}PASS${COLOR_RESET}"
else
RESULTS+=("FAIL")
echo " - ${NAMES[$i]}: ${COLOR_FAIL}FAIL${COLOR_RESET} (exit=$code)"
fi
done

echo "[4/4] Result matrix"
for i in "${!NAMES[@]}"; do
if [[ "${RESULTS[$i]}" == "PASS" ]]; then
status_colored="${COLOR_PASS}${RESULTS[$i]}${COLOR_RESET}"
else
status_colored="${COLOR_FAIL}${RESULTS[$i]}${COLOR_RESET}"
fi
echo "- ${NAMES[$i]} | ${status_colored} | exit=${EXIT_CODES[$i]}"
echo " gcc: ${GCC_LINES[$i]}"
echo " ld : ${LD_LINES[$i]}"
done

if [[ "$FAIL_ON_ANY" == "1" ]]; then
for code in "${EXIT_CODES[@]}"; do
if [[ "$code" -ne 0 ]]; then
exit 1
fi
done
fi

exit 0