Many fixes: Fix CI/CD; Output from failed extractors; preserve minor numbers in firmware output names

.github/workflows/build.yaml

@@ -1,60 +1,100 @@
-name: Build and Release Container
+name: Build and Test Container
 
 on:
-  push:
+  pull_request:
     branches:
       - main
 
 jobs:
-  build-and-release:
-    runs-on: ubuntu-latest
+  build-and-test:
+    runs-on: rehosting-arc
+    permissions:
+      contents: write
+      packages: write
 
     steps:
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v3
+      - name: Checkout code
+        uses: actions/checkout@v4
         with:
-          username: rehosting
-          password: ${{secrets.DOCKERHUB_TOKEN}}
+          fetch-depth: 0
 
-      - name: Pull latest Docker image for cache
-        run: docker pull rehosting/fw2tar:latest || true
+      - name: Cache firmware downloads
+        uses: actions/cache@v4
+        with:
+          path: tests/tmp_downloads
+          key: firmware-downloads-${{ hashFiles('tests/end_to_end.sh') }}
+          restore-keys: |
+            firmware-downloads-
 
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Setup runner
+      - name: Trust Harbor's self-signed certificate
         run: |
-              sudo apt-get update;
-              sudo apt-get install -yy curl jq
+          echo "Fetching certificate from ${{ secrets.REHOSTING_ARC_REGISTRY }}"
+          openssl s_client -showcerts -connect ${{ secrets.REHOSTING_ARC_REGISTRY }}:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM | sudo tee /usr/local/share/ca-certificates/harbor.crt > /dev/null
+          sudo update-ca-certificates
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: |
+            image=moby/buildkit:master
+            network=host
+          buildkitd-config-inline: |
+            [registry."${{ secrets.REHOSTING_ARC_REGISTRY }}"]
+              insecure = true
+              http = true
 
-      - name: Get next version
-        uses: reecetech/version-increment@2024.4.4
-        id: version
+      - name: Log in to Rehosting Arc Registry
+        uses: docker/login-action@v3
         with:
-          use_api: true
+          registry: ${{ secrets.REHOSTING_ARC_REGISTRY }}
+          username: ${{ secrets.REHOSTING_ARC_REGISTRY_USER }}
+          password: ${{ secrets.REHOSTING_ARC_REGISTRY_PASSWORD }}
+
 
       - name: Build Docker image
         id: docker_build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           push: true
+          load: true
+          build-args: |
+            REGISTRY=${{ secrets.REHOSTING_ARC_REGISTRY }}/proxy
+          secrets: |
+            github_token=${{ secrets.GITHUB_TOKEN }}
           tags: |
-            rehosting/fw2tar:${{ github.sha }}
-            rehosting/fw2tar:${{ steps.version.outputs.version }}
-            rehosting/fw2tar:latest
-          cache-from: type=registry,ref=rehosting/fw2tar:latest
-          cache-to: type=inline
-
-      - name: Create release
-        id: create_release
-        uses: actions/create-release@v1
+            ${{ secrets.REHOSTING_ARC_REGISTRY }}/rehosting/fw2tar:${{ github.sha }}
+          cache-from: |
+            type=registry,ref=${{ secrets.REHOSTING_ARC_REGISTRY }}/rehosting/fw2tar:cache,mode=max
+            type=registry,ref=${{ secrets.REHOSTING_ARC_REGISTRY }}/rehosting/fw2tar:cache-PR-${{github.event.number}},mode=max
+          cache-to: |
+            type=registry,ref=${{ secrets.REHOSTING_ARC_REGISTRY }}/rehosting/fw2tar:cache,mode=max
+            type=registry,ref=${{ secrets.REHOSTING_ARC_REGISTRY }}/rehosting/fw2tar:cache-PR-${{ github.event.number }}
+
+      - name: Run Tests
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          FW2TAR_IMAGE: ${{ secrets.REHOSTING_ARC_REGISTRY }}/rehosting/fw2tar:${{ github.sha }}
+          CLEANUP_DOWNLOADS: true
+        run: ./tests/end_to_end.sh
+
+      - name: Upload AX1800 Results
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ax1800_listing.txt
+          path: tests/results/ax1800_listing.txt.new
+
+      - name: Upload RAX545 Results
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: rax54s_listing.txt
+          path: tests/results/rax54s_listing.txt.new
+
+      - name: Upload RAX545 Binwalk Log
+        if: failure()
+        uses: actions/upload-artifact@v4
         with:
-          tag_name: ${{ steps.version.outputs.v-version }}
-          release_name: Release ${{ steps.version.outputs.v-version }} ${{ github.ref }}
-          body: |
-            Release ${{ steps.version.outputs.v-version }} @${{ github.ref }}
-          draft: false
-          prerelease: false
+          name: rax54s_binwalk_log.zip
+          path: tests/tmp_downloads/RAX54Sv2-V1.1.4.28.zip_out/RAX54Sv2-V1.1.4.28.binwalk.log

.github/workflows/publish.yaml

@@ -0,0 +1,93 @@
+name: Release Container
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build-and-release:
+    runs-on: rehosting-arc
+    permissions:
+      contents: write
+      packages: write
+
+    steps:
+      - name: Get next version
+        uses: reecetech/version-increment@2024.4.4
+        id: version
+        with:
+          use_api: true
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: rehosting
+          password: ${{secrets.DOCKERHUB_TOKEN}}
+
+      - name: Trust Harbor's self-signed certificate
+        run: |
+          echo "Fetching certificate from ${{ secrets.REHOSTING_ARC_REGISTRY }}"
+          openssl s_client -showcerts -connect ${{ secrets.REHOSTING_ARC_REGISTRY }}:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM | sudo tee /usr/local/share/ca-certificates/harbor.crt > /dev/null
+          sudo update-ca-certificates
+
+      - name: Log in to Rehosting Arc Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ secrets.REHOSTING_ARC_REGISTRY }}
+          username: ${{ secrets.REHOSTING_ARC_REGISTRY_USER }}
+          password: ${{ secrets.REHOSTING_ARC_REGISTRY_PASSWORD }}
+
+      - name: Install dependencies and label git workspace safe
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install git curl jq gzip tmux
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: |
+            image=moby/buildkit:master
+            network=host
+          buildkitd-config-inline: |
+            [registry."${{ secrets.REHOSTING_ARC_REGISTRY }}"]
+              insecure = true
+              http = true
+
+      - name: Build Docker image and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          build-args: |
+            REGISTRY=${{ secrets.REHOSTING_ARC_REGISTRY }}/proxy
+          secrets: |
+            github_token=${{ secrets.GITHUB_TOKEN }}
+          tags: |
+            rehosting/fw2tar:${{ github.sha }}
+            rehosting/fw2tar:${{ steps.version.outputs.v-version }}
+            rehosting/fw2tar:latest
+          cache-from: |
+            type=registry,ref=${{ secrets.REHOSTING_ARC_REGISTRY }}/rehosting/fw2tar:cache,mode=max
+          cache-to: |
+            type=registry,ref=${{ secrets.REHOSTING_ARC_REGISTRY }}/rehosting/fw2tar:cache,mode=max
+
+      - name: Create release
+        id: create_release
+        uses: softprops/action-gh-release@v2.3.2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ steps.version.outputs.v-version }}
+          name: Release ${{ steps.version.outputs.v-version }} ${{ github.ref }}
+          body: |
+            Release ${{ steps.version.outputs.v-version }} @${{ github.ref }}
+          draft: false
+          generate_release_notes: true
+          prerelease: false

.gitignore

@@ -2,6 +2,7 @@
 *.sif
 *.tar.gz
 *.new
+tests/tmp_downloads/
 
 
 # Added by cargo

Dockerfile

@@ -1,4 +1,10 @@
-FROM ubuntu:22.04
+# Accept registry as build argument (not sensitive)
+ARG REGISTRY="docker.io"
+FROM ${REGISTRY}/ubuntu:22.04
+
+# Copy download helper early for use throughout the build
+COPY download_github_asset.sh /usr/local/bin/download_github_asset.sh
+RUN chmod +x /usr/local/bin/download_github_asset.sh
 
 # Install unblob dependencies, curl, and fakeroot
 ENV DEBIAN_FRONTEND=noninteractive
@@ -9,6 +15,14 @@ ENV HOME=/root
 ENV FW2TAR_LOG=warn
 ENV FW2TAR_LOG_STYLE=always
 
+# Debug: Show if GitHub token is available (using BuildKit secrets)
+RUN --mount=type=secret,id=github_token \
+    if [ -s /run/secrets/github_token ]; then \
+        echo "GitHub token is available for authenticated downloads"; \
+    else \
+        echo "No GitHub token provided - using unauthenticated downloads"; \
+    fi
+
 RUN apt-get update && \
   apt-get install -q -y \
     android-sdk-libsparse-utils \
@@ -86,7 +100,11 @@ RUN pip install --upgrade pip && \
     python3 -m pip install python-lzo==1.14 && \
     poetry config virtualenvs.create false
 
-RUN curl -L -o sasquatch_1.0.deb "https://github.com/onekey-sec/sasquatch/releases/download/sasquatch-v4.5.1-4/sasquatch_1.0_$(dpkg --print-architecture).deb" && \
+RUN --mount=type=secret,id=github_token \
+    GITHUB_TOKEN="$(cat /run/secrets/github_token 2>/dev/null || echo '')" \
+    /usr/local/bin/download_github_asset.sh \
+        "https://github.com/onekey-sec/sasquatch/releases/download/sasquatch-v4.5.1-4/sasquatch_1.0_$(dpkg --print-architecture).deb" \
+        sasquatch_1.0.deb && \
     dpkg -i sasquatch_1.0.deb && \
     rm sasquatch_1.0.deb
 
@@ -103,19 +121,23 @@ RUN git clone --depth=1 https://github.com/davidribyrne/cramfs.git /cramfs && \
 RUN git clone --depth=1 https://github.com/rehosting/unblob.git /unblob
 RUN cd /unblob && poetry install --only main
 
-# Install Rust
-RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+# Install Rust with cache mount for better performance
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    curl https://sh.rustup.rs -sSf | bash -s -- -y
 
 # Add .cargo/bin to PATH
 ENV PATH="/root/.cargo/bin:${PATH}"
 
-# Install binwalk v3
-RUN cargo install binwalk
+# Install binwalk v3 with cache mount
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    cargo install binwalk
 
-# Install fw2tar
+# Install fw2tar with cache mount
 COPY ./Cargo.toml ./Cargo.lock /fw2tar_src/
 COPY ./src /fw2tar_src/src/
-RUN cargo install --path /fw2tar_src
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/fw2tar_src/target \
+    cargo install --path /fw2tar_src
 
 # Explicitly install unblob deps - mostly captured above, but some of the .debs get updated and installed via curl
 RUN sh -c /unblob/unblob/install-deps.sh
@@ -136,7 +158,11 @@ RUN --mount=type=ssh git clone git@github.com:rehosting/fakeroot.git /fakeroot &
 
 # Patch to fix unblob #767 that hasn't yet been upstreamed. Pip install didn't work. I don't understand poetry
 #RUN pip install git+https://github.com/qkaiser/arpy.git
-RUN curl "https://raw.githubusercontent.com/qkaiser/arpy/23faf88a88488c41fc4348ea2b70996803f84f40/arpy.py" -o /usr/local/lib/python3.10/dist-packages/arpy.py
+RUN --mount=type=secret,id=github_token \
+    GITHUB_TOKEN="$(cat /run/secrets/github_token 2>/dev/null || echo '')" \
+    /usr/local/bin/download_github_asset.sh \
+        "https://raw.githubusercontent.com/qkaiser/arpy/23faf88a88488c41fc4348ea2b70996803f84f40/arpy.py" \
+        /usr/local/lib/python3.10/dist-packages/arpy.py
 
 # Copy wrapper script into container so we can copy out - note we don't put it on guest path
 COPY ./fw2tar /usr/local/src/fw2tar_wrapper

download_github_asset.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -euo pipefail
+
+# Simple download script with GitHub token support
+# Usage: download_github_asset.sh <url> [output_file]
+
+URL="$1"
+OUTPUT_FILE="${2:-}"
+
+# If GITHUB_TOKEN is set and URL is from GitHub, use it
+CURL_OPTS=(-L --connect-timeout 30 --max-time 300)
+if [[ -n "${GITHUB_TOKEN:-}" ]] && [[ "$URL" == *"github.com"* ]]; then
+    CURL_OPTS+=(-H "Authorization: token $GITHUB_TOKEN")
+    echo "Using GitHub token for authenticated request"
+fi
+
+# Add output file if specified
+if [[ -n "$OUTPUT_FILE" ]]; then
+    CURL_OPTS+=(-o "$OUTPUT_FILE")
+fi
+
+echo "Downloading: $URL"
+curl "${CURL_OPTS[@]}" "$URL"