develop → main

.github/workflows/build.yaml

@@ -1,26 +1,39 @@
-name: Funnel Build and Cache
+name: Build Funnel
 
 on:
   workflow_call:
+    inputs:
+      cache:
+        type: boolean
+        default: true
 
 jobs:
   build:
     runs-on: ubuntu-latest
 
     steps:
     - name: Check out code
-      uses: actions/checkout@v4
+      uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+    - name: Cache Funnel binary
+      id: binary-cache
+      uses: actions/cache@v4
+      with:
+        path: funnel
+        key: funnel-binary-${{ hashFiles('**/*.go', 'go.mod', 'go.sum') }}
 
     - name: Set up Go 1.x
-      uses: actions/setup-go@v5
+      if: steps.binary-cache.outputs.cache-hit != 'true'
+      uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
       with:
         go-version-file: go.mod
 
     - name: Build Funnel
+      if: steps.binary-cache.outputs.cache-hit != 'true'
       run: make build
 
     - name: Upload Funnel binary as artifact
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
       with:
         name: funnel
         path: funnel

.github/workflows/compliance.yaml

@@ -19,11 +19,7 @@
 name: Compliance Test
 
 on:
-  # Run on push to main/master/develop (e.g. after PR is merged)
   push:
-    branches:
-      - main
-      - master
 
 # Cancel redundant jobs (if a new job is triggered for the same workflow and ref)
 concurrency:
@@ -38,41 +34,55 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        version: [1.0.0, 1.1.0]
-        db: ["boltdb", "mongodb"]
+        version:
+          - 1.0.0
+          - 1.1.0
+        # TODO: Re-enable postgres and s3 tests once we have these backends working in CI
+        db:
+          - local
+          # - postgres
         # Temporary: Only test local compute backend until we bring up a K8s cluster in Github Actions
         # compute: ["local", "kubernetes"]
-        compute: ["local"]
-        storage: ["local", "s3"]
+        compute:
+          - local
+        storage:
+          - local
+          # - s3
     needs: build
     runs-on: ubuntu-latest
     steps:
-      # Required to access the 'tests/mongo.config.yml' file
+      # Required to access the tests config files (e.g. tests/postgres.config.yml)
       # Perhaps uploading it as an artifact would be more efficient?
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2
 
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
         with:
           name: funnel
 
       - name: Start Funnel server
         run: |
           touch config.yml
 
-          if [ ${{ matrix.db }} = "mongodb" ]; then
-            make start-mongodb
-            cat `pwd`/tests/mongo.config.yml >> config.yml
+          if [ ${{ matrix.db }} = "postgres" ]; then
+            make start-postgres
+            for i in $(seq 1 30); do
+              docker exec funnel-postgres-test pg_isready -U postgres && break
+              echo "Waiting for Postgres... ($i/30)"
+              sleep 1
+              [ $i -eq 30 ] && { echo "Postgres did not become ready in time"; exit 1; }
+            done
+            cat `pwd`/tests/postgres.config.yml >> config.yml
           fi
 
           if [ ${{ matrix.storage }} = "s3" ]; then
             docker run -d -p 9000:9000 --name minio \
-                       -e "MINIO_ROOT_USER=minioadmin" \
-                       -e "MINIO_ROOT_PASSWORD=minioadmin" \
+                       -e "MINIO_ROOT_USER=example-key" \
+                       -e "MINIO_ROOT_PASSWORD=example-secret" \
                        -v /tmp/data:/data \
                        -v /tmp/config:/root/.minio \
                        minio/minio server /data
-            cat `pwd`/tests/s3.config.yml >> config.yml
+            cat `pwd`/tests/minio-s3.config.yml >> config.yml
           fi
 
           if [ ${{ matrix.compute }} = "kubernetes" ]; then
@@ -82,6 +92,23 @@ jobs:
 
           chmod +x funnel
           ./funnel server run --config `pwd`/config.yml &> funnel.logs &
+          echo $! > funnel.pid
+
+      - name: Wait for Funnel to be ready
+        run: |
+          for i in $(seq 1 10); do
+            if ! kill -0 $(cat funnel.pid) 2>/dev/null; then
+              echo "Funnel process exited unexpectedly:"
+              cat funnel.logs
+              exit 1
+            fi
+            curl -sf http://localhost:8000/v1/service-info && exit 0
+            echo "Waiting for Funnel... ($i/10)"
+            sleep 1
+          done
+          echo "Funnel did not become ready in time:"
+          cat funnel.logs
+          exit 1
 
       - name: Run OpenAPI Test Runner
         run: |
@@ -91,4 +118,6 @@ jobs:
           source venv/bin/activate
           pip install -r requirements.txt
           python setup.py install
-          openapi-test-runner report --version "${{ matrix.version }}" --server "http://localhost:8000/"
+          output=$(openapi-test-runner report --version "${{ matrix.version }}" --server "http://localhost:8000/" 2>&1)
+          echo "$output"
+          echo "$output" | grep -q "Failed - 0" || { echo "Compliance tests failed"; exit 1; }

.github/workflows/docker.yaml

@@ -1,14 +1,18 @@
 # This workflow builds and pushes a multi-arch image to Quay.io
-# Adapted from actions-arm64-native-example by @gartnera
-# ref: https://github.com/gartnera/actions-arm64-native-example/blob/main/.github/workflows/build.yml
+# Adapted from actions-arm64-native_example by @gartnera
+# ref: https://github.com/gartnera/actions-arm64-native_example/blob/main/.github/workflows/build.yml
 
-name: build
+name: Build + Push Docker Image
 
 on:
   push:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
-  QUAY_REPO: quay.io/${{ github.repository_owner }}/${{ github.event.repository.name }}
+  QUAY_REPO: quay.io/ohsu-comp-bio/funnel
 
 jobs:
   build:
@@ -29,31 +33,33 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
         with:
           images: |
             ${{ env.QUAY_REPO }}
 
       - name: Login to Quay.io
-        uses: docker/login-action@v3
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
         with:
           registry: quay.io
           username: ${{ secrets.QUAY_USERNAME }}
           password: ${{ secrets.QUAY_ROBOT_TOKEN }}
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
 
       - name: Build and push by digest
         id: build
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
         with:
           platforms: ${{ matrix.platform }}
           labels: ${{ steps.meta.outputs.labels }}
           outputs: type=image,name=${{ env.QUAY_REPO }},push-by-digest=true,name-canonical=true,push=true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
       - name: Export digest
         run: |
@@ -62,7 +68,7 @@ jobs:
           touch "${{ runner.temp }}/digests/${digest#sha256:}"          
 
       - name: Upload digest
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: digests-${{ env.PLATFORM_PAIR }}
           path: ${{ runner.temp }}/digests/*
@@ -75,33 +81,47 @@ jobs:
       - build
     steps:
       - name: Download digests
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
         with:
           path: ${{ runner.temp }}/digests
           pattern: digests-*
           merge-multiple: true
 
       - name: Login to Quay.io
-        uses: docker/login-action@v3
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
         with:
           registry: quay.io
           username: ${{ secrets.QUAY_USERNAME }}
           password: ${{ secrets.QUAY_ROBOT_TOKEN }}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5
         with:
-          images: |
-            ${{ env.QUAY_REPO }}
+          images: ${{ env.QUAY_REPO }}
+          flavor: |
+            latest=false
           tags: |
-            type=ref,event=branch
-            type=ref,event=pr
-            type=ref,event=tag
-            type=sha,prefix=
+            # Semver
+            type=semver,pattern={{version}},value=${{ github.ref_name }}
+
+            # Latest
+            type=raw,value=latest,enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') && !contains(github.ref_name, '-') }}
+
+            # Branch pushes
+            type=ref,event=branch,latest=false
+
+            # PR pushes
+            type=ref,event=pr,latest=false
+
+            # Git Tag
+            type=ref,event=tag,latest=false
+
+            # Commit SHA
+            type=sha,prefix=,latest=false
 
       - name: Create manifest list and push
         working-directory: ${{ runner.temp }}/digests

.github/workflows/hugo.yaml

@@ -1,5 +1,5 @@
 # Sample workflow for building and deploying a Hugo site to GitHub Pages
-name: hugo.yaml
+name: Publish Website
 
 on:
   # Runs on pushes targeting the default branch
@@ -35,13 +35,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Setup Hugo
-        uses: peaceiris/actions-hugo@v3
+        uses: peaceiris/actions-hugo@75d2e84710de30f6ff7268e08f310b60ef14033f # v3
         with:
           hugo-version: '0.152.2'
           extended: true
 
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           submodules: recursive
 
@@ -54,9 +54,10 @@ jobs:
       - name: Build site with Hugo
         run: |
           make website
+          cp install.sh ./docs/install.sh
 
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
+        uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3
         with:
           path: docs
 
@@ -70,4 +71,4 @@ jobs:
     steps:
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v4
+        uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4