[WIP] cleanup server to be human readable and semantically cleaner. need to…

.codexignore

@@ -0,0 +1,7 @@
+.gocache/
+bin/
+data/
+META/
+arango-fhir-proto
+arango-fhir-server
+.DS_Store

.dockerignore

@@ -7,5 +7,13 @@ bin
 arango-fhir-proto
 arango-fhir-server
 META
+META_SMALL
 data
 experimental
+docs
+examples
+scripts
+Makefile
+README.md
+TODO.md
+**/*_test.go

.github/workflows/build.yaml

@@ -1,34 +1,151 @@
-name: Build and publish arango-fhir-server image
+name: Loom CI
 
 on:
+  pull_request:
+    types: [opened, synchronize, reopened]
   push:
+    branches:
+      - main
+      - development
+    tags:
+      - 'v*'
   workflow_dispatch:
 
+permissions:
+  contents: read
+
+concurrency:
+  group: loom-image-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  IMAGE: quay.io/ohsu-comp-bio/loom
+
 jobs:
-  build:
+  tests:
+    name: Tests
+    uses: ./.github/workflows/tests.yaml
+
+  build-and-push:
+    name: Build and push ${{ matrix.arch }} image
+    needs: tests
+    # Never expose Quay credentials to code from a forked pull request.
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-latest' || 'ubuntu-24.04-arm' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [amd64, arm64]
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=sha,format=short
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=raw,value=latest,enable=${{ github.ref_type == 'tag' && !contains(github.ref_name, '-rc.') }}
+
+      - name: Log in to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_ROBOT_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: linux/${{ matrix.arch }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true
+          cache-from: type=gha,scope=loom-${{ matrix.arch }}
+          cache-to: type=gha,mode=max,scope=loom-${{ matrix.arch }}
+
+      - name: Export image digest
+        run: |
+          set -euo pipefail
+          mkdir -p /tmp/loom-digests
+          printf '%s\n' "${{ steps.build.outputs.digest }}" > "/tmp/loom-digests/${{ matrix.arch }}.txt"
+
+      - name: Upload image digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: loom-digest-${{ matrix.arch }}
+          path: /tmp/loom-digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    name: Push multi-architecture manifest
     runs-on: ubuntu-latest
+    needs: build-and-push
     steps:
-    - name: Check out code
-      uses: actions/checkout@v4
-
-    - name: Login to Quay.io
-      uses: docker/login-action@v3
-      with:
-        registry: quay.io
-        username: ${{ secrets.QUAY_USERNAME }}
-        password: ${{ secrets.QUAY_ROBOT_TOKEN }}
-
-    - name: Build and push image
-      run: |
-        BRANCH=$(echo ${GITHUB_REF#refs/*/} | tr / _)
-        REPO=quay.io/ohsu-comp-bio/arango-fhir-proto
-        echo "Setting image tag to $REPO:$BRANCH"
-
-        docker login quay.io
-        docker build -t $REPO:$BRANCH .
-
-        if [[ $BRANCH == 'main' ]]; then
-          docker image tag $REPO:main $REPO:latest
-        fi
-
-        docker push --all-tags $REPO
+      - name: Download image digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/loom-digests
+          pattern: loom-digest-*
+          merge-multiple: true
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=sha,format=short
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=raw,value=latest,enable=${{ github.ref_type == 'tag' && !contains(github.ref_name, '-rc.') }}
+
+      - name: Log in to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_ROBOT_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Create and push manifest list
+        env:
+          DOCKER_METADATA_OUTPUT_TAGS: ${{ steps.meta.outputs.tags }}
+        run: |
+          set -euo pipefail
+          tag_args=()
+          while IFS= read -r tag; do
+            if [ -n "$tag" ]; then
+              tag_args+=("-t" "$tag")
+            fi
+          done <<< "$DOCKER_METADATA_OUTPUT_TAGS"
+
+          sources=()
+          for digest_file in /tmp/loom-digests/*.txt; do
+            digest=$(tr -d '\r\n' < "$digest_file")
+            if [ -n "$digest" ]; then
+              sources+=("${IMAGE}@${digest}")
+            fi
+          done
+
+          if [ "${#sources[@]}" -eq 0 ]; then
+            echo "No image digests were exported" >&2
+            exit 1
+          fi
+          docker buildx imagetools create "${tag_args[@]}" "${sources[@]}"

.github/workflows/tests.yaml

@@ -1,9 +1,11 @@
 name: Go Tests
 
-on: [ pull_request ]
+on:
+  workflow_call:
 
 jobs:
   test:
+    name: Go tests
     runs-on: ubuntu-latest
 
     steps:
@@ -13,7 +15,8 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v5
       with:
-        go-version: '1.26.2'
+        go-version-file: go.mod
+        cache: true
 
     - name: Generate FHIR code
       run: make generate-fhir
@@ -23,3 +26,11 @@ jobs:
 
     - name: Run Go tests
       run: make test
+
+    - name: Build server image
+      uses: docker/build-push-action@v6
+      with:
+        context: .
+        file: ./Dockerfile
+        push: false
+        tags: loom:ci

.gitignore

@@ -1,2 +1,7 @@
 .DS_Store
-
+.gocache/
+bin/
+data/
+META/
+/arango-fhir-proto
+/arango-fhir-server

Dockerfile

@@ -12,16 +12,18 @@ RUN --mount=type=cache,target=/go/pkg/mod \
     go mod download
 
 COPY cmd ./cmd
+COPY fhirschema ./fhirschema
+COPY fhirstructs ./fhirstructs
+COPY graphqlapi ./graphqlapi
 COPY internal ./internal
-COPY queries ./queries
 COPY schemas ./schemas
 
 ARG TARGETOS=linux
 ARG TARGETARCH=amd64
 RUN --mount=type=cache,target=/go/pkg/mod \
     --mount=type=cache,target=/root/.cache/go-build \
     GOOS="$TARGETOS" GOARCH="$TARGETARCH" \
-    go build \
+    go build -mod=mod \
       -trimpath \
       -ldflags="-s -w" \
       -o /out/arango-fhir-server ./cmd/arango-fhir-server
@@ -35,8 +37,10 @@ WORKDIR /app
 
 COPY --from=builder /out/arango-fhir-server /app/arango-fhir-server
 COPY --from=builder /src/schemas /app/schemas
-COPY --from=builder /src/queries /app/queries
 
 USER arango-fhir
 EXPOSE 8080
+STOPSIGNAL SIGTERM
+HEALTHCHECK --interval=10s --timeout=3s --start-period=10s --retries=6 \
+  CMD wget -q -O - http://127.0.0.1:8080/healthz >/dev/null || exit 1
 ENTRYPOINT ["/app/arango-fhir-server"]

Makefile

@@ -0,0 +1,95 @@
+.PHONY: build build-cli build-server clean compiler-bench compiler-arango-bench dataframe-demo dataframe-profile dataframe-boundaries dataframe-test conformance generate-fhir generate-graphql graphql-check gqlgen-check test docker-build docker-run
+
+GO ?= go
+GOCACHE_DIR ?= $(CURDIR)/.gocache
+GOFLAGS ?=
+SCHEMA_PATH ?= schemas/graph-fhir.json
+IMAGE ?= arango-fhir-proto:local
+BENCH_TIME ?= 10x
+BENCH_COUNT ?= 5
+GRAPHQL_URL ?= http://127.0.0.1:8080/graphql
+DATAFRAME_REPEAT ?= 1
+DATAFRAME_LIMIT ?= 0
+DATAFRAME_TIMEOUT ?= 5m
+DATAFRAME_PRINT_RESPONSE ?= false
+DATAFRAME_QUERY ?= examples/meta_gdc_case_matrix.graphql
+DATAFRAME_VARIABLES ?= examples/meta_gdc_case_matrix.variables.json
+DATAFRAME_PROFILE_VARIABLES ?= examples/meta_gdc_case_matrix.variables.json
+DATAFRAME_PROFILE_LIMIT ?= 1000
+
+build: build-cli build-server
+
+build-cli:
+	mkdir -p bin $(GOCACHE_DIR)
+	GOCACHE=$(GOCACHE_DIR) GOTOOLCHAIN=auto $(GO) build $(GOFLAGS) -o bin/arango-fhir-proto ./cmd/arango-fhir-proto
+
+build-server:
+	mkdir -p bin $(GOCACHE_DIR)
+	GOCACHE=$(GOCACHE_DIR) GOTOOLCHAIN=auto $(GO) build $(GOFLAGS) -o bin/arango-fhir-server ./cmd/arango-fhir-server
+
+generate-graphql:
+	mkdir -p $(GOCACHE_DIR)
+	@status=0; fix_status=0; \
+	GOFLAGS="$(GOFLAGS) -mod=mod" GOCACHE=$(GOCACHE_DIR) GOTOOLCHAIN=auto $(GO) run -mod=mod github.com/99designs/gqlgen generate || status=$$?; \
+	if [ -f graphqlapi/generated.go ]; then \
+		GOCACHE=$(GOCACHE_DIR) GOTOOLCHAIN=auto $(GO) run ./cmd/gqlgenfix graphqlapi/generated.go || fix_status=$$?; \
+	fi; \
+	test $$fix_status -eq 0; \
+	test $$status -eq 0 -o -f graphqlapi/generated.go
+
+generate-fhir:
+	mkdir -p $(GOCACHE_DIR)
+	GOCACHE=$(GOCACHE_DIR) GOTOOLCHAIN=auto $(GO) run ./cmd/generate -schema $(SCHEMA_PATH) -structs-out fhirstructs -metadata-out fhirschema/generated.go
+	gofmt -w fhirstructs/model.go fhirstructs/validate.go fhirstructs/extract.go fhirstructs/helpers.go fhirschema/generated.go
+
+graphql-check:
+	mkdir -p $(GOCACHE_DIR)
+	GOCACHE=$(GOCACHE_DIR) GOTOOLCHAIN=auto $(GO) test $(GOFLAGS) ./graphqlapi ./internal/dataframe -count=1
+
+gqlgen-check: graphql-check
+
+test:
+	mkdir -p $(GOCACHE_DIR)
+	GOCACHE=$(GOCACHE_DIR) GOTOOLCHAIN=auto $(GO) test $(GOFLAGS) ./... -count=1
+
+compiler-bench:
+	mkdir -p $(GOCACHE_DIR)
+	GOCACHE=$(GOCACHE_DIR) GOTOOLCHAIN=auto $(GO) test $(GOFLAGS) ./conformance/compiler -run '^$$' -bench '^BenchmarkCompilerOracle$$' -benchmem
+
+# Requires a locally loaded META fixture. Override BENCH_TIME/BENCH_COUNT for
+# a longer run, for example: make compiler-arango-bench BENCH_TIME=3s BENCH_COUNT=10.
+compiler-arango-bench:
+	mkdir -p $(GOCACHE_DIR)
+	LOOM_COMPILER_ARANGO_INTEGRATION=1 GOCACHE=$(GOCACHE_DIR) GOTOOLCHAIN=auto $(GO) test $(GOFLAGS) ./internal/dataframe -run '^$$' -bench '^BenchmarkGenericCompilerAgainstArango$$' -benchmem -benchtime=$(BENCH_TIME) -count=$(BENCH_COUNT)
+
+# Requires `arango-fhir-server --no-auth` and a loaded META project. Prints the
+# actual GraphQL dataframe response and per-request wall-clock timings.
+dataframe-demo:
+	mkdir -p $(GOCACHE_DIR)
+	GOCACHE=$(GOCACHE_DIR) GOTOOLCHAIN=auto $(GO) run ./cmd/dataframe-query -url $(GRAPHQL_URL) -query $(DATAFRAME_QUERY) -variables $(DATAFRAME_VARIABLES) -repeat $(DATAFRAME_REPEAT) -limit $(DATAFRAME_LIMIT) -timeout $(DATAFRAME_TIMEOUT) -print-response=$(DATAFRAME_PRINT_RESPONSE)
+
+# Requires a loaded META fixture database. Compiles the checked-in GDC fixture,
+# writes exact rendered AQL, then runs Arango EXPLAIN and PROFILE 2.
+dataframe-profile:
+	mkdir -p $(GOCACHE_DIR)
+	GOCACHE=$(GOCACHE_DIR) GOTOOLCHAIN=auto $(GO) run ./cmd/dataframe-profile -variables $(DATAFRAME_PROFILE_VARIABLES) -limit $(DATAFRAME_PROFILE_LIMIT)
+
+dataframe-boundaries:
+	./scripts/check_dataframe_package_boundaries.sh
+
+dataframe-test: dataframe-boundaries
+	mkdir -p $(GOCACHE_DIR)
+	GOCACHE=$(GOCACHE_DIR) GOTOOLCHAIN=auto $(GO) test $(GOFLAGS) ./internal/dataframe/spec ./internal/dataframe/semantic ./internal/dataframe/compiler/ir ./internal/dataframe/compiler/lower ./internal/dataframe/compiler/optimize ./internal/dataframe/compiler/render/aql ./internal/dataframe/compiler ./internal/dataframe/runtime ./internal/dataframe -count=1
+
+conformance:
+	mkdir -p $(GOCACHE_DIR)
+	GOCACHE=$(GOCACHE_DIR) GOTOOLCHAIN=auto $(GO) test $(GOFLAGS) ./conformance/... -count=1
+
+docker-build:
+	docker build -t $(IMAGE) .
+
+docker-run:
+	docker run --rm -p 8080:8080 $(IMAGE)
+
+clean:
+	rm -rf bin