Modified and simplified CI

.github/workflows/arroyo-webui.yml

@@ -0,0 +1,56 @@
+name: Arroyo WebUI CI
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - 'webui/**'
+      - 'crates/arroyo-openapi/**'
+      - '.github/workflows/arroyo-webui.yml'
+  pull_request:
+    branches: [ main ]
+    paths:
+      - 'webui/**'
+      - 'crates/arroyo-openapi/**'
+      - '.github/workflows/arroyo-webui.yml'
+  workflow_dispatch:
+
+
+jobs:
+  lint-console:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out
+        uses: actions/checkout@v4
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 9.0.6
+          run_install: |
+            - recursive: true
+              args: [--frozen-lockfile, --strict-peer-dependencies]
+
+      - name: Run prettier
+        run: |
+          cd webui
+          pnpm check
+
+  build-console:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out
+        uses: actions/checkout@v4
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 9.0.6
+          run_install: |
+            - recursive: true
+              args: [--frozen-lockfile, --strict-peer-dependencies]
+
+      - name: Build console
+        run: |
+          cd webui
+          pnpm build

.github/workflows/ci.yml

@@ -1,41 +1,27 @@
 name: CI
 
-on: [push, pull_request]
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
 
 jobs:
-  lint-console:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out
-        uses: actions/checkout@v3
-      - name: Setup pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          version: 9.0.6
-          run_install: |
-            - recursive: true
-              args: [--frozen-lockfile, --strict-peer-dependencies]
-      - name: Run prettier
-        run: |
-          cd webui
-          pnpm check
-
   build-rust:
     runs-on: ubuntu-latest
     env:
       DATABASE_URL: "postgres://arroyo:arroyo@localhost:5432/arroyo"
     steps:
       - name: Check out
-        uses: actions/checkout@v3
-      - uses: actions/setup-java@v3
-        with:
-          distribution: 'temurin'
-          java-version: '11'
+        uses: actions/checkout@v4
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@stable
         with:
           components: clippy, rustfmt
           toolchain: 1.88
+      - name: Run sccache
+        uses: mozilla-actions/sccache-action@v0.0.4
       - uses: actions/cache@v4
         with:
           path: |
@@ -50,87 +36,40 @@ jobs:
       - uses: actions/setup-python@v5
         name: Setup Python
         with:
-          python-version: '3.12'            
+          python-version: '3.12'
       - name: Setup pnpm
         uses: pnpm/action-setup@v4
         with:
           version: 9.0.6
           run_install: |
             - recursive: true
-              args: [--frozen-lockfile, --strict-peer-dependencies]        
-      - name: Install OpenAPI Generator
+              args: [--frozen-lockfile, --strict-peer-dependencies]
+      - name: Install dependencies
         run: |
-          pnpm install @openapitools/openapi-generator-cli -g
-          cd crates/arroyo-openapi && openapi-generator-cli version
+          curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
+          sudo apt-get update
+          sudo apt-get install -y cmake clang unzip libsasl2-dev protobuf-compiler postgresql
       - name: Setup Postgres
         run: |
-          sudo apt-get update
-          sudo apt-get install postgresql
           sudo systemctl start postgresql
           sudo -u postgres psql -c "CREATE USER arroyo WITH PASSWORD 'arroyo' SUPERUSER;"
           sudo -u postgres createdb arroyo
-          pushd /tmp
-            wget https://github.com/rust-db/refinery/releases/download/0.8.7/refinery-0.8.7-x86_64-unknown-linux-musl.tar.gz
-            tar xvfz refinery*.tar.gz
-            mv /tmp/refinery*-musl/refinery /tmp
-          popd
-          /tmp/refinery migrate -e DATABASE_URL -p crates/arroyo-api/migrations
-      - name: Install dependencies
-        run: |
-          curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
-          sudo apt-get install -y cmake clang ruby unzip postgresql libsasl2-dev netcat
-          wget https://github.com/protocolbuffers/protobuf/releases/download/v21.8/protoc-21.8-linux-x86_64.zip
-          unzip protoc*.zip
-          sudo mv bin/protoc /usr/local/bin
-          curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
-      - name: Install Kafka
-        run: |
-          wget --progress=dot --show-progress https://archive.apache.org/dist/kafka/3.5.0/kafka_2.12-3.5.0.tgz
-          tar xvfz kafka*.tgz
-          mkdir /tmp/kraft-combined-logs
-          kafka_*/bin/kafka-storage.sh format -t 9v5PspiySuWU2l5NjTgRuA -c kafka_*/config/kraft/server.properties
-          kafka_*/bin/kafka-server-start.sh -daemon kafka_*/config/kraft/server.properties
-      - name: Install mosquitto
-        run: |
-          sudo apt-get install -y mosquitto
-          sudo service mosquitto start
-      - name: Check Formatting
-        run: cargo fmt -- --check
+          for f in $(ls crates/arroyo-api/migrations/V*.sql | sort -V); do
+            psql "$DATABASE_URL" -f "$f"
+          done
       - name: Build console
         run: |
           cd webui
           pnpm build
       - name: Build
         run: cargo build --all-features
+        env:
+          RUSTC_WRAPPER: sccache
       - name: Run Clippy
         run: cargo clippy --all-features --all-targets --workspace -- -D warnings
+        env:
+          RUSTC_WRAPPER: sccache
       - name: Test
-        run: cargo nextest run -E 'kind(lib)' --all-features
-      - name: Integ postgres
-        run: |
-          mkdir /tmp/arroyo-integ
-          ARROYO__DISABLE_TELEMETRY=true ARROYO__CHECKPOINT_URL=file:///tmp/arroyo-integ ARROYO__COMPILER__ARTIFACT_URL=file:///tmp/artifacts target/debug/arroyo cluster &
-          cargo nextest run --package integ -E 'kind(test)'
-      - name: Integ sqlite
-        run: |
-          killall arroyo
-          ARROYO__DISABLE_TELEMETRY=true ARROYO__CHECKPOINT_URL=file:///tmp/arroyo-integ ARROYO__COMPILER__ARTIFACT_URL=file:///tmp/artifacts ARROYO__DATABASE__TYPE=sqlite target/debug/arroyo cluster &
-          timeout=10; while ! nc -z localhost 5115 && [ $timeout -gt 0 ]; do sleep 1; timeout=$((timeout - 1)); done; [ $timeout -gt 0 ]
-          cargo nextest run --package integ -E 'kind(test)'
-
-  build-console:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out
-        uses: actions/checkout@v3
-      - name: Setup pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          version: 9.0.6
-          run_install: |
-            - recursive: true
-              args: [--frozen-lockfile, --strict-peer-dependencies]
-      - name: Build console
-        run: |
-          cd webui
-          pnpm build
+        run: cargo nextest run -E 'kind(lib) & !test(/kafka/) & !test(/mqtt/)' --all-features
+        env:
+          RUSTC_WRAPPER: sccache