Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 3 additions & 33 deletions .github/actions/java-test/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ runs:
# temporarily work around https://github.com/actions/runner-images/issues/13341
# by disabling caching for macOS
if: ${{ runner.os != 'macOS' }}
uses: actions/cache@v4
uses: actions/cache@v5
with:
path: |
~/.m2/repository
Expand All @@ -68,36 +68,6 @@ runs:
restore-keys: |
${{ runner.os }}-java-maven-

- name: Run Maven compile
shell: bash
run: |
./mvnw -B package -DskipTests scalafix:scalafix -Dscalafix.mode=CHECK -Psemanticdb ${{ inputs.maven_opts }}

- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: '24'

- name: Install prettier
shell: bash
run: |
npm install -g prettier

- name: Run prettier
shell: bash
run: |
npx prettier "**/*.md" --write

- name: Mark workspace as safe for git
shell: bash
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"

- name: Check for any local git changes (such as generated docs)
shell: bash
run: |
./dev/ci/check-working-tree-clean.sh

- name: Run all tests
shell: bash
if: ${{ inputs.suites == '' }}
Expand All @@ -106,7 +76,7 @@ runs:
SPARK_LOCAL_HOSTNAME: "localhost"
SPARK_LOCAL_IP: "127.0.0.1"
run: |
MAVEN_OPTS="-Xmx4G -Xms2G -XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B -Prelease clean install ${{ inputs.maven_opts }}
MAVEN_OPTS="-Xmx4G -Xms2G -XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B -Prelease install ${{ inputs.maven_opts }}
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removing clean doesn't seem strictly related to the goal of this PR, but I also don't think that it should cause issues.

- name: Run specified tests
shell: bash
if: ${{ inputs.suites != '' }}
Expand All @@ -117,7 +87,7 @@ runs:
run: |
MAVEN_SUITES="$(echo "${{ inputs.suites }}" | paste -sd, -)"
echo "Running with MAVEN_SUITES=$MAVEN_SUITES"
MAVEN_OPTS="-Xmx4G -Xms2G -DwildcardSuites=$MAVEN_SUITES -XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B -Prelease clean install ${{ inputs.maven_opts }}
MAVEN_OPTS="-Xmx4G -Xms2G -DwildcardSuites=$MAVEN_SUITES -XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B -Prelease install ${{ inputs.maven_opts }}
- name: Upload crash logs
if: failure()
uses: actions/upload-artifact@v6
Expand Down
66 changes: 66 additions & 0 deletions .github/workflows/pr_build_linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,74 @@
rustup component add rustfmt
cd native && cargo fmt --all -- --check

lint-java:
needs: lint
name: Lint Java (${{ matrix.profile.name }})
runs-on: ubuntu-latest
container:
image: amd64/rust
env:
JAVA_TOOL_OPTIONS: ${{ matrix.profile.java_version == '17' && '--add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED' || '' }}
strategy:
matrix:
profile:
- name: "Spark 3.4, JDK 11, Scala 2.12"
java_version: "11"
maven_opts: "-Pspark-3.4 -Pscala-2.12"
- name: "Spark 3.5, JDK 17, Scala 2.12"
java_version: "17"
maven_opts: "-Pspark-3.5 -Pscala-2.12"
- name: "Spark 4.0, JDK 17"
java_version: "17"
maven_opts: "-Pspark-4.0"
fail-fast: false
steps:
- uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3
- uses: actions/checkout@v6

- name: Setup Rust & Java toolchain
uses: ./.github/actions/setup-builder
with:
rust-version: ${{ env.RUST_VERSION }}
jdk-version: ${{ matrix.profile.java_version }}

- name: Cache Maven dependencies
uses: actions/cache@v5
with:
path: |
~/.m2/repository
/root/.m2/repository
key: ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }}-lint
restore-keys: |
${{ runner.os }}-java-maven-

- name: Run scalafix check
run: |
./mvnw -B package -DskipTests scalafix:scalafix -Dscalafix.mode=CHECK -Psemanticdb ${{ matrix.profile.maven_opts }}

- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: '24'

- name: Install prettier
run: |
npm install -g prettier

- name: Run prettier
run: |
npx prettier "**/*.md" --write

- name: Mark workspace as safe for git
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"

- name: Check for any local git changes (such as generated docs)
run: |
./dev/ci/check-working-tree-clean.sh

# Build native library once and share with all test jobs
build-native:

Check warning

Code scanning / CodeQL

Workflow does not contain permissions Medium

Actions job or workflow does not limit the permissions of the GITHUB_TOKEN. Consider setting an explicit permissions block, using the following as a minimal starting point: {contents: read}
needs: lint
name: Build Native Library
runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion-comet', github.run_id) || 'ubuntu-latest' }}
Expand Down
Loading