E2E AutoTest #23
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E AutoTest | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| test_plan: | |
| description: "Test plan to run (leave empty for all)" | |
| required: false | |
| default: "" | |
| type: string | |
| vsix_urls: | |
| description: "VSIX URLs to install, comma-separated (e.g. https://host/redhat.java-1.42.0.vsix,https://host/vscode-java-debug-0.58.0.vsix)" | |
| required: false | |
| default: "" | |
| type: string | |
| jobs: | |
| # ── Job 1: Discover test plans ────────────────────────── | |
| discover: | |
| if: ${{ inputs.test_plan == '' }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.scan.outputs.matrix }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Scan test plans | |
| id: scan | |
| run: | | |
| plans=$(ls test-plans/*.yaml | xargs -I{} basename {} .yaml | grep -v java-fresh-import | jq -R . | jq -sc .) | |
| echo "matrix=$plans" >> "$GITHUB_OUTPUT" | |
| echo "Found plans: $plans" | |
| # ── Job 2: Run each test plan in parallel ─────────────── | |
| e2e-test: | |
| if: ${{ inputs.test_plan == '' }} | |
| needs: discover | |
| runs-on: windows-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| plan: ${{ fromJson(needs.discover.outputs.matrix) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Clone vscode-java (test projects) | |
| run: git clone --depth 1 https://github.com/redhat-developer/vscode-java.git ../vscode-java | |
| - name: Clone eclipse.jdt.ls (Gradle test projects) | |
| run: git clone --depth 1 https://github.com/eclipse-jdtls/eclipse.jdt.ls.git ../eclipse.jdt.ls | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Setup Java 25 (for java25 test plans) | |
| if: contains(matrix.plan, 'java25') | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: 25-ea | |
| - name: Create JDK 25 path | |
| if: contains(matrix.plan, 'java25') | |
| shell: pwsh | |
| run: | | |
| New-Item -ItemType Junction -Path "C:\Program Files\Java\jdk-25" -Target $env:JAVA_HOME | |
| - name: Setup Java 21 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: 21 | |
| - name: Install autotest CLI | |
| run: npm install -g @vscjava/vscode-autotest | |
| - name: Download VSIX files | |
| if: ${{ inputs.vsix_urls != '' }} | |
| shell: pwsh | |
| run: | | |
| New-Item -ItemType Directory -Path vsix -Force | Out-Null | |
| $urls = "${{ inputs.vsix_urls }}" -split "," | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne "" } | |
| foreach ($url in $urls) { | |
| $fileName = [System.IO.Path]::GetFileName(($url -split '\?')[0]) | |
| Write-Host "Downloading: $url → vsix/$fileName" | |
| Invoke-WebRequest -Uri $url -OutFile "vsix/$fileName" -UseBasicParsing | |
| } | |
| Write-Host "Downloaded VSIX files:" | |
| Get-ChildItem vsix -Filter "*.vsix" | ForEach-Object { Write-Host " $($_.Name) ($([math]::Round($_.Length/1MB, 1)) MB)" } | |
| - name: Run ${{ matrix.plan }} | |
| shell: pwsh | |
| env: | |
| AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }} | |
| run: | | |
| $autotestArgs = @("run", "test-plans/${{ matrix.plan }}.yaml") | |
| if (Test-Path vsix) { | |
| $vsixFiles = (Get-ChildItem vsix -Filter "*.vsix" | ForEach-Object { $_.FullName }) -join "," | |
| if ($vsixFiles) { $autotestArgs += @("--vsix", $vsixFiles) } | |
| } | |
| Write-Host "Running: autotest $($autotestArgs -join ' ')" | |
| & autotest @autotestArgs | |
| - name: Upload results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: results-${{ matrix.plan }} | |
| path: test-results/ | |
| retention-days: 30 | |
| # ── Job 2b: Run a single test plan (when specified) ───── | |
| e2e-single: | |
| if: ${{ inputs.test_plan != '' }} | |
| runs-on: windows-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Clone vscode-java | |
| run: git clone --depth 1 https://github.com/redhat-developer/vscode-java.git ../vscode-java | |
| - name: Clone eclipse.jdt.ls | |
| run: git clone --depth 1 https://github.com/eclipse-jdtls/eclipse.jdt.ls.git ../eclipse.jdt.ls | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Setup Java 25 | |
| if: contains(inputs.test_plan, 'java25') | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: 25-ea | |
| - name: Create JDK 25 path | |
| if: contains(inputs.test_plan, 'java25') | |
| shell: pwsh | |
| run: | | |
| New-Item -ItemType Junction -Path "C:\Program Files\Java\jdk-25" -Target $env:JAVA_HOME | |
| - name: Setup Java 21 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: 21 | |
| - name: Install autotest CLI | |
| run: npm install -g @vscjava/vscode-autotest | |
| - name: Download VSIX files | |
| if: ${{ inputs.vsix_urls != '' }} | |
| shell: pwsh | |
| run: | | |
| New-Item -ItemType Directory -Path vsix -Force | Out-Null | |
| $urls = "${{ inputs.vsix_urls }}" -split "," | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne "" } | |
| foreach ($url in $urls) { | |
| $fileName = [System.IO.Path]::GetFileName(($url -split '\?')[0]) | |
| Write-Host "Downloading: $url → vsix/$fileName" | |
| Invoke-WebRequest -Uri $url -OutFile "vsix/$fileName" -UseBasicParsing | |
| } | |
| - name: Run ${{ inputs.test_plan }} | |
| shell: pwsh | |
| env: | |
| AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }} | |
| run: | | |
| $autotestArgs = @("run", "test-plans/${{ inputs.test_plan }}") | |
| if (Test-Path vsix) { | |
| $vsixFiles = (Get-ChildItem vsix -Filter "*.vsix" | ForEach-Object { $_.FullName }) -join "," | |
| if ($vsixFiles) { $autotestArgs += @("--vsix", $vsixFiles) } | |
| } | |
| Write-Host "Running: autotest $($autotestArgs -join ' ')" | |
| & autotest @autotestArgs | |
| - name: Upload results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-test-results | |
| path: test-results/ | |
| retention-days: 30 | |
| # ── Job 3: Aggregate analysis ─────────────────────────── | |
| analyze: | |
| if: ${{ always() && inputs.test_plan == '' }} | |
| needs: e2e-test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Install autotest CLI | |
| run: npm install -g @vscjava/vscode-autotest | |
| - name: Download all results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: results-* | |
| path: all-results | |
| merge-multiple: false | |
| - name: Organize results | |
| run: | | |
| mkdir -p test-results | |
| for dir in all-results/results-*/; do | |
| find "$dir" -name "results.json" -exec dirname {} \; | while read d; do | |
| name=$(basename "$d") | |
| cp -r "$d" "test-results/$name" | |
| done | |
| done | |
| echo "Organized results:" | |
| ls test-results/ | |
| - name: Analyze results | |
| env: | |
| AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }} | |
| run: autotest analyze test-results --output test-results | |
| - name: Write Job Summary | |
| if: always() | |
| run: | | |
| if [ -f test-results/summary.md ]; then | |
| cat test-results/summary.md >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Upload aggregate results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-aggregate-summary | |
| path: test-results/summary.md | |
| retention-days: 30 |