Better UI tests

.github/workflows/build.yml

@@ -109,7 +109,7 @@ jobs:
 
       # Run tests
       - name: Run Tests
-        run: ./gradlew check
+        run: ./gradlew check -x uiTest
 
       # Collect Tests Result of failed tests
       - name: Collect Tests Result

.github/workflows/run-ui-tests.yml

@@ -1,6 +1,6 @@
-# GitHub Actions Workflow for launching UI tests on Linux, Windows, and Mac in the following steps:
-# - Prepare and launch IDE with your plugin and robot-server plugin, which is needed to interact with the UI.
-# - Wait for IDE to start.
+# GitHub Actions Workflow for launching UI tests on Linux and Mac in the following steps:
+# - Prepare and launch IDE with your plugin and robot-server plugin, which is necessary to interact with the UI.
+# - Wait for IDE to start
 # - Run UI tests with a separate Gradle task.
 #
 # Please check https://github.com/JetBrains/intellij-ui-test-robot for information about UI tests with IntelliJ Platform.
@@ -12,30 +12,29 @@ on:
   workflow_dispatch
 
 jobs:
-
   testUI:
-    runs-on: ${{ matrix.os }}
+    name: Run UI Test on ${{ matrix.osName }}
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      contents: write
+      checks: write
+      pull-requests: write
     strategy:
       fail-fast: false
       matrix:
         include:
-          - os: ubuntu-latest
-            runIde: |
-              export DISPLAY=:99.0
-              Xvfb -ac :99 -screen 0 1920x1080x16 &
-              gradle runIdeForUiTests &
-          - os: windows-latest
-            runIde: start gradlew.bat runIdeForUiTests
-          - os: macos-latest
-            runIde: ./gradlew runIdeForUiTests &
+          - runner: ubuntu-latest
+            osName: Linux
+          - runner: macos-latest
+            osName: macOS
+#          - runner: windows-latest
+#            osName: Windows
 
     steps:
-
-      # Check out the current repository
       - name: Fetch Sources
         uses: actions/checkout@v6
 
-      # Set up Java environment for the next steps
+      # Set up the Java environment for the next steps
       - name: Setup Java
         uses: actions/setup-java@v5
         with:
@@ -48,18 +47,94 @@ jobs:
         with:
           cache-read-only: true
 
-      # Run IDEA prepared for UI testing
-      - name: Run IDE
-        run: ${{ matrix.runIde }}
+      # Set up Xvfb for Linux headless mode
+      - name: Setup Xvfb for Linux
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y xvfb x11-utils libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0
+          export DISPLAY=:99.0
+          Xvfb -ac :99 -screen 0 1920x1080x16 &
+          sleep 3
+          echo "DISPLAY=:99.0" >> $GITHUB_ENV
+
+      # Run UI test with IDE Starter framework
+      - name: Run UI Test
+        env:
+          UI_BRANCH_NAME: "${{ matrix.osName }}-${{ github.ref_name }}"
+          DISPLAY: ${{ runner.os == 'Linux' && ':99.0' || '' }}
+        run: |
+          echo "UI on ${{ matrix.osName }} run #: ${{ github.run_number }} and id: ${{ github.run_id }}"
+          ./gradlew uiTest
+        timeout-minutes: 30
+
+      # List build directory for debugging
+      - name: List build directory structure
+        if: always()
+        shell: bash
+        run: |
+          echo "=== Build directory structure ==="
+          if [ -d "build" ]; then
+            find build -type d -name "log" -o -name "screenshots" -o -name "ui-hierarchy" 2>/dev/null | head -20 || true
+            echo "=== Looking for IDE test output ==="
+            find build -type d -path "*/ide-tests/*" 2>/dev/null | head -20 || true
+          else
+            echo "Build directory not found"
+          fi
 
-      # Wait for IDEA to be started
-      - name: Health Check
-        uses: jtalk/url-health-check-action@v4
+      - name: Collect Test Results
+        if: always()
+        uses: actions/upload-artifact@v6
         with:
-          url: http://127.0.0.1:8082
-          max-attempts: 15
-          retry-delay: 30s
+          name: tests-result-${{ matrix.osName }}
+          path: |
+            build/reports/
+            build/test-results/
+            build/out/
+            build/idea-sandbox/
+            build/**/*.log
+            build/**/screenshots/
+            build/**/ui-hierarchy/
+          if-no-files-found: warn
+        continue-on-error: true
 
-      # Run tests
-      - name: Tests
-        run: ./gradlew test
+      - name: Collect Test Report XML
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: test-report-${{ matrix.osName }}
+          path: ${{ github.workspace }}/build/test-results/uiTest/*.xml
+
+  # Test report job using dorny/test-reporter
+  test-report:
+    name: UI Test Report
+    needs: testUI
+    runs-on: ubuntu-latest
+    if: always()
+    permissions:
+      contents: write
+      checks: write
+      pull-requests: write
+      actions: read
+
+    steps:
+      - name: Fetch Sources
+        uses: actions/checkout@v6
+        with:
+          token: '${{ secrets.GITHUB_TOKEN }}'
+
+      - name: Download all test reports
+        uses: actions/download-artifact@v7
+        with:
+          pattern: test-report-*
+          path: test-reports
+          merge-multiple: true
+
+      - name: Generate Test Report
+        uses: dorny/test-reporter@v2.5.0
+        if: success() || failure()
+        with:
+          name: UI Test Report Dashboard
+          reporter: java-junit
+          fail-on-error: 'true'
+          path: 'test-reports/*.xml'

.gitignore

@@ -4,8 +4,10 @@
 .idea
 .intellijPlatform
 .qodana
+allure-results
 build
 idea-flex.skeleton
 java_pid*.hprof
 jflex-*.jar
+out
 src/main/gen

.run/Run Tests.run.xml

@@ -5,7 +5,7 @@
       <option name="executionName" />
       <option name="externalProjectPath" value="$PROJECT_DIR$" />
       <option name="externalSystemIdString" value="GRADLE" />
-      <option name="scriptParameters" value="" />
+      <option name="scriptParameters" value="-x uiTest" />
       <option name="taskDescriptions">
         <list />
       </option>
@@ -18,8 +18,11 @@
     </ExternalSystemSettings>
     <ExternalSystemDebugServerProcess>true</ExternalSystemDebugServerProcess>
     <ExternalSystemReattachDebugProcess>true</ExternalSystemReattachDebugProcess>
+    <ExternalSystemDebugDisabled>false</ExternalSystemDebugDisabled>
     <DebugAllEnabled>false</DebugAllEnabled>
     <RunAsTest>true</RunAsTest>
+    <GradleProfilingDisabled>false</GradleProfilingDisabled>
+    <GradleCoverageDisabled>false</GradleCoverageDisabled>
     <method v="2" />
   </configuration>
-</component>
+</component>

.run/Run UI Tests.run.xml

@@ -0,0 +1,27 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Run UI Tests" type="GradleRunConfiguration" factoryName="Gradle">
+    <ExternalSystemSettings>
+      <option name="executionName" />
+      <option name="externalProjectPath" value="$PROJECT_DIR$" />
+      <option name="externalSystemIdString" value="GRADLE" />
+      <option name="scriptParameters" value="" />
+      <option name="taskDescriptions">
+        <list />
+      </option>
+      <option name="taskNames">
+        <list>
+          <option value="uiTest" />
+        </list>
+      </option>
+      <option name="vmOptions" />
+    </ExternalSystemSettings>
+    <ExternalSystemDebugServerProcess>true</ExternalSystemDebugServerProcess>
+    <ExternalSystemReattachDebugProcess>true</ExternalSystemReattachDebugProcess>
+    <ExternalSystemDebugDisabled>false</ExternalSystemDebugDisabled>
+    <DebugAllEnabled>false</DebugAllEnabled>
+    <RunAsTest>false</RunAsTest>
+    <GradleProfilingDisabled>false</GradleProfilingDisabled>
+    <GradleCoverageDisabled>false</GradleCoverageDisabled>
+    <method v="2" />
+  </configuration>
+</component>

CHANGELOG.md

@@ -4,9 +4,16 @@
 
 ## [Unreleased]
 
+### Added
+
+- Simple UI test to verify the plugin is installed and a Gradle task to run it
+- Codecov YAML file that was missed from the template
+
 ### Fixed
 
 - Broken links in CHANGELOG.md caused by missing "v" in tag creation build by adding `versionPrefix` default
+- Run UI tests GitHub Action workflow to use the added UI test
+- Exclude the UI test from build GitHub Action workflow
 
 ### Changed
 

build.gradle.kts

@@ -1,3 +1,4 @@
+import org.gradle.kotlin.dsl.register
 import org.jetbrains.changelog.Changelog
 import org.jetbrains.changelog.markdownToHTML
 import org.jetbrains.intellij.platform.gradle.TestFrameworkType
@@ -10,6 +11,7 @@ plugins {
     alias(libs.plugins.qodana) // Gradle Qodana Plugin
     alias(libs.plugins.kover) // Gradle Kover Plugin
     alias(libs.plugins.grammarkit) // Gradle Grammar-Kit Plugin
+    idea // IntelliJ IDEA support
 }
 
 group = providers.gradleProperty("pluginGroup").get()
@@ -22,16 +24,41 @@ sourceSets {
             srcDirs("src/main/gen")
         }
     }
+    create("uiTest",Action<SourceSet>{
+        compileClasspath += sourceSets.main.get().output + sourceSets.test.get().output
+        runtimeClasspath += sourceSets.main.get().output + sourceSets.test.get().output
+    })
+}
+
+idea {
+    module {
+        generatedSourceDirs.add(file("src/main/gen"))
+        testSources.from(sourceSets.getByName("uiTest").kotlin.srcDirs)
+        testResources.from(sourceSets.getByName("uiTest").resources.srcDirs)
+    }
+}
+
+val uiTestImplementation: Configuration by configurations.getting {
+    extendsFrom(configurations.testImplementation.get())
+}
+
+val uiTestRuntimeOnly: Configuration by configurations.getting {
+    extendsFrom(configurations.testRuntimeOnly.get())
 }
 
 // Set the JVM language level used to build the project.
 kotlin {
-    jvmToolchain(21)
+    jvmToolchain {
+        languageVersion = JavaLanguageVersion.of(21)
+        @Suppress("UnstableApiUsage")
+        vendor = JvmVendorSpec.JETBRAINS
+    }
 }
 
 // Configure project's dependencies
 repositories {
     mavenCentral()
+    maven("https://cache-redirector.jetbrains.com/packages.jetbrains.team/maven/p/ij/intellij-ide-starter")
 
     // IntelliJ Platform Gradle Plugin Repositories Extension - read more: https://plugins.jetbrains.com/docs/intellij/tools-intellij-platform-gradle-plugin-repositories-extension.html
     intellijPlatform {
@@ -59,7 +86,16 @@ dependencies {
 
         testFramework(TestFrameworkType.Platform)
         testBundledPlugin("org.jetbrains.kotlin")
+
+        // IntelliJ UI Tests framework - read more: https://github.com/JetBrains/intellij-ide-starter
+        testFramework(TestFrameworkType.Starter, configurationName = "uiTestImplementation")
+        testFramework(TestFrameworkType.JUnit5, configurationName = "uiTestImplementation")
     }
+
+    uiTestImplementation("org.junit.jupiter:junit-jupiter:5.7.1")
+    uiTestImplementation("org.kodein.di:kodein-di-jvm:7.20.2")
+    uiTestImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:1.10.1")
+    uiTestRuntimeOnly("org.junit.platform:junit-platform-launcher")
 }
 
 // Configure IntelliJ Platform Gradle Plugin - read more: https://plugins.jetbrains.com/docs/intellij/tools-intellij-platform-gradle-plugin-extension.html
@@ -169,28 +205,40 @@ tasks {
         gradleVersion = providers.gradleProperty("gradleVersion").get()
     }
 
+    runIde {
+        systemProperties(
+            "ide.native.launcher" to true,
+            "ide.show.tips.on.startup.default.value" to false,
+            "jb.consents.confirmation.enabled" to false
+        )
+    }
+
     publishPlugin {
         dependsOn(patchChangelog)
     }
-}
 
-intellijPlatformTesting {
-//    runIde {
-//        register("runIdeForUiTests") {
-//            task {
-//                jvmArgumentProviders += CommandLineArgumentProvider {
-//                    listOf(
-//                        "-Drobot-server.port=8082",
-//                        "-Dide.mac.message.dialogs.as.sheets=false",
-//                        "-Djb.privacy.policy.text=<!--999.999-->",
-//                        "-Djb.consents.confirmation.enabled=false",
-//                    )
-//                }
-//            }
-//
-//            plugins {
-//                robotServerPlugin()
-//            }
-//        }
-//    }
+    register<Test>("uiTest") {
+        description = "Runs only the UI tests that start the IDE"
+        group = "verification"
+        val uiTestSourceSet = sourceSets["uiTest"]
+        testClassesDirs = uiTestSourceSet.output.classesDirs
+        classpath = uiTestSourceSet.runtimeClasspath
+        systemProperty("path.to.build.plugin", buildPlugin.get().archiveFile.get().asFile.absolutePath)
+        systemProperty("idea.home.path", prepareTestSandbox.get().getDestinationDir().parentFile.absolutePath)
+        systemProperty("platform.version", providers.gradleProperty("platformVersion").get())
+        systemProperty(
+            "allure.results.directory", project.layout.buildDirectory.get().asFile.absolutePath + "/allure-results"
+        )
+        useJUnitPlatform()
+
+        // Add required JVM arguments
+        jvmArgumentProviders += CommandLineArgumentProvider {
+            mutableListOf(
+                "--add-opens=java.base/java.lang=ALL-UNNAMED",
+                "--add-opens=java.desktop/javax.swing=ALL-UNNAMED"
+            )
+        }
+
+        dependsOn(buildPlugin)
+    }
 }

codecov.yml

@@ -0,0 +1,15 @@
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+        threshold: 0%
+        base: auto
+    patch:
+      default:
+        informational: true
+
+
+ignore:
+  - src/test/
+  - src/uiTest/