diff --git a/.github/workflows/matlab-tests.yml b/.github/workflows/matlab-tests.yml index 5cfee63..20407b3 100644 --- a/.github/workflows/matlab-tests.yml +++ b/.github/workflows/matlab-tests.yml @@ -55,13 +55,13 @@ jobs: release: ${{ env.MATLAB_RELEASE }} - name: Prepare artifact directories - run: mkdir -p artifacts/logs + run: mkdir -p artifacts/logs/checkStyle - name: Run quality guardrails uses: matlab-actions/run-build@v3 with: tasks: checkStyle - startup-options: -logfile artifacts/logs/matlab.log + startup-options: -logfile artifacts/logs/checkStyle/matlab.log - name: Upload quality artifacts if: always() @@ -75,7 +75,7 @@ jobs: artifacts/logs/** unit: - name: Unit And Coverage + name: Unit Tests runs-on: ubuntu-latest env: MLM_LICENSE_TOKEN: ${{ secrets.MLM_LICENSE_TOKEN }} @@ -90,19 +90,55 @@ jobs: release: ${{ env.MATLAB_RELEASE }} - name: Prepare artifact directories - run: mkdir -p artifacts/logs + run: mkdir -p artifacts/logs/testUnit - - name: Run unit tests and coverage + - name: Run unit tests uses: matlab-actions/run-build@v3 with: - tasks: testUnit coverage - startup-options: -logfile artifacts/logs/matlab.log + tasks: testUnit + startup-options: -logfile artifacts/logs/testUnit/matlab.log - - name: Upload unit and coverage artifacts + - name: Upload unit artifacts if: always() uses: actions/upload-artifact@v4 with: - name: matlab-unit-coverage + name: matlab-unit + if-no-files-found: warn + retention-days: 14 + path: | + artifacts/test-results/** + artifacts/logs/** + + coverage: + name: Coverage Report + if: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' + runs-on: ubuntu-latest + env: + MLM_LICENSE_TOKEN: ${{ secrets.MLM_LICENSE_TOKEN }} + + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Set up MATLAB + uses: matlab-actions/setup-matlab@v3 + with: + release: ${{ env.MATLAB_RELEASE }} + + - name: Prepare artifact directories + run: mkdir -p artifacts/logs/coverage + + - name: Run coverage report + uses: matlab-actions/run-build@v3 + with: + tasks: coverage + startup-options: -logfile artifacts/logs/coverage/matlab.log + + - name: Upload coverage artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: matlab-coverage if-no-files-found: warn retention-days: 14 path: | @@ -126,13 +162,13 @@ jobs: release: ${{ env.MATLAB_RELEASE }} - name: Prepare artifact directories - run: mkdir -p artifacts/logs + run: mkdir -p artifacts/logs/testIntegration - name: Run integration tests uses: matlab-actions/run-build@v3 with: tasks: testIntegration - startup-options: -logfile artifacts/logs/matlab.log + startup-options: -logfile artifacts/logs/testIntegration/matlab.log - name: Upload integration artifacts if: always() @@ -162,13 +198,13 @@ jobs: release: ${{ env.MATLAB_RELEASE }} - name: Prepare artifact directories - run: mkdir -p artifacts/logs + run: mkdir -p artifacts/logs/testGuiStructural - name: Run GUI structural tests uses: matlab-actions/run-build@v3 with: tasks: testGuiStructural - startup-options: -logfile artifacts/logs/matlab.log + startup-options: -logfile artifacts/logs/testGuiStructural/matlab.log - name: Upload GUI structural artifacts if: always() @@ -200,13 +236,13 @@ jobs: release: ${{ env.MATLAB_RELEASE }} - name: Prepare artifact directories - run: mkdir -p artifacts/logs + run: mkdir -p artifacts/logs/testGuiGesture - name: Run GUI gesture tests uses: matlab-actions/run-build@v3 with: tasks: testGuiGesture - startup-options: -logfile artifacts/logs/matlab.log + startup-options: -logfile artifacts/logs/testGuiGesture/matlab.log - name: Upload GUI gesture artifacts if: always() diff --git a/README.md b/README.md index 41b08cf..74b727c 100644 --- a/README.md +++ b/README.md @@ -105,10 +105,10 @@ buildtool testLabkitUiGui testAppsGui ``` Script arguments are build task names; selector flags such as `--suite`, -`--test`, and `--gui` are not supported. GitHub Actions runs quality, -unit/coverage, and integration jobs on pushes and pull requests to `main`; -manual and scheduled runs also cover GUI structural and non-blocking gesture -jobs. +`--test`, and `--gui` are not supported. GitHub Actions runs shell-wrapper, +quality, unit, and integration jobs on pushes and pull requests to `main`; +manual and scheduled runs add coverage, GUI structural, and non-blocking +gesture jobs. ## Repository Layout diff --git a/buildfile.m b/buildfile.m index 5a91c69..adbc232 100644 --- a/buildfile.m +++ b/buildfile.m @@ -4,143 +4,90 @@ plan = buildplan(localfunctions); plan.DefaultTasks = "test"; - plan("checkStyle").Description = "Run project/style guardrails."; - plan("test").Description = "Run the full non-GUI test entry point."; - plan("testUnit").Description = "Run official unit tests."; - plan("testIntegration").Description = "Run official integration tests."; - plan("testProject").Description = "Run project guardrails."; - plan("testLabkitDta").Description = "Run DTA facade/parser tests."; - plan("testLabkitBiosignal").Description = "Run biosignal facade tests."; - plan("testLabkitUi").Description = "Run reusable UI non-GUI tests."; - plan("testLabkitUiGui").Description = "Run reusable UI GUI tests."; - plan("testAppsElectrochem").Description = "Run electrochem app non-GUI tests."; - plan("testAppsElectrochemGui").Description = "Run electrochem app GUI tests."; - plan("testAppsDicGui").Description = "Run DIC app GUI tests."; - plan("testAppsImageMeasurement").Description = "Run image-measurement app non-GUI tests."; - plan("testAppsImageMeasurementGui").Description = "Run image-measurement app GUI tests."; - plan("testAppsWearableGui").Description = "Run wearable app GUI tests."; - plan("testAppsGui").Description = "Run all app GUI tests."; - plan("testAppsSmokeGui").Description = "Run cross-app GUI smoke tests."; - plan("testGuiStructural").Description = "Run noninteractive GUI structural tests."; - plan("testGuiGesture").Description = "Run noninteractive/manual GUI gesture tests."; - plan("coverage").Description = "Run official tests with coverage artifacts."; - plan("checkProject").Description = "Verify MATLAB Project metadata and path setup."; - plan("packageDryRun").Description = "Verify package boundary inventory without exporting."; + catalog = taskCatalog(); + for k = 1:numel(catalog) + plan(catalog(k).Name).Description = catalog(k).Description; + end end function checkStyleTask(~) - runBuildTests("checkStyle", ... - "Suites", "project", ... - "Tags", "Style"); + runCatalogTask("checkStyle"); end function testTask(~) - runBuildTests("test", ... - "IncludeGui", false); + runCatalogTask("test"); end function testUnitTask(~) - runBuildTests("testUnit", ... - "Tags", "Unit"); + runCatalogTask("testUnit"); end function testIntegrationTask(~) - runBuildTests("testIntegration", ... - "Tags", "Integration"); + runCatalogTask("testIntegration"); end function testProjectTask(~) - runBuildTests("testProject", ... - "Suites", "project"); + runCatalogTask("testProject"); end function testLabkitDtaTask(~) - runBuildTests("testLabkitDta", ... - "Suites", "labkit/dta"); + runCatalogTask("testLabkitDta"); end function testLabkitBiosignalTask(~) - runBuildTests("testLabkitBiosignal", ... - "Suites", "labkit/biosignal"); + runCatalogTask("testLabkitBiosignal"); end function testLabkitUiTask(~) - runBuildTests("testLabkitUi", ... - "Suites", "labkit/ui", ... - "IncludeGui", false); + runCatalogTask("testLabkitUi"); end function testLabkitUiGuiTask(~) - runBuildTests("testLabkitUiGui", ... - "Suites", "labkit/ui", ... - "IncludeGui", true); + runCatalogTask("testLabkitUiGui"); end function testAppsElectrochemTask(~) - runBuildTests("testAppsElectrochem", ... - "Suites", "apps/electrochem", ... - "IncludeGui", false); + runCatalogTask("testAppsElectrochem"); end function testAppsElectrochemGuiTask(~) - runBuildTests("testAppsElectrochemGui", ... - "Suites", "apps/electrochem", ... - "IncludeGui", true); + runCatalogTask("testAppsElectrochemGui"); end function testAppsDicGuiTask(~) - runBuildTests("testAppsDicGui", ... - "Suites", "apps/dic", ... - "IncludeGui", true); + runCatalogTask("testAppsDicGui"); end function testAppsImageMeasurementTask(~) - runBuildTests("testAppsImageMeasurement", ... - "Suites", "apps/image_measurement", ... - "IncludeGui", false); + runCatalogTask("testAppsImageMeasurement"); end function testAppsImageMeasurementGuiTask(~) - runBuildTests("testAppsImageMeasurementGui", ... - "Suites", "apps/image_measurement", ... - "IncludeGui", true); + runCatalogTask("testAppsImageMeasurementGui"); end function testAppsWearableGuiTask(~) - runBuildTests("testAppsWearableGui", ... - "Suites", "apps/wearable", ... - "IncludeGui", true); + runCatalogTask("testAppsWearableGui"); end function testAppsGuiTask(~) - runBuildTests("testAppsGui", ... - "Suites", "apps", ... - "IncludeGui", true); + runCatalogTask("testAppsGui"); end function testAppsSmokeGuiTask(~) - runBuildTests("testAppsSmokeGui", ... - "Suites", "apps/smoke", ... - "IncludeGui", true); + runCatalogTask("testAppsSmokeGui"); end function testGuiStructuralTask(~) - runBuildTests("testGuiStructural", ... - "Suites", "gui", ... - "Tags", "Structural", ... - "IncludeGui", true); + runCatalogTask("testGuiStructural"); end function testGuiGestureTask(~) - runBuildTests("testGuiGesture", ... - "Tags", "Gesture", ... - "IncludeGui", true); + runCatalogTask("testGuiGesture"); end function coverageTask(~) - runBuildTests("coverage", ... - "Tags", ["Unit", "Integration"], ... - "IncludeCoverage", true); + runCatalogTask("coverage"); end function checkProjectTask(~) @@ -187,6 +134,92 @@ function packageDryRunTask(~) numel(packageCandidates), numel(validationOnly)); end +function catalog = taskCatalog() + catalog = [ ... + taskSpec("checkStyle", "Run project/style guardrails.", "Suites", "project", "Tags", "Style"), ... + taskSpec("test", "Run the full non-GUI test entry point.", "IncludeGui", false), ... + taskSpec("testUnit", "Run official unit tests.", "Tags", "Unit"), ... + taskSpec("testIntegration", "Run official integration tests.", "Tags", "Integration"), ... + taskSpec("testProject", "Run project guardrails.", "Suites", "project"), ... + taskSpec("testLabkitDta", "Run DTA facade/parser tests.", "Suites", "labkit/dta"), ... + taskSpec("testLabkitBiosignal", "Run biosignal facade tests.", "Suites", "labkit/biosignal"), ... + taskSpec("testLabkitUi", "Run reusable UI non-GUI tests.", "Suites", "labkit/ui", "IncludeGui", false), ... + taskSpec("testLabkitUiGui", "Run reusable UI GUI tests.", "Suites", "labkit/ui", "IncludeGui", true), ... + taskSpec("testAppsElectrochem", "Run electrochem app non-GUI tests.", "Suites", "apps/electrochem", "IncludeGui", false), ... + taskSpec("testAppsElectrochemGui", "Run electrochem app GUI tests.", "Suites", "apps/electrochem", "IncludeGui", true), ... + taskSpec("testAppsDicGui", "Run DIC app GUI tests.", "Suites", "apps/dic", "IncludeGui", true), ... + taskSpec("testAppsImageMeasurement", "Run image-measurement app non-GUI tests.", "Suites", "apps/image_measurement", "IncludeGui", false), ... + taskSpec("testAppsImageMeasurementGui", "Run image-measurement app GUI tests.", "Suites", "apps/image_measurement", "IncludeGui", true), ... + taskSpec("testAppsWearableGui", "Run wearable app GUI tests.", "Suites", "apps/wearable", "IncludeGui", true), ... + taskSpec("testAppsGui", "Run all app GUI tests.", "Suites", "apps", "IncludeGui", true), ... + taskSpec("testAppsSmokeGui", "Run cross-app GUI smoke tests.", "Suites", "apps/smoke", "IncludeGui", true), ... + taskSpec("testGuiStructural", "Run noninteractive GUI structural tests.", "Suites", "gui", "Tags", "Structural", "IncludeGui", true), ... + taskSpec("testGuiGesture", "Run noninteractive/manual GUI gesture tests.", "Tags", "Gesture", "IncludeGui", true), ... + taskSpec("coverage", "Run official tests with coverage artifacts.", "Tags", ["Unit", "Integration"], "IncludeCoverage", true), ... + taskSpec("checkProject", "Verify MATLAB Project metadata and path setup.", "RunTests", false), ... + taskSpec("packageDryRun", "Verify package boundary inventory without exporting.", "RunTests", false)]; +end + +function spec = taskSpec(name, description, varargin) + p = inputParser; + p.FunctionName = "taskSpec"; + p.addParameter("RunTests", true, @isLogicalScalar); + p.addParameter("Suites", strings(1, 0), @isStringLikeList); + p.addParameter("Tags", strings(1, 0), @isStringLikeList); + p.addParameter("IncludeGui", [], @isEmptyOrLogicalScalar); + p.addParameter("IncludeCoverage", [], @isEmptyOrLogicalScalar); + p.addParameter("Required", true, @isLogicalScalar); + p.parse(varargin{:}); + + runTests = logical(p.Results.RunTests); + spec = struct( ... + "Name", string(name), ... + "Description", string(description), ... + "RunTests", runTests, ... + "Suites", normalizeTextList(p.Results.Suites), ... + "Tags", normalizeTextList(p.Results.Tags), ... + "IncludeGui", normalizeOptionalLogical(p.Results.IncludeGui), ... + "IncludeCoverage", normalizeOptionalLogical(p.Results.IncludeCoverage), ... + "Required", runTests && logical(p.Results.Required)); +end + +function runCatalogTask(runName) + spec = findTaskSpec(runName); + if ~spec.RunTests + error("LabKit:Build:CatalogTaskNotRunnable", ... + "Build task %s is not a test-runner task.", runName); + end + + args = taskRunArguments(spec); + runBuildTests(spec.Name, args{:}); +end + +function spec = findTaskSpec(runName) + catalog = taskCatalog(); + matches = [catalog.Name] == string(runName); + if ~any(matches) + error("LabKit:Build:UnknownCatalogTask", ... + "Unknown build task catalog entry: %s.", runName); + end + spec = catalog(matches); +end + +function args = taskRunArguments(spec) + args = {}; + if ~isempty(spec.Suites) + args = [args, {"Suites", spec.Suites}]; %#ok + end + if ~isempty(spec.Tags) + args = [args, {"Tags", spec.Tags}]; %#ok + end + if ~isempty(spec.IncludeGui) + args = [args, {"IncludeGui", spec.IncludeGui}]; %#ok + end + if ~isempty(spec.IncludeCoverage) + args = [args, {"IncludeCoverage", spec.IncludeCoverage}]; %#ok + end +end + function runBuildTests(runName, varargin) root = fileparts(mfilename("fullpath")); addpath(fullfile(root, "tests")); @@ -341,6 +374,39 @@ function assertRelativePathsExist(root, relativePaths) clear cleanup end +function values = normalizeTextList(values) + if isempty(values) + values = strings(1, 0); + elseif ischar(values) + values = string({values}); + elseif iscell(values) + values = string(values); + else + values = string(values); + end + values = values(:).'; + values = values(strlength(values) > 0); +end + +function value = normalizeOptionalLogical(value) + if isempty(value) + return; + end + value = logical(value); +end + +function tf = isStringLikeList(value) + tf = ischar(value) || isstring(value) || iscellstr(value); +end + +function tf = isLogicalScalar(value) + tf = (islogical(value) || isnumeric(value)) && isscalar(value); +end + +function tf = isEmptyOrLogicalScalar(value) + tf = isempty(value) || isLogicalScalar(value); +end + function normalized = normalizePaths(paths) normalized = strings(size(paths)); for k = 1:numel(paths) diff --git a/docs/testing.md b/docs/testing.md index e171268..c95c9a4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -42,11 +42,14 @@ buildtool packageDryRun - `buildtool test` is the full non-GUI entry point. - `buildtool checkStyle` runs official project/style guardrails. - `buildtool coverage` generates official JUnit, HTML test result, Cobertura, - and HTML coverage artifacts. Coverage is report-only. + and HTML coverage artifacts. Coverage is report-only and runs in manual or + scheduled CI, not as a default PR quality gate. - Official runner artifacts are namespaced by build task run name under `artifacts/test-results//`, `artifacts/coverage//`, `artifacts/gui//`, and `artifacts/logs//` so combined task - invocations do not overwrite each other. + invocations do not overwrite each other. GitHub Actions writes MATLAB logs + to the matching run-name log directory, such as + `artifacts/logs/testUnit/matlab.log`. - `buildtool testGuiGesture` runs focused noninteractive gesture coverage for runtime, anchor editor, and scale-bar interaction lifecycle checks. - `buildtool checkProject` verifies `LabKit.prj` path and startup metadata. @@ -82,6 +85,13 @@ Advanced targeted debugging can call the internal runner directly: runLabKitTests("Tests", "AppHookHelpersTest", "FailIfNoTests", true) ``` +To inspect test selection without executing tests or writing artifacts, use +the internal list-only mode: + +```matlab +runLabKitTests("Suites", "labkit/dta", "ListOnly", true) +``` + Use direct `runLabKitTests(...)` calls only for local diagnosis. Build tasks remain the official entry points for CI, PR validation, and handoff commands. @@ -93,10 +103,12 @@ remain the official entry points for CI, PR validation, and handoff commands. | Focused GUI build tasks | Local MATLAB with graphics support | Noninteractive launch, layout, and callback wiring checks for selected app families. | | Manual GUI validation | User-run app windows | Interactive file selection, drawing, visual inspection, and full workflow feel. | -CI runs quality, unit/coverage, and integration jobs on pushes and pull +CI runs shell-wrapper, quality, unit, and integration jobs on pushes and pull requests to `main` through `.github/workflows/matlab-tests.yml`. Manual and -scheduled CI runs also execute GUI structural and non-blocking GUI gesture jobs. -Do not describe CI as full interactive GUI workflow validation. +scheduled CI runs also execute coverage, GUI structural, and non-blocking GUI +gesture jobs. Coverage is intentionally outside the default PR gate to keep PR +feedback focused and avoid duplicate test execution. Do not describe CI as full +interactive GUI workflow validation. ## Focused Build Tasks diff --git a/tests/integration/project/BuildTaskFrameworkGuardrailTest.m b/tests/integration/project/BuildTaskFrameworkGuardrailTest.m new file mode 100644 index 0000000..0ecdea1 --- /dev/null +++ b/tests/integration/project/BuildTaskFrameworkGuardrailTest.m @@ -0,0 +1,249 @@ +classdef BuildTaskFrameworkGuardrailTest < matlab.unittest.TestCase + %BUILDTASKFRAMEWORKGUARDRAILTEST Guardrails for build task routing. + + methods (Test, TestTags = {'Integration', 'Style'}) + function buildTaskCatalogMatchesTaskFunctions(testCase) + root = setupLabKitTestPath(); + catalog = extractBuildfileCatalog(root); + taskFunctions = extractTaskFunctionNames(root); + + testCase.verifyEqual(sort(catalog.Name(:).'), sort(taskFunctions(:).'), ... + 'Every public build task function should have one catalog entry.'); + testCase.verifyEqual(numel(unique(catalog.Name)), numel(catalog.Name), ... + 'Build task catalog entries should be unique.'); + testCase.verifyTrue(all(strlength(catalog.Description) > 0), ... + 'Build task catalog entries should carry non-empty descriptions.'); + end + + function documentedAndWrapperTasksStayInCatalog(testCase) + root = setupLabKitTestPath(); + catalog = extractBuildfileCatalog(root); + catalogNames = catalog.Name; + + docFiles = [ ... + fullfile(root, "README.md"), ... + fullfile(root, "docs", "testing.md")]; + for k = 1:numel(docFiles) + tasks = extractBuildtoolTasks(fileread(docFiles(k))); + verifyTaskSubset(testCase, tasks, catalogNames, ... + "Documented buildtool tasks in " + relativePath(root, docFiles(k))); + end + + wrapperFiles = [ ... + fullfile(root, "scripts", "run_matlab_tests.sh"), ... + fullfile(root, "scripts", "run_matlab_tests.ps1")]; + for k = 1:numel(wrapperFiles) + tasks = extractWrapperCommonTasks(fileread(wrapperFiles(k))); + verifyTaskSubset(testCase, tasks, catalogNames, ... + "Wrapper help tasks in " + relativePath(root, wrapperFiles(k))); + end + end + + function focusedBuildTasksMatchAtLeastOneTest(testCase) + setupLabKitTestPath(); + taskSpecs = focusedTaskSpecs(); + for k = 1:numel(taskSpecs) + spec = taskSpecs(k); + output = runLabKitTests(spec.Args{:}, ... + "RunName", spec.Name + "_list", ... + "ListOnly", true); + testCase.verifyGreaterThan(output.count, 0, ... + "Focused build task should match tests: " + spec.Name); + end + end + + function defaultRunnerSelectionExcludesGuiTests(testCase) + setupLabKitTestPath(); + output = runLabKitTests( ... + "IncludeGui", false, ... + "RunName", "default_list", ... + "ListOnly", true); + + testCase.verifyGreaterThan(output.count, 0, ... + 'Default non-GUI runner selection should not be empty.'); + testCase.verifyFalse(any(contains(output.tests.Tags, "GUI")), ... + 'Default non-GUI runner selection must not include GUI tests.'); + end + + function testFilesUseKnownTags(testCase) + root = setupLabKitTestPath(); + allowedTags = ["Unit", "Integration", "GUI", "Structural", ... + "Gesture", "Style", "Smoke"]; + files = collectTestFiles(fullfile(root, "tests")); + testCase.assertFalse(isempty(files), ... + 'Test tag guardrail should scan test files.'); + + for k = 1:numel(files) + content = fileread(files(k)); + tagGroups = regexp(content, 'TestTags\s*=\s*\{([^}]*)\}', ... + 'tokens'); + rel = relativePath(root, files(k)); + testCase.verifyFalse(isempty(tagGroups), ... + "Test file is missing TestTags: " + rel); + for g = 1:numel(tagGroups) + tags = extractQuotedTags(tagGroups{g}{1}); + testCase.verifyFalse(isempty(tags), ... + "TestTags block is empty: " + rel); + unknown = setdiff(tags, allowedTags); + testCase.verifyTrue(isempty(unknown), ... + "Unknown TestTags in " + rel + ": " + strjoin(unknown, ", ")); + end + end + end + + function ciCoverageRunsOnlyOnManualOrScheduledWorkflows(testCase) + root = setupLabKitTestPath(); + workflowPath = fullfile(root, ".github", "workflows", ... + "matlab-tests.yml"); + workflow = string(fileread(workflowPath)); + + testCase.verifyFalse(contains(workflow, "tasks: testUnit coverage"), ... + 'PR unit job should not duplicate coverage execution.'); + testCase.verifyTrue(contains(workflow, "tasks: coverage"), ... + 'Coverage should remain available through a dedicated job.'); + coverageJob = extractWorkflowJob(workflow, "coverage"); + testCase.verifyTrue(contains(coverageJob, ... + "github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'"), ... + 'Coverage job should only run for manual or scheduled workflows.'); + testCase.verifyTrue(contains(coverageJob, "artifacts/coverage/**"), ... + 'Coverage job should upload coverage artifacts.'); + end + end +end + +function catalog = extractBuildfileCatalog(root) + content = fileread(fullfile(root, "buildfile.m")); + tokens = regexp(content, 'taskSpec\("([^"]+)",\s*"([^"]+)"', 'tokens'); + names = strings(1, numel(tokens)); + descriptions = strings(1, numel(tokens)); + for k = 1:numel(tokens) + names(k) = string(tokens{k}{1}); + descriptions(k) = string(tokens{k}{2}); + end + catalog = table(names.', descriptions.', ... + 'VariableNames', {'Name', 'Description'}); +end + +function names = extractTaskFunctionNames(root) + content = fileread(fullfile(root, "buildfile.m")); + tokens = regexp(content, ... + '(?m)^function\s+([A-Za-z][A-Za-z0-9_]*)Task\s*\(~\)', 'tokens'); + names = strings(1, numel(tokens)); + for k = 1:numel(tokens) + names(k) = string(tokens{k}{1}); + end +end + +function tasks = extractBuildtoolTasks(content) + tokens = regexp(char(content), ... + 'buildtool[ \t]+([A-Za-z][A-Za-z0-9_]*(?:[ \t]+[A-Za-z][A-Za-z0-9_]*)*)', ... + 'tokens'); + tasks = strings(1, 0); + for k = 1:numel(tokens) + tasks = [tasks, split(string(tokens{k}{1})).']; %#ok + end + tasks = unique(tasks(strlength(tasks) > 0), 'stable'); +end + +function tasks = extractWrapperCommonTasks(content) + content = char(content); + startIndex = strfind(content, 'Common tasks:'); + stopIndex = strfind(content, 'Removed interface:'); + if isempty(startIndex) || isempty(stopIndex) + tasks = strings(1, 0); + return; + end + + block = content(startIndex(1):stopIndex(1)-1); + tokens = regexp(block, '(?m)^\s{2}([A-Za-z][A-Za-z0-9_]*)\s*$', ... + 'tokens'); + tasks = strings(1, numel(tokens)); + for k = 1:numel(tokens) + tasks(k) = string(tokens{k}{1}); + end + tasks = unique(tasks, 'stable'); +end + +function verifyTaskSubset(testCase, tasks, catalogNames, label) + missing = setdiff(tasks, catalogNames); + testCase.verifyTrue(isempty(missing), ... + label + " not in buildfile catalog: " + strjoin(missing, ", ")); +end + +function taskSpecs = focusedTaskSpecs() + taskSpecs = [ ... + listTaskSpec("testProject", {"Suites", "project"}), ... + listTaskSpec("testLabkitDta", {"Suites", "labkit/dta"}), ... + listTaskSpec("testLabkitBiosignal", {"Suites", "labkit/biosignal"}), ... + listTaskSpec("testLabkitUi", {"Suites", "labkit/ui", "IncludeGui", false}), ... + listTaskSpec("testLabkitUiGui", {"Suites", "labkit/ui", "IncludeGui", true}), ... + listTaskSpec("testAppsElectrochem", {"Suites", "apps/electrochem", "IncludeGui", false}), ... + listTaskSpec("testAppsElectrochemGui", {"Suites", "apps/electrochem", "IncludeGui", true}), ... + listTaskSpec("testAppsDicGui", {"Suites", "apps/dic", "IncludeGui", true}), ... + listTaskSpec("testAppsImageMeasurement", {"Suites", "apps/image_measurement", "IncludeGui", false}), ... + listTaskSpec("testAppsImageMeasurementGui", {"Suites", "apps/image_measurement", "IncludeGui", true}), ... + listTaskSpec("testAppsWearableGui", {"Suites", "apps/wearable", "IncludeGui", true}), ... + listTaskSpec("testAppsGui", {"Suites", "apps", "IncludeGui", true}), ... + listTaskSpec("testAppsSmokeGui", {"Suites", "apps/smoke", "IncludeGui", true}), ... + listTaskSpec("testGuiStructural", {"Suites", "gui", "Tags", "Structural", "IncludeGui", true}), ... + listTaskSpec("testGuiGesture", {"Tags", "Gesture", "IncludeGui", true})]; +end + +function spec = listTaskSpec(name, args) + spec = struct("Name", string(name), "Args", {args}); +end + +function files = collectTestFiles(root) + files = strings(1, 0); + entries = dir(root); + [~, order] = sort({entries.name}); + entries = entries(order); + for k = 1:numel(entries) + entry = entries(k); + if entry.isdir + if strcmp(entry.name, ".") || strcmp(entry.name, "..") + continue; + end + files = [files, collectTestFiles(fullfile(entry.folder, entry.name))]; %#ok + elseif endsWith(entry.name, "Test.m") + files(end+1) = string(fullfile(entry.folder, entry.name)); %#ok + end + end +end + +function tags = extractQuotedTags(content) + tokens = regexp(content, '''([^'']+)''', 'tokens'); + tags = strings(1, numel(tokens)); + for k = 1:numel(tokens) + tags(k) = string(tokens{k}{1}); + end +end + +function job = extractWorkflowJob(workflow, jobName) + lines = splitlines(string(workflow)); + jobHeader = " " + jobName + ":"; + startLine = find(lines == jobHeader, 1); + if isempty(startLine) + job = ""; + return; + end + + stopLine = numel(lines); + for k = startLine + 1:numel(lines) + line = lines(k); + if startsWith(line, " ") && ~startsWith(line, " ") && ... + endsWith(line, ":") + stopLine = k - 1; + break; + end + end + job = strjoin(lines(startLine:stopLine), newline); +end + +function rel = relativePath(root, filepath) + rel = replace(string(filepath), "\", "/"); + root = replace(string(root), "\", "/"); + if startsWith(rel, root + "/") + rel = extractAfter(rel, strlength(root) + 1); + end +end diff --git a/tests/runLabKitTests.m b/tests/runLabKitTests.m index 2b4c286..4167c90 100644 --- a/tests/runLabKitTests.m +++ b/tests/runLabKitTests.m @@ -14,6 +14,7 @@ % FailIfNoTests Error when no official tests match. % ArtifactsRoot Root artifact directory. % RunName Name used in artifact titles and console output. +% ListOnly List matched tests without executing or writing artifacts. root = fileparts(fileparts(mfilename("fullpath"))); addpath(fullfile(root, "tests", "support")); @@ -21,10 +22,6 @@ opts = parseOptions(root, varargin{:}); restoreRunName = setRunNameEnvironment(opts.RunName); - paths = labkitArtifactPaths( ... - "Root", opts.ArtifactsRoot, ... - "RunName", opts.RunName, ... - "Create", true); suite = discoverOfficialSuite(root, opts); fprintf("LabKit official test run: %s\n", opts.RunName); @@ -35,6 +32,24 @@ "No official matlab.unittest tests matched the requested selection."); end + if opts.ListOnly + listing = suiteListingTable(suite); + printSuiteListing(listing); + output = struct( ... + "official", matlab.unittest.Test.empty(1, 0), ... + "artifacts", struct(), ... + "runName", opts.RunName, ... + "listOnly", true, ... + "count", height(listing), ... + "tests", listing); + clear restoreRunName + return; + end + + paths = labkitArtifactPaths( ... + "Root", opts.ArtifactsRoot, ... + "RunName", opts.RunName, ... + "Create", true); runner = matlab.unittest.TestRunner.withTextOutput( ... "OutputDetail", opts.OutputDetail, ... "LoggingLevel", opts.LoggingLevel); @@ -91,6 +106,7 @@ p.addParameter("FailIfNoTests", true, @isLogicalScalar); p.addParameter("ArtifactsRoot", fullfile(root, "artifacts"), @isTextScalar); p.addParameter("RunName", "local", @isTextScalar); + p.addParameter("ListOnly", false, @isLogicalScalar); p.addParameter("OutputDetail", "Concise", @isTextScalar); p.addParameter("LoggingLevel", "Concise", @isTextScalar); p.parse(varargin{:}); @@ -99,6 +115,7 @@ opts.IncludeGui = logical(opts.IncludeGui); opts.IncludeCoverage = logical(opts.IncludeCoverage); opts.FailIfNoTests = logical(opts.FailIfNoTests); + opts.ListOnly = logical(opts.ListOnly); opts.Suites = normalizeTextList(opts.Suites); opts.Tests = normalizeTextList(opts.Tests); opts.Tags = normalizeTextList(opts.Tags); @@ -121,6 +138,33 @@ suite = filterSuiteByTags(suite, opts.Tags, opts.ExcludeTags); end +function listing = suiteListingTable(suite) + names = strings(numel(suite), 1); + tags = strings(numel(suite), 1); + for k = 1:numel(suite) + names(k) = string(suite(k).Name); + suiteTags = string(suite(k).Tags); + if isempty(suiteTags) + tags(k) = ""; + else + tags(k) = strjoin(suiteTags, ","); + end + end + listing = table(names, tags, 'VariableNames', {'Name', 'Tags'}); +end + +function printSuiteListing(listing) + if isempty(listing) + fprintf("No tests matched.\n"); + return; + end + + fprintf("Matched official tests:\n"); + for k = 1:height(listing) + fprintf(" %s [%s]\n", char(listing.Name(k)), char(listing.Tags(k))); + end +end + function groups = discoverOfficialGroups(testsRoot) groups = struct("key", {}, "suite", {}); roots = ["unit", "integration", "gui"]; diff --git a/tests/unit/project/PlatformSkeletonTest.m b/tests/unit/project/PlatformSkeletonTest.m index ddc30c6..238e66a 100644 --- a/tests/unit/project/PlatformSkeletonTest.m +++ b/tests/unit/project/PlatformSkeletonTest.m @@ -19,6 +19,8 @@ function artifactPathsUseRoadmapLayout(testCase) fullfile("coverage", "cobertura.xml"))); testCase.verifyTrue(endsWith(string(paths.coverageHtml), ... fullfile("coverage", "html"))); + testCase.verifyTrue(endsWith(string(paths.matlabLog), ... + fullfile("logs", "matlab.log"))); testCase.verifyTrue(endsWith(string(paths.guiTrace), ... fullfile("gui", "trace"))); testCase.verifyTrue(endsWith(string(paths.guiSnapshots), ... @@ -36,6 +38,8 @@ function artifactPathsUseRoadmapLayout(testCase) fullfile("coverage", "testUnit", "cobertura.xml"))); testCase.verifyTrue(endsWith(string(runPaths.coverageHtml), ... fullfile("coverage", "testUnit", "html"))); + testCase.verifyTrue(endsWith(string(runPaths.matlabLog), ... + fullfile("logs", "testUnit", "matlab.log"))); testCase.verifyTrue(endsWith(string(runPaths.guiTrace), ... fullfile("gui", "testUnit", "trace"))); testCase.verifyTrue(endsWith(string(runPaths.guiSnapshots), ...