From e7c534de6436b9c89f0c2fdff8c0fcb2fbd6f8e7 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 10 Oct 2026 13:37:35 -0400 Subject: [PATCH 1/2] fix(test): make plan inspection accurate and safe for cloud development --- Skills/xcforge/references/test-tools.md | 4 +- Sources/XCForgeKit/Skill/EmbeddedSkill.swift | 4 +- .../XCForgeKit/Tools/TestPlanInspector.swift | 30 ++++++++++-- .../TestPlanInspectorTests.swift | 49 +++++++++++++++++++ 4 files changed, 81 insertions(+), 6 deletions(-) diff --git a/Skills/xcforge/references/test-tools.md b/Skills/xcforge/references/test-tools.md index 006356b..4038168 100644 --- a/Skills/xcforge/references/test-tools.md +++ b/Skills/xcforge/references/test-tools.md @@ -182,7 +182,9 @@ List available test identifiers for a scheme. Use to discover the correct filter ## test_plan_inspect -Parse and summarize a `.xctestplan` file without running tests. +Parse and summarize a `.xctestplan` file without building, resolving packages, launching a simulator, or enumerating compiled tests. Reports timeout allowances, target execution ordering, declared selections, tag filters and disabled targets (`skipped: true` or legacy `enabled: false`). + +This is the safe inspection tool in an editing loop where native builds are forbidden. `list_tests` / `xcforge test list` can compile the app to enumerate tests and requires permission for native execution. Static selections and tag filters do not prove an executed test count; use the qualification run's results for that. | Parameter | Required | Default | Description | |-----------|----------|---------|-------------| diff --git a/Sources/XCForgeKit/Skill/EmbeddedSkill.swift b/Sources/XCForgeKit/Skill/EmbeddedSkill.swift index 2ea9d40..8156f81 100644 --- a/Sources/XCForgeKit/Skill/EmbeddedSkill.swift +++ b/Sources/XCForgeKit/Skill/EmbeddedSkill.swift @@ -3868,7 +3868,9 @@ List available test identifiers for a scheme. Use to discover the correct filter ## test_plan_inspect -Parse and summarize a `.xctestplan` file without running tests. +Parse and summarize a `.xctestplan` file without building, resolving packages, launching a simulator, or enumerating compiled tests. Reports timeout allowances, target execution ordering, declared selections, tag filters and disabled targets (`skipped: true` or legacy `enabled: false`). + +This is the safe inspection tool in an editing loop where native builds are forbidden. `list_tests` / `xcforge test list` can compile the app to enumerate tests and requires permission for native execution. Static selections and tag filters do not prove an executed test count; use the qualification run's results for that. | Parameter | Required | Default | Description | |-----------|----------|---------|-------------| diff --git a/Sources/XCForgeKit/Tools/TestPlanInspector.swift b/Sources/XCForgeKit/Tools/TestPlanInspector.swift index 0035563..7591cb2 100644 --- a/Sources/XCForgeKit/Tools/TestPlanInspector.swift +++ b/Sources/XCForgeKit/Tools/TestPlanInspector.swift @@ -37,14 +37,23 @@ public struct TestPlanOptions: Codable { public var codeCoverage: Bool? public var threadSanitizerEnabled: Bool? public var addressSanitizerEnabled: Bool? + public var testTimeoutsEnabled: Bool? + public var defaultTestExecutionTimeAllowance: Double? + public var maximumTestExecutionTimeAllowance: Double? + public var testExecutionOrdering: String? } public struct TestPlanTarget: Codable { public var target: TestTargetRef public var parallelizable: Bool? public var enabled: Bool? + /// Xcode's actual target-disable key. `enabled` remains readable for older tool plans. + public var skipped: Bool? + public var options: TestPlanOptions? public var skippedTests: [SkippedTest]? public var selectedTests: [SkippedTest]? + + var isEnabled: Bool { enabled != false && skipped != true } } public struct TestTargetRef: Codable { @@ -132,7 +141,7 @@ public enum TestPlanInspector { guard let data = FileManager.default.contents(atPath: path), let decoded = try? JSONDecoder().decode(XCTestPlan.self, from: data) else { return nil } - return decoded.testTargets.filter { $0.enabled != false }.map(\.target.name) + return decoded.testTargets.filter(\.isEnabled).map(\.target.name) } // MARK: - Private @@ -228,13 +237,19 @@ public enum TestPlanInspector { for target in plan.testTargets { let skipped = target.skippedTests?.count ?? 0 var targetLine = " \(target.target.name)" - if target.enabled == false { targetLine += " (disabled)" } + if !target.isEnabled { targetLine += " (disabled)" } if let parallel = target.parallelizable { targetLine += " (parallelizable: \(parallel))" } if skipped > 0 { targetLine += " [\(skipped) skipped]" } if let selected = target.selectedTests, !selected.isEmpty { targetLine += " [only \(selected.count) selected]" } lines.append(targetLine) + if let options = target.options { lines.append(formatOptions(options, indent: " ")) } + for test in (target.selectedTests ?? []).prefix(10) { + lines.append(" select: \(test.identifier)") + } + let selected = target.selectedTests?.count ?? 0 + if selected > 10 { lines.append(" ... \(selected - 10) more selected") } for test in (target.skippedTests ?? []).prefix(10) { lines.append(" skip: \(test.identifier)") } @@ -248,12 +263,15 @@ public enum TestPlanInspector { if hasMultiTagFilter(tagSettings) { lines.append( " Warning: with several included tags, a plan set to match all runs only tests that carry " - + "every one of them. Compare `xcforge test list --testplan ` with the scheme's count.") + + "every one of them. Confirm the intended selection before native qualification.") } } lines.append("") - lines.append("Run `xcforge test list --testplan ` to see exactly which tests this plan runs.") + lines.append("Metadata only: no build, package resolution, simulator launch or test enumeration was performed.") + lines.append( + "`xcforge test list` can compile the app to enumerate tests; use it only where native builds are permitted. " + + "Declared selections and tag filters do not establish the executed test count.") return lines.joined(separator: "\n") } @@ -263,6 +281,10 @@ public enum TestPlanInspector { if let v = options.codeCoverage { parts.append("codeCoverage: \(v)") } if let v = options.threadSanitizerEnabled { parts.append("threadSanitizer: \(v)") } if let v = options.addressSanitizerEnabled { parts.append("addressSanitizer: \(v)") } + if let v = options.testTimeoutsEnabled { parts.append("testTimeoutsEnabled: \(v)") } + if let v = options.defaultTestExecutionTimeAllowance { parts.append("defaultTestExecutionTimeAllowance: \(v)s") } + if let v = options.maximumTestExecutionTimeAllowance { parts.append("maximumTestExecutionTimeAllowance: \(v)s") } + if let v = options.testExecutionOrdering { parts.append("testExecutionOrdering: \(v)") } guard !parts.isEmpty else { return "\(indent)(no options set)" } return parts.map { "\(indent)\($0)" }.joined(separator: "\n") } diff --git a/Tests/XCForgeKitTests/TestPlanInspectorTests.swift b/Tests/XCForgeKitTests/TestPlanInspectorTests.swift index 5ea9441..050ac5b 100644 --- a/Tests/XCForgeKitTests/TestPlanInspectorTests.swift +++ b/Tests/XCForgeKitTests/TestPlanInspectorTests.swift @@ -116,5 +116,54 @@ struct TestPlanInspectorTests { #expect(summary.contains("Sanitizer Config")) #expect(summary.contains("MyAppTests")) #expect(summary.contains("1 skipped")) + #expect(summary.contains("Metadata only: no build")) + #expect(summary.contains("can compile the app")) + #expect(!summary.contains("Run `xcforge test list")) + } + + @Test("Xcode skipped targets and legacy disabled targets are excluded from discovery") + func excludesSkippedTargets() throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("xcforge-targets-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let file = directory.appendingPathComponent("ci.xctestplan") + try #""" + {"testTargets":[ + {"target":{"name":"ActiveTests"}}, + {"target":{"name":"SkippedUITests"},"skipped":true}, + {"target":{"name":"LegacyDisabledTests"},"enabled":false}, + {"target":{"name":"ExplicitActiveTests"},"skipped":false} + ]} + """#.write(to: file, atomically: true, encoding: .utf8) + #expect(TestPlanInspector.testTargetNames(atPath: file.path) == ["ActiveTests", "ExplicitActiveTests"]) + } + + @Test("metadata inspection reports execution budgets and declared selections") + func reportsTimeoutsAndSelectionsWithoutEnumeration() async throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("xcforge-budgets-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let file = directory.appendingPathComponent("ci.xctestplan") + try #""" + { + "configurations":[{"name":"Smoke", "options":{"maximumTestExecutionTimeAllowance":120}}], + "defaultOptions":{"testTimeoutsEnabled":true,"defaultTestExecutionTimeAllowance":60}, + "testTargets":[ + {"target":{"name":"AppUITests"},"options":{"testExecutionOrdering":"alphabetical"}, + "selectedTests":["CritiqueFlowUITests/testCompletes()"]}, + {"target":{"name":"DisabledTests"},"skipped":true} + ] + } + """#.write(to: file, atomically: true, encoding: .utf8) + let summary = try await TestPlanInspector.inspectTestPlan(name: "ci", project: directory.path, env: .live) + #expect(summary.contains("testTimeoutsEnabled: true")) + #expect(summary.contains("defaultTestExecutionTimeAllowance: 60.0s")) + #expect(summary.contains("maximumTestExecutionTimeAllowance: 120.0s")) + #expect(summary.contains("testExecutionOrdering: alphabetical")) + #expect(summary.contains("select: CritiqueFlowUITests/testCompletes()")) + #expect(summary.contains("DisabledTests (disabled)")) + #expect(!summary.contains("Run `xcforge test list")) } } From 93918d89e82b233f9812bbb10ddabd2181df2ba1 Mon Sep 17 00:00:00 2001 From: Justin Date: Sat, 10 Oct 2026 14:22:53 -0400 Subject: [PATCH 2/2] chore(release): prepare xcforge v1.7.1 --- CHANGELOG.md | 5 +++++ Sources/XCForgeKit/Support/XCForgeVersion.swift | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc21147..4636164 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/), and this ## [Unreleased] +## [1.7.1] - 2026-10-10 + +### Fixed +- Static test-plan inspection honors Xcode's skipped targets, displays timeout settings and declared test selections, and keeps metadata inspection separate from compiling test discovery. The embedded agent skill now documents the same distinction. + ## [1.7.0] - 2026-10-10 ### Added diff --git a/Sources/XCForgeKit/Support/XCForgeVersion.swift b/Sources/XCForgeKit/Support/XCForgeVersion.swift index c005357..e608643 100644 --- a/Sources/XCForgeKit/Support/XCForgeVersion.swift +++ b/Sources/XCForgeKit/Support/XCForgeVersion.swift @@ -1,4 +1,4 @@ /// The version the MCP server reports and `xcforge --version` prints. public enum XCForgeVersion { - public static let current = "1.7.0" + public static let current = "1.7.1" }