diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9659fb6..dabd8b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,7 +88,7 @@ jobs: - name: Check Zig formatting run: >- zig fmt --check - build.zig build.zig.zon src + build.zig build.zig.zon src benchmark test/acceptance.zig test/dogfood.zig test/readme.zig test/readme - name: Run ReleaseSafe unit, acceptance, dogfood, and consumer tests @@ -97,6 +97,18 @@ jobs: - name: Build and validate ReleaseSafe guide and API docs run: zig build docs -Doptimize=ReleaseSafe + - name: Measure extraction and cached rule performance + run: zig build benchmark-check + + - name: Upload performance evidence + if: always() && hashFiles('zig-out/benchmark/results.json') != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: performance-results + path: zig-out/benchmark/results.json + if-no-files-found: error + retention-days: 30 + - name: Upload validated documentation site uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0b3e7dd..ef3403b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,6 +27,7 @@ the repository documentation records decisions that outlive an individual ticket python -B scripts/check_ci.py zig build test -Doptimize=ReleaseSafe zig build docs -Doptimize=ReleaseSafe + zig build benchmark-check ``` 5. Open a pull request that links the issue, explains Zig-specific trade-offs, and records the exact @@ -65,7 +66,8 @@ README/license root commit is the one unavoidable exception to the pull-request CI runs Debug tests on current GitHub-hosted Linux, Windows, and macOS runners. A Linux quality job checks formatting, reruns the complete suite in `ReleaseSafe`, builds the documentation site and Zig -API docs, and uploads the validated artifact. The complete test command includes unit tests, +API docs, runs the deterministic extraction benchmark, and uploads both validated artifacts. The +complete test command includes unit tests, acceptance fixtures, executable dogfood rules, and the standalone README consumer. Zig 0.16.0 is installed by `scripts/install_zig.py` from exact official archive URLs. The script @@ -73,3 +75,9 @@ checks both the published byte size and SHA-256 before extraction. External work pinned to immutable commits with their reviewed release tag in a comment. CI deliberately uses no dependency or build cache; adding one requires a key containing the Zig version and all relevant ZON hash/lock inputs. + +Performance work starts with evidence. `zig build benchmark` generates the versioned fixture and +writes `zig-out/benchmark/results.json`; `zig build benchmark-check` also enforces the broad budgets +recorded in `benchmark/budgets.json`. Update those budgets only from an uncached hosted Linux CI run, +and preserve substantial headroom for shared-runner variance. Correctness, deterministic graph +output, cache behavior, and zero live tracked allocations remain hard gates in both modes. diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..ee922e1 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,27 @@ +# Extraction benchmark + +`zig build benchmark` generates a deterministic Zig project under `.zig-cache/benchmark`, runs the +public extraction pipeline stage by stage in `ReleaseSafe`, exercises one cold and twenty cached +fluent checks, and writes machine-readable evidence to `zig-out/benchmark/results.json`. + +The fixture has 240 source files in eight nested folders. Every file imports twelve peers and the +standard library, producing a dense but deterministic graph. Generation is deliberately outside +the measurements. The benchmark times enumeration, source loading, tokenize/parse, +resolution/classification, normalization, projection, the first rule check, and cached rule checks +separately. A tracking allocator records allocation calls, bytes requested, and peak live bytes for +each stage. Graph counts, a stable digest, projection cardinality, rule results, cache behavior, and +zero live tracked bytes are correctness gates even in report-only mode. + +`zig build benchmark-check` additionally enforces `budgets.json`. Budgets are intentionally broad +hosted-CI regression tripwires, not microbenchmark promises or laptop-derived product guarantees. +The evidence block records the hosted run used to choose them. Update it only from an uncached +`ubuntu-latest` quality run, retaining enough headroom for normal shared-runner variance. + +The current runtime ceilings are rounded above 25 times the recorded green Linux baseline, with +100–200 ms floors for filesystem stages too small for a useful ratio. The total ceiling is 20 +seconds. The 64 MiB caller-allocator ceiling is more than eleven times the recorded peak. These are +coarse order-of-magnitude regression guards; changing fixture scale requires a new evidence run and +budget review rather than silently widening them. + +The generated fixture is versioned in its directory name. Change that version whenever its shape or +contents change, so stale files cannot silently influence a new benchmark definition. diff --git a/benchmark/budgets.json b/benchmark/budgets.json new file mode 100644 index 0000000..90db75a --- /dev/null +++ b/benchmark/budgets.json @@ -0,0 +1,36 @@ +{ + "schema_version": 1, + "evidence": { + "status": "hosted-ci-green", + "runner": "ubuntu-latest", + "run_url": "https://github.com/LukasNiessen/ArchUnitZig/actions/runs/32723254284", + "observed_commit": "8fcd3476c5db9828a9bdbde2afdb6f19a3983abb", + "baseline": { + "total_duration_ns": 784640735, + "peak_live_bytes": 5966099, + "stages": { + "enumeration_ns": 480079, + "source_loading_ns": 3344149, + "tokenize_parse_ns": 37220524, + "resolution_classification_ns": 151594513, + "normalization_ns": 36955236, + "projection_ns": 53994820, + "first_check_ns": 183165402, + "cached_checks_ns": 317886012, + "cached_check_per_iteration_ns": 15894301 + } + } + }, + "max_total_ns": 20000000000, + "max_peak_live_bytes": 67108864, + "stages": { + "enumeration_ns": 100000000, + "source_loading_ns": 200000000, + "tokenize_parse_ns": 1000000000, + "resolution_classification_ns": 4000000000, + "normalization_ns": 1000000000, + "projection_ns": 1500000000, + "first_check_ns": 5000000000, + "cached_check_per_iteration_ns": 500000000 + } +} diff --git a/benchmark/contracts.zig b/benchmark/contracts.zig new file mode 100644 index 0000000..c823180 --- /dev/null +++ b/benchmark/contracts.zig @@ -0,0 +1,46 @@ +const std = @import("std"); + +test { + _ = @import("tracking_allocator.zig"); +} + +test "budgets cover every measured stage with positive values" { + var parsed = try std.json.parseFromSlice( + std.json.Value, + std.testing.allocator, + @embedFile("budgets.json"), + .{}, + ); + defer parsed.deinit(); + const root = parsed.value.object; + try std.testing.expectEqual(@as(i64, 1), root.get("schema_version").?.integer); + try std.testing.expect(root.get("max_total_ns").?.integer > 0); + try std.testing.expect(root.get("max_peak_live_bytes").?.integer > 0); + const evidence = root.get("evidence").?.object; + try std.testing.expectEqualStrings("hosted-ci-green", evidence.get("status").?.string); + try std.testing.expect(std.mem.startsWith( + u8, + evidence.get("run_url").?.string, + "https://github.com/LukasNiessen/ArchUnitZig/actions/runs/", + )); + const baseline = evidence.get("baseline").?.object; + try std.testing.expect(root.get("max_total_ns").?.integer >= baseline.get("total_duration_ns").?.integer * 25); + try std.testing.expect(root.get("max_peak_live_bytes").?.integer >= baseline.get("peak_live_bytes").?.integer * 8); + const stages = root.get("stages").?.object; + const baseline_stages = baseline.get("stages").?.object; + const expected = [_][]const u8{ + "enumeration_ns", + "source_loading_ns", + "tokenize_parse_ns", + "resolution_classification_ns", + "normalization_ns", + "projection_ns", + "first_check_ns", + "cached_check_per_iteration_ns", + }; + try std.testing.expectEqual(expected.len, stages.count()); + for (expected) |name| { + try std.testing.expect(stages.get(name).?.integer > 0); + try std.testing.expect(stages.get(name).?.integer >= baseline_stages.get(name).?.integer * 25); + } +} diff --git a/benchmark/extraction.zig b/benchmark/extraction.zig new file mode 100644 index 0000000..9c859c6 --- /dev/null +++ b/benchmark/extraction.zig @@ -0,0 +1,655 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const archunit = @import("archunit"); +const tracking = @import("tracking_allocator.zig"); + +const Allocator = std.mem.Allocator; +const Io = std.Io; +const TrackingAllocator = tracking.TrackingAllocator; + +const fixture_file_count: usize = 240; +const imports_per_file: usize = 12; +const fixture_depth: usize = 8; +const cached_iterations: usize = 20; +const fixture_root = ".zig-cache/benchmark/extraction-v1-f240-i12-d8"; +const fixture_source_dir = fixture_root ++ "/src/level_0/level_1/level_2/level_3/level_4/level_5/level_6/level_7"; +const results_path = "zig-out/benchmark/results.json"; +const expected_graph_digest = "f0559163589aa672d1e8f68265967864ffcdf2f52665af936efe6cd2947193b5"; + +const expected_source_files = fixture_file_count + 2; +const expected_zig_files = fixture_file_count + 1; +const expected_internal_import_edges = fixture_file_count * imports_per_file; +const expected_external_import_edges = fixture_file_count + 1; +const expected_graph_edges = expected_zig_files + expected_internal_import_edges + expected_external_import_edges; + +const Stage = enum { + enumeration, + source_loading, + tokenize_parse, + resolution_classification, + normalization, + projection, + first_check, + cached_checks, +}; + +const all_stages = [_]Stage{ + .enumeration, + .source_loading, + .tokenize_parse, + .resolution_classification, + .normalization, + .projection, + .first_check, + .cached_checks, +}; + +const Evidence = struct { + status: []const u8, + runner: []const u8, + run_url: ?[]const u8, + observed_commit: ?[]const u8, + baseline: ?Baseline = null, +}; + +const BaselineStages = struct { + enumeration_ns: u64, + source_loading_ns: u64, + tokenize_parse_ns: u64, + resolution_classification_ns: u64, + normalization_ns: u64, + projection_ns: u64, + first_check_ns: u64, + cached_checks_ns: u64, + cached_check_per_iteration_ns: u64, +}; + +const Baseline = struct { + total_duration_ns: u64, + peak_live_bytes: usize, + stages: BaselineStages, +}; + +const StageBudgets = struct { + enumeration_ns: u64, + source_loading_ns: u64, + tokenize_parse_ns: u64, + resolution_classification_ns: u64, + normalization_ns: u64, + projection_ns: u64, + first_check_ns: u64, + cached_check_per_iteration_ns: u64, +}; + +const Budgets = struct { + schema_version: u32, + evidence: Evidence, + max_total_ns: u64, + max_peak_live_bytes: usize, + stages: StageBudgets, +}; + +const StageMeasurement = struct { + stage: Stage, + duration_ns: u64, + allocation_calls: usize, + allocated_bytes: usize, + peak_live_bytes: usize, +}; + +const BenchmarkResult = struct { + schema_version: u32 = 1, + zig_version: []const u8 = builtin.zig_version_string, + optimize_mode: []const u8 = @tagName(builtin.mode), + target_os: []const u8 = @tagName(builtin.os.tag), + target_arch: []const u8 = @tagName(builtin.cpu.arch), + fixture: struct { + version: u32 = 1, + source_files: usize = fixture_file_count, + imports_per_file: usize = imports_per_file, + folder_depth: usize = fixture_depth, + cached_iterations: usize = cached_iterations, + } = .{}, + enumerated_files: usize, + zig_files: usize, + graph_edges: usize, + projected_edges: usize, + stable_graph_digest: []u8, + total_duration_ns: u64, + overall_peak_live_bytes: usize, + final_live_bytes: usize, + measurements: [all_stages.len]StageMeasurement, + budgets: Budgets, + budgets_enforced: bool, + + fn deinit(self: *BenchmarkResult, allocator: Allocator) void { + allocator.free(self.stable_graph_digest); + self.* = undefined; + } +}; + +const LoadedSource = struct { + path: []const u8, + contents: ?[:0]u8 = null, + + fn deinit(self: *LoadedSource, allocator: Allocator) void { + if (self.contents) |contents| allocator.free(contents); + } +}; + +const ParsedSource = struct { + result: ?archunit.ParseResult = null, + + fn deinit(self: *ParsedSource, allocator: Allocator) void { + if (self.result) |*result| result.deinit(allocator); + } +}; + +const ClassifiedSource = struct { + references: std.ArrayList(archunit.ClassifiedReference) = .empty, + + fn deinit(self: *ClassifiedSource, allocator: Allocator) void { + for (self.references.items) |*reference| reference.deinit(allocator); + self.references.deinit(allocator); + } +}; + +pub fn main(init: std.process.Init) !void { + const io = init.io; + const allocator = init.gpa; + const args = try init.minimal.args.toSlice(init.arena.allocator()); + var enforce_budgets = false; + for (args[1..]) |arg| { + if (std.mem.eql(u8, arg, "--enforce-budgets")) { + enforce_budgets = true; + } else { + return error.InvalidArgument; + } + } + + var parsed_budgets = try std.json.parseFromSlice(Budgets, allocator, @embedFile("budgets.json"), .{}); + defer parsed_budgets.deinit(); + if (parsed_budgets.value.schema_version != 1) return error.UnsupportedBudgetSchema; + + try generateFixture(allocator, io); + var result = try runBenchmark(allocator, io, parsed_budgets.value, enforce_budgets); + defer result.deinit(allocator); + try writeResult(allocator, io, &result); + try printResult(io, &result); + if (enforce_budgets) try enforceBudgets(&result); +} + +fn runBenchmark( + child_allocator: Allocator, + io: Io, + budgets: Budgets, + budgets_enforced: bool, +) !BenchmarkResult { + archunit.clearGraphCache(); + defer archunit.clearGraphCache(); + + var tracker = TrackingAllocator{ .child = child_allocator }; + const allocator = tracker.allocator(); + var measurements: [all_stages.len]StageMeasurement = undefined; + var owned_digest: ?[]u8 = null; + errdefer if (owned_digest) |digest| child_allocator.free(digest); + var zig_file_count: usize = 0; + + { + var diagnostics = archunit.ErrorContext.init(allocator); + defer diagnostics.deinit(); + + var window = tracker.beginWindow(); + var started = benchmarkTime(io); + var source_files = try archunit.enumerateSourceFiles( + allocator, + io, + fixture_root, + .{}, + &diagnostics, + ); + var ended = benchmarkTime(io); + measurements[@intFromEnum(Stage.enumeration)] = makeMeasurement( + &tracker, + .enumeration, + started, + ended, + window, + ); + defer source_files.deinit(allocator); + try expectEqual(expected_source_files, source_files.items().len, "enumerated source count"); + + const loaded = try allocator.alloc(LoadedSource, source_files.items().len); + defer { + for (loaded) |*entry| entry.deinit(allocator); + allocator.free(loaded); + } + for (source_files.items(), loaded) |path, *entry| entry.* = .{ .path = path }; + + window = tracker.beginWindow(); + started = benchmarkTime(io); + for (loaded) |*entry| { + if (!std.mem.endsWith(u8, entry.path, ".zig")) continue; + const absolute_path = try std.fs.path.join(allocator, &.{ fixture_root, entry.path }); + defer allocator.free(absolute_path); + entry.contents = try Io.Dir.cwd().readFileAllocOptions( + io, + absolute_path, + allocator, + .limited(std.math.maxInt(usize)), + .of(u8), + 0, + ); + } + ended = benchmarkTime(io); + measurements[@intFromEnum(Stage.source_loading)] = makeMeasurement( + &tracker, + .source_loading, + started, + ended, + window, + ); + + const parsed = try allocator.alloc(ParsedSource, loaded.len); + defer { + for (parsed) |*entry| entry.deinit(allocator); + allocator.free(parsed); + } + for (parsed) |*entry| entry.* = .{}; + + window = tracker.beginWindow(); + started = benchmarkTime(io); + for (loaded, parsed) |entry, *parsed_entry| { + const contents = entry.contents orelse continue; + parsed_entry.result = try archunit.parseSource( + allocator, + entry.path, + contents, + .strict, + &diagnostics, + ); + zig_file_count += 1; + } + ended = benchmarkTime(io); + measurements[@intFromEnum(Stage.tokenize_parse)] = makeMeasurement( + &tracker, + .tokenize_parse, + started, + ended, + window, + ); + try expectEqual(expected_zig_files, zig_file_count, "parsed Zig source count"); + + const classified = try allocator.alloc(ClassifiedSource, loaded.len); + defer { + for (classified) |*entry| entry.deinit(allocator); + allocator.free(classified); + } + for (classified) |*entry| entry.* = .{}; + const sources = try allocator.alloc(archunit.SourceReferences, loaded.len); + defer allocator.free(sources); + + window = tracker.beginWindow(); + started = benchmarkTime(io); + for (loaded, parsed, classified, sources) |loaded_entry, parsed_entry, *classified_entry, *source| { + if (parsed_entry.result) |parse_result| { + try classified_entry.references.ensureTotalCapacity( + allocator, + parse_result.references.items.len, + ); + for (parse_result.references.items) |reference| { + var classified_reference = switch (reference.kind) { + .zig_file, .zon_file, .embedded_file => blk: { + var resolution = (try archunit.resolveRelativeReference( + allocator, + io, + fixture_root, + loaded_entry.path, + reference, + &diagnostics, + )).?; + defer resolution.deinit(allocator); + break :blk try archunit.classifyReference(allocator, .{ .file = .{ + .reference = reference, + .resolution = resolution, + } }); + }, + else => try archunit.classifyReference(allocator, .{ .raw = reference }), + }; + classified_entry.references.append(allocator, classified_reference) catch |failure| { + classified_reference.deinit(allocator); + return failure; + }; + } + } + source.* = .{ + .source_path = loaded_entry.path, + .references = classified_entry.references.items, + }; + } + ended = benchmarkTime(io); + measurements[@intFromEnum(Stage.resolution_classification)] = makeMeasurement( + &tracker, + .resolution_classification, + started, + ended, + window, + ); + + window = tracker.beginWindow(); + started = benchmarkTime(io); + var graph = try archunit.normalizeGraph(allocator, sources); + ended = benchmarkTime(io); + measurements[@intFromEnum(Stage.normalization)] = makeMeasurement( + &tracker, + .normalization, + started, + ended, + window, + ); + defer graph.deinit(allocator); + try expectEqual(expected_graph_edges, graph.len(), "normalized graph edge count"); + + const digest = stableGraphDigest(&graph); + const digest_hex = std.fmt.bytesToHex(digest, .lower); + if (!std.mem.eql(u8, &digest_hex, expected_graph_digest)) return error.UnexpectedGraphDigest; + owned_digest = try child_allocator.dupe(u8, &digest_hex); + var second_graph = try archunit.normalizeGraph(allocator, sources); + defer second_graph.deinit(allocator); + const second_digest = stableGraphDigest(&second_graph); + if (!std.mem.eql(u8, &digest, &second_digest)) return error.NondeterministicGraph; + + window = tracker.beginWindow(); + started = benchmarkTime(io); + var projected = try archunit.projectEdges(allocator, &graph, archunit.identity()); + ended = benchmarkTime(io); + measurements[@intFromEnum(Stage.projection)] = makeMeasurement( + &tracker, + .projection, + started, + ended, + window, + ); + defer projected.deinit(allocator); + try expectEqual(graph.len(), projected.len(), "identity projection edge count"); + } + try expectEqual(0, tracker.active_bytes, "live bytes after staged extraction"); + + { + var files_scope = try archunit.files(allocator, .{ .locator = fixture_root }); + defer files_scope.deinit(); + var source_scope = try files_scope.inPath(&.{.{ .glob = "src/**" }}); + defer source_scope.deinit(); + var should = try source_scope.should(); + defer should.deinit(); + var rule = try should.haveName(.{ .glob = "*.zig" }); + defer rule.deinit(allocator); + + var options = archunit.CheckOptions.init(allocator, io); + options.clear_cache = true; + var window = tracker.beginWindow(); + var started = benchmarkTime(io); + var first_violations = try rule.check(options); + var ended = benchmarkTime(io); + measurements[@intFromEnum(Stage.first_check)] = makeMeasurement( + &tracker, + .first_check, + started, + ended, + window, + ); + const first_check_passed = first_violations.passes(); + first_violations.deinit(allocator); + if (!first_check_passed) return error.BenchmarkRuleViolation; + + options.clear_cache = false; + window = tracker.beginWindow(); + started = benchmarkTime(io); + for (0..cached_iterations) |_| { + var violations = try rule.check(options); + const passed = violations.passes(); + violations.deinit(allocator); + if (!passed) return error.BenchmarkRuleViolation; + } + ended = benchmarkTime(io); + measurements[@intFromEnum(Stage.cached_checks)] = makeMeasurement( + &tracker, + .cached_checks, + started, + ended, + window, + ); + } + try expectEqual(0, tracker.active_bytes, "live bytes after fluent checks"); + + var total_duration_ns: u64 = 0; + for (measurements) |measurement| total_duration_ns += measurement.duration_ns; + return .{ + .enumerated_files = expected_source_files, + .zig_files = zig_file_count, + .graph_edges = expected_graph_edges, + .projected_edges = expected_graph_edges, + .stable_graph_digest = owned_digest.?, + .total_duration_ns = total_duration_ns, + .overall_peak_live_bytes = tracker.peak_live_bytes, + .final_live_bytes = tracker.active_bytes, + .measurements = measurements, + .budgets = budgets, + .budgets_enforced = budgets_enforced, + }; +} + +fn generateFixture(allocator: Allocator, io: Io) !void { + try Io.Dir.cwd().createDirPath(io, fixture_source_dir); + try Io.Dir.cwd().writeFile(io, .{ + .sub_path = fixture_root ++ "/build.zig.zon", + .data = + \\.{ + \\ .name = .archunit_benchmark_fixture, + \\ .version = "0.0.0", + \\ .minimum_zig_version = "0.16.0", + \\ .fingerprint = 0x8a572a6e6f5a965a, + \\ .paths = .{ "build.zig", "src" }, + \\} + \\ + , + }); + try Io.Dir.cwd().writeFile(io, .{ + .sub_path = fixture_root ++ "/build.zig", + .data = + \\const std = @import("std"); + \\pub fn build(b: *std.Build) void { _ = b; } + \\ + , + }); + + for (0..fixture_file_count) |file_index| { + var output: Io.Writer.Allocating = .init(allocator); + defer output.deinit(); + try output.writer.writeAll("const std = @import(\"std\");\n"); + for (0..imports_per_file) |import_index| { + const target_index = (file_index + import_index + 1) % fixture_file_count; + try output.writer.print( + "const dep_{d:0>2} = @import(\"file_{d:0>3}.zig\");\n", + .{ import_index, target_index }, + ); + } + try output.writer.print("pub const id: usize = {d};\n", .{file_index}); + try output.writer.writeAll("pub fn touch() usize {\n return @sizeOf(std.mem.Allocator)"); + for (0..imports_per_file) |import_index| { + try output.writer.print(" + dep_{d:0>2}.id", .{import_index}); + } + try output.writer.writeAll(";\n}\n"); + const contents = try output.toOwnedSlice(); + defer allocator.free(contents); + const source_path = try std.fmt.allocPrint( + allocator, + fixture_source_dir ++ "/file_{d:0>3}.zig", + .{file_index}, + ); + defer allocator.free(source_path); + try Io.Dir.cwd().writeFile(io, .{ .sub_path = source_path, .data = contents }); + } +} + +fn stableGraphDigest(graph: *const archunit.Graph) [32]u8 { + var hash = std.crypto.hash.sha2.Sha256.init(.{}); + for (graph.items()) |edge| { + hashLengthPrefixed(&hash, edge.source); + hashLengthPrefixed(&hash, edge.target); + hash.update(&.{@intFromBool(edge.external)}); + inline for (std.meta.fields(archunit.ImportKind)) |field| { + const value: archunit.ImportKind = @enumFromInt(field.value); + hash.update(&.{@intFromBool(edge.import_kinds.contains(value))}); + } + inline for (std.meta.fields(archunit.TargetClass)) |field| { + const value: archunit.TargetClass = @enumFromInt(field.value); + hash.update(&.{@intFromBool(edge.target_classes.contains(value))}); + } + inline for (std.meta.fields(archunit.TargetAvailability)) |field| { + const value: archunit.TargetAvailability = @enumFromInt(field.value); + hash.update(&.{@intFromBool(edge.target_availabilities.contains(value))}); + } + for (edge.locations.items) |location| { + hashInteger(&hash, location.byte_offset); + hashInteger(&hash, location.line); + hashInteger(&hash, location.column); + } + hash.update(&.{0xff}); + } + var digest: [32]u8 = undefined; + hash.final(&digest); + return digest; +} + +fn hashLengthPrefixed(hash: anytype, value: []const u8) void { + hashInteger(hash, value.len); + hash.update(value); +} + +fn hashInteger(hash: anytype, value: anytype) void { + var encoded: [8]u8 = undefined; + std.mem.writeInt(u64, &encoded, @intCast(value), .little); + hash.update(&encoded); +} + +fn benchmarkTime(io: Io) i96 { + return Io.Clock.awake.now(io).nanoseconds; +} + +fn makeMeasurement( + tracker: *const TrackingAllocator, + stage: Stage, + started_at: i96, + ended_at: i96, + snapshot: tracking.Snapshot, +) StageMeasurement { + std.debug.assert(ended_at >= started_at); + return .{ + .stage = stage, + .duration_ns = @intCast(ended_at - started_at), + .allocation_calls = tracker.allocation_calls - snapshot.allocation_calls, + .allocated_bytes = tracker.allocated_bytes - snapshot.allocated_bytes, + .peak_live_bytes = tracker.window_peak_live_bytes, + }; +} + +fn stageBudget(budgets: StageBudgets, stage: Stage) u64 { + return switch (stage) { + .enumeration => budgets.enumeration_ns, + .source_loading => budgets.source_loading_ns, + .tokenize_parse => budgets.tokenize_parse_ns, + .resolution_classification => budgets.resolution_classification_ns, + .normalization => budgets.normalization_ns, + .projection => budgets.projection_ns, + .first_check => budgets.first_check_ns, + .cached_checks => budgets.cached_check_per_iteration_ns * cached_iterations, + }; +} + +fn enforceBudgets(result: *const BenchmarkResult) !void { + var failed = false; + for (result.measurements) |measurement| { + const limit = stageBudget(result.budgets.stages, measurement.stage); + if (measurement.duration_ns > limit) { + std.debug.print("budget exceeded: {s} took {d} ns, limit {d} ns\n", .{ + @tagName(measurement.stage), + measurement.duration_ns, + limit, + }); + failed = true; + } + } + if (result.total_duration_ns > result.budgets.max_total_ns) { + std.debug.print("budget exceeded: total took {d} ns, limit {d} ns\n", .{ + result.total_duration_ns, + result.budgets.max_total_ns, + }); + failed = true; + } + if (result.overall_peak_live_bytes > result.budgets.max_peak_live_bytes) { + std.debug.print("budget exceeded: peak live bytes {d}, limit {d}\n", .{ + result.overall_peak_live_bytes, + result.budgets.max_peak_live_bytes, + }); + failed = true; + } + if (failed) return error.PerformanceBudgetExceeded; +} + +fn writeResult(allocator: Allocator, io: Io, result: *const BenchmarkResult) !void { + var output: Io.Writer.Allocating = .init(allocator); + defer output.deinit(); + var json: std.json.Stringify = .{ + .writer = &output.writer, + .options = .{ .whitespace = .indent_2 }, + }; + try json.write(result.*); + try output.writer.writeByte('\n'); + const contents = try output.toOwnedSlice(); + defer allocator.free(contents); + if (std.fs.path.dirname(results_path)) |parent| try Io.Dir.cwd().createDirPath(io, parent); + try Io.Dir.cwd().writeFile(io, .{ .sub_path = results_path, .data = contents }); +} + +fn printResult(io: Io, result: *const BenchmarkResult) !void { + var stdout_buffer: [4096]u8 = undefined; + var stdout_writer = Io.File.stdout().writer(io, &stdout_buffer); + const stdout = &stdout_writer.interface; + try stdout.print( + "ArchUnitZig extraction benchmark: {d} files, {d} graph edges, digest {s}\n", + .{ result.enumerated_files, result.graph_edges, result.stable_graph_digest }, + ); + for (result.measurements) |measurement| { + try stdout.print( + " {s}: {d:.3} ms, {d} allocations, {d} requested bytes, {d} peak live bytes\n", + .{ + @tagName(measurement.stage), + @as(f64, @floatFromInt(measurement.duration_ns)) / std.time.ns_per_ms, + measurement.allocation_calls, + measurement.allocated_bytes, + measurement.peak_live_bytes, + }, + ); + } + try stdout.print( + " total: {d:.3} ms, overall peak {d} bytes; budgets {s}\n result: {s}\n", + .{ + @as(f64, @floatFromInt(result.total_duration_ns)) / std.time.ns_per_ms, + result.overall_peak_live_bytes, + if (result.budgets_enforced) "enforced" else "report-only", + results_path, + }, + ); + try stdout.flush(); +} + +fn expectEqual(expected: usize, actual: usize, subject: []const u8) !void { + if (expected == actual) return; + std.debug.print("benchmark correctness failure: {s}: expected {d}, found {d}\n", .{ + subject, + expected, + actual, + }); + return error.BenchmarkCorrectnessFailure; +} diff --git a/benchmark/tracking_allocator.zig b/benchmark/tracking_allocator.zig new file mode 100644 index 0000000..586fdaf --- /dev/null +++ b/benchmark/tracking_allocator.zig @@ -0,0 +1,116 @@ +const std = @import("std"); + +const Allocator = std.mem.Allocator; + +pub const Snapshot = struct { + allocation_calls: usize, + allocated_bytes: usize, +}; + +pub const TrackingAllocator = struct { + child: Allocator, + active_bytes: usize = 0, + peak_live_bytes: usize = 0, + window_peak_live_bytes: usize = 0, + allocation_calls: usize = 0, + allocated_bytes: usize = 0, + + const vtable: Allocator.VTable = .{ + .alloc = alloc, + .resize = resize, + .remap = remap, + .free = free, + }; + + pub fn allocator(self: *TrackingAllocator) Allocator { + return .{ .ptr = self, .vtable = &vtable }; + } + + pub fn beginWindow(self: *TrackingAllocator) Snapshot { + self.window_peak_live_bytes = self.active_bytes; + return .{ + .allocation_calls = self.allocation_calls, + .allocated_bytes = self.allocated_bytes, + }; + } + + fn recordGrowth(self: *TrackingAllocator, amount: usize) void { + self.active_bytes += amount; + self.allocated_bytes += amount; + self.peak_live_bytes = @max(self.peak_live_bytes, self.active_bytes); + self.window_peak_live_bytes = @max(self.window_peak_live_bytes, self.active_bytes); + } + + fn recordShrink(self: *TrackingAllocator, amount: usize) void { + std.debug.assert(self.active_bytes >= amount); + self.active_bytes -= amount; + } + + fn alloc(context: *anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[*]u8 { + const self: *TrackingAllocator = @ptrCast(@alignCast(context)); + const memory = self.child.rawAlloc(len, alignment, ret_addr) orelse return null; + self.allocation_calls += 1; + self.recordGrowth(len); + return memory; + } + + fn resize( + context: *anyopaque, + memory: []u8, + alignment: std.mem.Alignment, + new_len: usize, + ret_addr: usize, + ) bool { + const self: *TrackingAllocator = @ptrCast(@alignCast(context)); + if (!self.child.rawResize(memory, alignment, new_len, ret_addr)) return false; + self.recordResize(memory.len, new_len); + return true; + } + + fn remap( + context: *anyopaque, + memory: []u8, + alignment: std.mem.Alignment, + new_len: usize, + ret_addr: usize, + ) ?[*]u8 { + const self: *TrackingAllocator = @ptrCast(@alignCast(context)); + const remapped = self.child.rawRemap(memory, alignment, new_len, ret_addr) orelse return null; + self.recordResize(memory.len, new_len); + return remapped; + } + + fn recordResize(self: *TrackingAllocator, old_len: usize, new_len: usize) void { + if (new_len > old_len) { + self.recordGrowth(new_len - old_len); + } else { + self.recordShrink(old_len - new_len); + } + } + + fn free( + context: *anyopaque, + memory: []u8, + alignment: std.mem.Alignment, + ret_addr: usize, + ) void { + const self: *TrackingAllocator = @ptrCast(@alignCast(context)); + self.child.rawFree(memory, alignment, ret_addr); + self.recordShrink(memory.len); + } +}; + +test "tracking allocator records growth and returns to zero live bytes" { + var tracker = TrackingAllocator{ .child = std.testing.allocator }; + const allocator = tracker.allocator(); + const snapshot = tracker.beginWindow(); + { + var values: std.ArrayList(u32) = .empty; + defer values.deinit(allocator); + try values.appendSlice(allocator, &.{ 1, 2, 3, 4, 5 }); + try std.testing.expect(tracker.allocation_calls > snapshot.allocation_calls); + try std.testing.expect(tracker.allocated_bytes > snapshot.allocated_bytes); + try std.testing.expect(tracker.peak_live_bytes >= values.capacity * @sizeOf(u32)); + } + try std.testing.expectEqual(@as(usize, 0), tracker.active_bytes); +} diff --git a/build.zig b/build.zig index bd8094c..bd1b1a2 100644 --- a/build.zig +++ b/build.zig @@ -93,11 +93,56 @@ pub fn build(b: *std.Build) void { const run_readme_tests = b.addRunArtifact(readme_tests); run_readme_tests.setCwd(b.path("test/fixtures/readme-consumer")); + const benchmark_archunit = b.createModule(.{ + .root_source_file = b.path("src/root.zig"), + .target = target, + .optimize = .ReleaseSafe, + }); + const benchmark_regex_dependency = b.dependency("regex", .{ + .target = target, + .optimize = .ReleaseSafe, + }); + benchmark_archunit.addImport("regex", benchmark_regex_dependency.module("regex")); + + const benchmark_module = b.createModule(.{ + .root_source_file = b.path("benchmark/extraction.zig"), + .target = target, + .optimize = .ReleaseSafe, + }); + benchmark_module.addImport("archunit", benchmark_archunit); + const benchmark_executable = b.addExecutable(.{ + .name = "archunit-extraction-benchmark", + .root_module = benchmark_module, + }); + const run_benchmark = b.addRunArtifact(benchmark_executable); + run_benchmark.setCwd(b.path(".")); + if (b.args) |args| run_benchmark.addArgs(args); + + const benchmark_contracts = b.createModule(.{ + .root_source_file = b.path("benchmark/contracts.zig"), + .target = target, + .optimize = optimize, + }); + const benchmark_tests = b.addTest(.{ .root_module = benchmark_contracts }); + benchmark_tests.step.dependOn(&run_readme_tests.step); + const run_benchmark_tests = b.addRunArtifact(benchmark_tests); + run_benchmark_tests.setCwd(b.path(".")); + + const benchmark_step = b.step("benchmark", "Measure extraction and cached rule performance"); + benchmark_step.dependOn(&run_benchmark.step); + + const benchmark_check_run = b.addRunArtifact(benchmark_executable); + benchmark_check_run.setCwd(b.path(".")); + benchmark_check_run.addArg("--enforce-budgets"); + const benchmark_check_step = b.step("benchmark-check", "Measure and enforce hosted-CI performance budgets"); + benchmark_check_step.dependOn(&benchmark_check_run.step); + const format_check = b.addFmt(.{ .paths = &.{ "build.zig", "build.zig.zon", "src", + "benchmark", "test/acceptance.zig", "test/dogfood.zig", "test/readme.zig", @@ -111,7 +156,7 @@ pub fn build(b: *std.Build) void { .linkage = .static, .root_module = archunit, }); - docs_library.step.dependOn(&run_readme_tests.step); + docs_library.step.dependOn(&run_benchmark_tests.step); const build_docs = b.addSystemCommand(&.{ python, @@ -147,5 +192,5 @@ pub fn build(b: *std.Build) void { const test_step = b.step("test", "Run formatting checks and tests"); test_step.dependOn(&format_check.step); - test_step.dependOn(&run_readme_tests.step); + test_step.dependOn(&run_benchmark_tests.step); } diff --git a/build.zig.zon b/build.zig.zon index 9d4cd1f..c3570c2 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -13,6 +13,7 @@ "build.zig", "build.zig.zon", "src", + "benchmark", "test", "docs", "LICENSE", diff --git a/docs/architecture.md b/docs/architecture.md index 43acd1a..7ab8928 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -116,8 +116,9 @@ These boundaries are enforced against the repository by the external dogfood sui `test/dogfood.zig` imports only the public `archunit` package and runs after the external acceptance suite in `zig build test`. Its project locator and working directory are both `.`, exactly as they would be for a consumer checking its own repository. The production corpus deliberately excludes -`build.zig` and `test/**`; the latter is the explicit exclusion for all test fixtures rather than an -accidental consequence of nested-project discovery. The separate negative cycle control enters the +`build.zig`, repository tooling under `benchmark/**`, and `test/**`; the latter is the explicit +exclusion for all test fixtures rather than an accidental consequence of nested-project discovery. +The separate negative cycle control enters the violating acceptance fixture through its own working directory and excludes only its malformed `testdata/**` subtree. diff --git a/docs/decisions.md b/docs/decisions.md index 537e7c2..9607b5e 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -1096,8 +1096,9 @@ violating twin. ### D046 — Dogfood distinguishes production dependencies from test consumers The repository checks its own architecture through a separate test root that imports only the -public `archunit` package. Its production corpus is explicit: `src/**` is analyzed while `build.zig` -and `test/**` are extraction exclusions. The complete external acceptance fixture remains a +public `archunit` package. Its production corpus is explicit: `src/**` is analyzed while `build.zig`, +repository tooling under `benchmark/**`, and `test/**` are extraction exclusions. The complete +external acceptance fixture remains a separate gate, and its violating twin supplies the negative cycle proof. Strict layer assignment and a top-folder custom predicate together ensure that connected and isolated production files must belong to a named architecture area. @@ -1213,3 +1214,37 @@ the same Git reference cancel through the workflow concurrency key. Consequence: toolchain downloads are reproducible and tamper-evident, allocator and path behavior are exercised on the three supported runner families, documentation cannot deploy ahead of code quality, and no optional matrix leg or `continue-on-error` can turn a red supported platform into a green PR. + +### D050 — Performance gates use deterministic hosted-CI evidence + +Performance evidence comes from a generated, versioned project under `.zig-cache/benchmark`, not +from the repository itself or a developer's changing checkout. Its 240 Zig sources sit below eight +folder levels and each import twelve deterministic peers plus `std`. Together with `build.zig` and +`build.zig.zon`, this fixes file, syntax, import, and graph cardinalities. The benchmark pins a digest +over sorted edges, classification sets, import kinds, and locations, then independently normalises a +second graph and requires the same digest. Fixture generation is outside every measured window. + +The benchmark deliberately composes public facade primitives instead of duplicating private +extractor logic. It times enumeration, source loading, tokenize/parse, resolution/classification, +normalisation, and identity projection independently while retaining real outputs between stages. +It then builds one public fluent rule, clears the process cache for its first check, and repeats the +same check twenty times without clearing it. This makes cold extraction and cache-hit behavior +visible without pretending a synthetic parser loop represents the shipped product path. + +A forwarding allocator counts allocation calls, newly requested bytes, and peak live caller-owned +bytes per stage. Every manual pipeline owner is destroyed before fluent checks, every rule owner is +destroyed afterward, and both boundaries require zero tracked live bytes. The process-wide graph +cache intentionally retains its existing page-allocator implementation, so reported peaks describe +the public caller allocator rather than unstable whole-process RSS; cache isolation and page-backed +cache lifetime remain covered by the exhaustive correctness/leak suite. + +`zig build benchmark` always enforces graph counts, the pinned digest, projection cardinality, +passing rule results, deterministic re-normalisation, and zero live tracked bytes, then writes +`zig-out/benchmark/results.json`. `zig build benchmark-check` additionally applies generous runtime +and memory regression budgets from `benchmark/budgets.json`. Runtime is not a public guarantee: the +budget evidence names the uncached `ubuntu-latest` run used to set it, with substantial variance +headroom. CI uploads the raw JSON so future optimisation begins from comparable stage evidence. + +Consequence: correctness and cache isolation cannot be traded for a faster number; performance +regressions are visible by stage; allocations can be profiled without platform-specific tooling; +and budget changes remain auditable against hosted evidence rather than one workstation. diff --git a/scripts/check_ci.py b/scripts/check_ci.py index 5480f2a..27d677c 100644 --- a/scripts/check_ci.py +++ b/scripts/check_ci.py @@ -38,9 +38,14 @@ def main() -> None: "windows-latest", "macos-latest", "zig fmt --check", + "build.zig build.zig.zon src benchmark", "zig build test -Doptimize=Debug", "zig build test -Doptimize=ReleaseSafe", "zig build docs -Doptimize=ReleaseSafe", + "zig build benchmark-check", + "name: performance-results", + "path: zig-out/benchmark/results.json", + "if: always() && hashFiles('zig-out/benchmark/results.json') != ''", "pages: write", "id-token: write", "enablement: true", diff --git a/test/dogfood.zig b/test/dogfood.zig index 29ee5cf..3a72f48 100644 --- a/test/dogfood.zig +++ b/test/dogfood.zig @@ -4,6 +4,7 @@ const archunit = @import("archunit"); const Allocator = std.mem.Allocator; const production_exclusions = [_][]const u8{ "build.zig", + "benchmark/**", "test/**", }; const fixture_exclusions = [_][]const u8{"testdata/**"}; @@ -211,9 +212,10 @@ fn expectReportedPath(rule: anytype, violations: *const archunit.ViolationList, } test "production corpus exclusions are explicit" { - try std.testing.expectEqual(@as(usize, 2), production_exclusions.len); + try std.testing.expectEqual(@as(usize, 3), production_exclusions.len); try std.testing.expectEqualStrings("build.zig", production_exclusions[0]); - try std.testing.expectEqualStrings("test/**", production_exclusions[1]); + try std.testing.expectEqualStrings("benchmark/**", production_exclusions[1]); + try std.testing.expectEqualStrings("test/**", production_exclusions[2]); var diagnostics = archunit.ErrorContext.init(std.testing.allocator); defer diagnostics.deinit();