diff --git a/.github/workflows/sdk-compliance.yml b/.github/workflows/sdk-compliance.yml index ba4a057..a89e0a5 100644 --- a/.github/workflows/sdk-compliance.yml +++ b/.github/workflows/sdk-compliance.yml @@ -18,4 +18,24 @@ jobs: with: adapter-dockerfile: "sdk_compliance_adapter/Dockerfile" adapter-context: "." - test-harness-version: "0.10.0" + test-harness-version: "1.0.0" + report-name: "sdk-compliance-report-dotnet-v0" + + coverage-inventory: + name: Verify compliance test inventory + needs: compliance + if: always() + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Test coverage checks + run: python3 -m unittest discover -s sdk_compliance_adapter -p 'test_*.py' + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: sdk-compliance-report-dotnet-v0 + path: report + - name: Verify all 47 tests were selected + run: python3 sdk_compliance_adapter/check_coverage.py --report report/sdk-compliance-report.md diff --git a/sdk_compliance_adapter/CONTRIBUTING.md b/sdk_compliance_adapter/CONTRIBUTING.md index 5721bb7..3ed9876 100644 --- a/sdk_compliance_adapter/CONTRIBUTING.md +++ b/sdk_compliance_adapter/CONTRIBUTING.md @@ -4,7 +4,36 @@ This package contains the PostHog .NET SDK compliance adapter used with the Post ## Running tests -Tests run automatically in CI via GitHub Actions. +Tests run automatically in CI via GitHub Actions using harness 1.0.0. The adapter +references the core SDK project and calls `PostHogClient.Capture`, `FlushAsync`, +and `GetFeatureFlagAsync`; the SDK owns serialization, compression, retries, +flag results, and feature-flag-called events. The adapter runs on .NET 9 and uses +the SDK's .NET 8 target. + +Health advertises `capture_v0` and `encoding_gzip`. Normal server-wire discovery +selects 30 V0 capture tests (including UTC timestamp overrides and gzip) and 17 +feature flag tests. `expected-tests.txt` records the full 47-test inventory. +V1, dedicated AI capture, and non-gzip codecs are not supported by this profile. +Feature flags exercise the existing single-key public getter with remote calls +and the adapter's configured options, not every SDK overload or default. + +CI keeps SDK assertion failures advisory, but verifies the report contains the +complete inventory. Missing reports, empty runs, and capture discovery regressions +fail the inventory check. + +### Focused coverage checks + +These checks use Python 3's standard library: + +```bash +python3 -m unittest discover -s sdk_compliance_adapter -p 'test_*.py' +python3 sdk_compliance_adapter/check_coverage.py --health-url http://localhost:8080/health +python3 sdk_compliance_adapter/check_coverage.py --report sdk-compliance-report.md +``` + +The health check requires a running adapter. The report check accepts the Markdown +report emitted by the pinned reusable workflow and checks selection independently +of assertion outcomes. ### Locally with Docker Compose @@ -35,7 +64,7 @@ docker run -d --name sdk-adapter --network test-network -p 8080:8080 posthog-dot docker run --rm \ --name test-harness \ --network test-network \ - ghcr.io/posthog/sdk-test-harness:0.10.0 \ + ghcr.io/posthog/sdk-test-harness:1.0.0 \ run --adapter-url http://sdk-adapter:8080 --mock-url http://test-harness:8081 # Cleanup diff --git a/sdk_compliance_adapter/Program.cs b/sdk_compliance_adapter/Program.cs index 1167bf1..97ebd32 100644 --- a/sdk_compliance_adapter/Program.cs +++ b/sdk_compliance_adapter/Program.cs @@ -24,7 +24,8 @@ app.MapGet("/health", () => new HealthResponse( SdkName: "posthog-dotnet", SdkVersion: VersionConstants.Version, - AdapterVersion: "1.0.0" + AdapterVersion: "1.0.0", + Capabilities: ["capture_v0", "encoding_gzip"] )); app.MapPost("/init", async (InitRequest request) => @@ -246,7 +247,8 @@ record HealthResponse( [property: JsonPropertyName("sdk_name")] string SdkName, [property: JsonPropertyName("sdk_version")] string SdkVersion, - [property: JsonPropertyName("adapter_version")] string AdapterVersion + [property: JsonPropertyName("adapter_version")] string AdapterVersion, + [property: JsonPropertyName("capabilities")] string[] Capabilities ); record InitRequest( diff --git a/sdk_compliance_adapter/check_coverage.py b/sdk_compliance_adapter/check_coverage.py new file mode 100644 index 0000000..661b891 --- /dev/null +++ b/sdk_compliance_adapter/check_coverage.py @@ -0,0 +1,58 @@ +"""Check health discovery and the test inventory of harness 1.0.0 reports.""" + +import argparse +from collections import Counter +import json +from pathlib import Path +import re +from urllib.request import urlopen + + +EXPECTED_TESTS = Path(__file__).with_name("expected-tests.txt").read_text().splitlines() + + +def check_health(health): + if health.get("sdk_name") != "posthog-dotnet": + raise ValueError("Expected the posthog-dotnet adapter") + if set(health.get("capabilities", [])) != {"capture_v0", "encoding_gzip"}: + raise ValueError("Expected capture_v0 and encoding_gzip capabilities") + + +def check_report(report): + # The reusable workflow emits Markdown, including every passed and failed test. + # Check selection only: SDK assertion failures remain advisory in CI. + if not report.startswith("# posthog-dotnet Compliance Report\n"): + raise ValueError("Expected a posthog-dotnet compliance report") + actual = [] + suite = None + for line in report.splitlines(): + heading = re.fullmatch(r"## (\w+) Tests", line) + if heading: + suite = heading[1].lower() + row = re.fullmatch(r"\| (.+) \| [✅❌] \| \d+ms \|", line) + if row: + actual.append(f"{suite}.{row[1].lower().replace(' ', '_')}") + if not EXPECTED_TESTS or Counter(actual) != Counter(EXPECTED_TESTS): + missing = sorted((Counter(EXPECTED_TESTS) - Counter(actual)).elements()) + unexpected = sorted((Counter(actual) - Counter(EXPECTED_TESTS)).elements()) + raise ValueError(f"Incorrect test inventory: missing={missing}, unexpected={unexpected}") + print("Inventory verified: 30 capture + 17 feature_flags = 47 tests") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--health-url", help="Running adapter's /health URL") + parser.add_argument("--report", type=Path, help="Harness Markdown report") + args = parser.parse_args() + if not args.health_url and not args.report: + parser.error("provide --health-url or --report") + if args.health_url: + with urlopen(args.health_url, timeout=10) as response: + check_health(json.load(response)) + print("Health capabilities verified") + if args.report: + check_report(args.report.read_text()) + + +if __name__ == "__main__": + main() diff --git a/sdk_compliance_adapter/docker-compose.yml b/sdk_compliance_adapter/docker-compose.yml index 160da03..c13d99f 100644 --- a/sdk_compliance_adapter/docker-compose.yml +++ b/sdk_compliance_adapter/docker-compose.yml @@ -9,7 +9,7 @@ services: # Test harness test-harness: - image: ghcr.io/posthog/sdk-test-harness:0.10.0 + image: ghcr.io/posthog/sdk-test-harness:1.0.0 command: ["run", "--adapter-url", "http://sdk-adapter:8080", "--mock-url", "http://test-harness:8081"] networks: - test-network diff --git a/sdk_compliance_adapter/expected-tests.txt b/sdk_compliance_adapter/expected-tests.txt new file mode 100644 index 0000000..f74aa16 --- /dev/null +++ b/sdk_compliance_adapter/expected-tests.txt @@ -0,0 +1,47 @@ +capture.format_validation.event_has_required_fields +capture.format_validation.event_has_uuid +capture.format_validation.event_has_lib_properties +capture.format_validation.distinct_id_is_string +capture.format_validation.token_is_present +capture.format_validation.custom_properties_preserved +capture.format_validation.event_has_timestamp +capture.format_validation.non_utc_event_timestamp_is_converted_to_utc +capture.retry_behavior.retries_on_503 +capture.retry_behavior.does_not_retry_on_400 +capture.retry_behavior.does_not_retry_on_401 +capture.retry_behavior.respects_retry_after_header +capture.retry_behavior.implements_backoff +capture.retry_behavior.retries_on_500 +capture.retry_behavior.retries_on_502 +capture.retry_behavior.retries_on_504 +capture.retry_behavior.max_retries_respected +capture.deduplication.generates_unique_uuids +capture.deduplication.preserves_uuid_on_retry +capture.deduplication.preserves_uuid_and_timestamp_on_retry +capture.deduplication.preserves_uuid_and_timestamp_on_batch_retry +capture.deduplication.no_duplicate_events_in_batch +capture.deduplication.different_events_have_different_uuids +capture.compression.sends_gzip_when_enabled +capture.batch_format.uses_proper_batch_structure +capture.batch_format.flush_with_no_events_sends_nothing +capture.batch_format.multiple_events_batched_together +capture.error_handling.does_not_retry_on_403 +capture.error_handling.does_not_retry_on_413 +capture.error_handling.retries_on_408 +feature_flags.request_payload.request_with_person_properties_device_id +feature_flags.request_payload.flags_request_uses_v2_query_param +feature_flags.request_payload.flags_request_hits_flags_path_not_decide +feature_flags.request_payload.flags_request_omits_authorization_header +feature_flags.request_payload.token_in_flags_body_matches_init +feature_flags.request_payload.groups_round_trip +feature_flags.request_payload.groups_default_to_empty_object +feature_flags.request_payload.disable_geoip_false_propagates_as_geoip_disable_false +feature_flags.request_payload.disable_geoip_omitted_defaults_to_false +feature_flags.request_payload.flag_keys_to_evaluate_contains_only_requested_key +feature_flags.request_lifecycle.no_flags_request_on_init_alone +feature_flags.request_lifecycle.no_flags_request_on_normal_capture +feature_flags.request_lifecycle.two_flag_calls_produce_two_remote_requests +feature_flags.request_lifecycle.mock_response_value_is_returned_to_caller +feature_flags.retry_behavior.retries_flags_on_502 +feature_flags.retry_behavior.retries_flags_on_504 +feature_flags.side_effect_events.get_feature_flag_captures_feature_flag_called_event diff --git a/sdk_compliance_adapter/test_check_coverage.py b/sdk_compliance_adapter/test_check_coverage.py new file mode 100644 index 0000000..285ec69 --- /dev/null +++ b/sdk_compliance_adapter/test_check_coverage.py @@ -0,0 +1,51 @@ +import unittest + +from check_coverage import EXPECTED_TESTS, check_health, check_report + + +def report_for(tests, status="✅"): + lines = ["# posthog-dotnet Compliance Report"] + for test in tests: + suite, name = test.split(".", 1) + lines.extend([ + f"## {suite.title()} Tests", + f"| {name.replace('_', ' ').title()} | {status} | 1ms |", + ]) + return "\n".join(lines) + "\n" + + +class CoverageChecksTests(unittest.TestCase): + def test_health_advertises_capture_and_gzip(self): + check_health({"sdk_name": "posthog-dotnet", "capabilities": ["capture_v0", "encoding_gzip"]}) + + def test_health_without_capabilities_is_rejected(self): + with self.assertRaises(ValueError): + check_health({"sdk_name": "posthog-dotnet"}) + + def test_current_inventory(self): + self.assertEqual(len(EXPECTED_TESTS), 47) + self.assertEqual(sum(test.startswith("capture.") for test in EXPECTED_TESTS), 30) + check_report(report_for(EXPECTED_TESTS)) + + def test_flags_only_report_is_rejected(self): + with self.assertRaises(ValueError): + check_report(report_for([test for test in EXPECTED_TESTS if test.startswith("feature_flags.")])) + + def test_empty_report_is_rejected(self): + with self.assertRaises(ValueError): + check_report(report_for([])) + + def test_missing_timestamp_case_is_rejected(self): + with self.assertRaises(ValueError): + check_report(report_for([test for test in EXPECTED_TESTS if "non_utc" not in test])) + + def test_duplicate_cannot_replace_missing_case(self): + with self.assertRaises(ValueError): + check_report(report_for(EXPECTED_TESTS[:-1] + [EXPECTED_TESTS[0]])) + + def test_assertion_failures_do_not_change_inventory(self): + check_report(report_for(EXPECTED_TESTS, status="❌")) + + +if __name__ == "__main__": + unittest.main()