diff --git a/.github/workflows/correctness.yml b/.github/workflows/correctness.yml new file mode 100644 index 0000000..7ba0178 --- /dev/null +++ b/.github/workflows/correctness.yml @@ -0,0 +1,168 @@ +name: Correctness Tests + +# Verifies cross-language parity between Python and Rust implementations of +# PromQL pattern matching and sketch serialisation. Ephemeral GitHub Actions +# VMs are well-suited for these deterministic correctness checks. + +on: + push: + branches: [ main ] + paths: + - 'asap-common/tests/**' + - 'asap-common/dependencies/**' + - 'asap-common/sketch-core/**' + - '.github/workflows/correctness.yml' + pull_request: + branches: [ main ] + paths: + - 'asap-common/tests/**' + - 'asap-common/dependencies/**' + - 'asap-common/sketch-core/**' + - '.github/workflows/correctness.yml' + workflow_dispatch: + +jobs: + # ── 1. Cross-language token matching (Python vs Rust) ────────────────────── + cross-language-token-matching: + name: Cross-language PromQL token matching + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + if [ -f asap-common/dependencies/py/requirements.txt ]; then + pip install -r asap-common/dependencies/py/requirements.txt + fi + pip install -e asap-common/dependencies/py/promql_utilities/ + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Install protoc + run: | + sudo apt-get update -qq + sudo apt-get install -y protobuf-compiler + + - name: Run sccache + uses: mozilla-actions/sccache-action@v0.0.4 + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-correctness-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} + + - name: Run master test runner (Python + Rust + comparison) + working-directory: asap-common/tests/compare_matched_tokens + run: python utilities/master_test_runner.py + env: + RUSTC_WRAPPER: sccache + + - name: Upload test artefacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: cross-language-token-results + path: | + asap-common/tests/compare_matched_tokens/python_tests/python_test_results.json + asap-common/tests/compare_matched_tokens/rust_tests/rust_test_results.json + asap-common/tests/compare_matched_tokens/comparison_tests/comparison_report.json + asap-common/tests/compare_matched_tokens/test_summary.json + if-no-files-found: warn + + # ── 2. Cross-language serialised pattern comparison ──────────────────────── + cross-language-pattern-serialisation: + name: Cross-language pattern serialisation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + if [ -f asap-common/dependencies/py/requirements.txt ]; then + pip install -r asap-common/dependencies/py/requirements.txt + fi + pip install -e asap-common/dependencies/py/promql_utilities/ + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Install protoc + run: | + sudo apt-get update -qq + sudo apt-get install -y protobuf-compiler + + - name: Run sccache + uses: mozilla-actions/sccache-action@v0.0.4 + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-correctness-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} + + - name: Generate Python patterns + working-directory: asap-common/tests/compare_patterns + run: python python_generate_patterns.py + + - name: Build and run Rust pattern generator + working-directory: asap-common/tests/compare_patterns + run: cargo run --release + env: + RUSTC_WRAPPER: sccache + + - name: Compare serialised patterns + working-directory: asap-common/tests/compare_patterns + run: python compare_serialized_patterns.py + + # ── 3. Rust pattern-matching unit tests ──────────────────────────────────── + rust-pattern-matching: + name: Rust pattern matching unit tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Install protoc + run: | + sudo apt-get update -qq + sudo apt-get install -y protobuf-compiler + + - name: Run sccache + uses: mozilla-actions/sccache-action@v0.0.4 + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-correctness-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} + + - name: Run Rust pattern matching tests + working-directory: asap-common/tests/rust_pattern_matching + run: cargo test --release + env: + RUSTC_WRAPPER: sccache diff --git a/asap-common/.gitignore b/asap-common/.gitignore index 9e7080c..102b6ea 100644 --- a/asap-common/.gitignore +++ b/asap-common/.gitignore @@ -8,4 +8,5 @@ dependencies/py/promql_utilities/promql_utilities.egg-info/ dependencies/rs/**/target/ tests/**/*.json +!tests/**/test_data/*.json tests/**/target/ diff --git a/asap-common/tests/compare_matched_tokens/rust_tests/Cargo.toml b/asap-common/tests/compare_matched_tokens/rust_tests/Cargo.toml index e38e483..c4155dc 100644 --- a/asap-common/tests/compare_matched_tokens/rust_tests/Cargo.toml +++ b/asap-common/tests/compare_matched_tokens/rust_tests/Cargo.toml @@ -1,3 +1,5 @@ +[workspace] + [package] name = "promql_cross_lang_tests" version = "0.1.0" diff --git a/asap-common/tests/compare_matched_tokens/rust_tests/src/pattern_tests.rs b/asap-common/tests/compare_matched_tokens/rust_tests/src/pattern_tests.rs index fcc210f..6b0fef9 100644 --- a/asap-common/tests/compare_matched_tokens/rust_tests/src/pattern_tests.rs +++ b/asap-common/tests/compare_matched_tokens/rust_tests/src/pattern_tests.rs @@ -19,22 +19,11 @@ impl PatternTester { // Rate pattern PromQLPattern::new( Self::build_rate_pattern(), - vec![ - "metric".to_string(), - "function".to_string(), - "range_vector".to_string(), - ], // Some("ONLY_TEMPORAL".to_string()), ), // Quantile over time pattern PromQLPattern::new( Self::build_quantile_over_time_pattern(), - vec![ - "metric".to_string(), - "function".to_string(), - "range_vector".to_string(), - "function_args".to_string(), - ], // Some("ONLY_TEMPORAL".to_string()), ), ]; @@ -44,34 +33,31 @@ impl PatternTester { // Sum aggregation pattern PromQLPattern::new( Self::build_sum_pattern(), - vec!["metric".to_string(), "aggregation".to_string()], // Some("ONLY_SPATIAL".to_string()), ), // Simple metric pattern PromQLPattern::new( Self::build_metric_pattern(), - vec!["metric".to_string()], // Some("ONLY_SPATIAL".to_string()), ), ]; // ONE_TEMPORAL_ONE_SPATIAL patterns let combined_patterns = vec![ - // Sum of rate pattern + // Aggregation of single-arg temporal functions PromQLPattern::new( Self::build_one_temporal_one_spatial_pattern(), - vec![ - "metric".to_string(), - "function".to_string(), - "aggregation".to_string(), - "range_vector".to_string(), - ], + // Some("ONE_TEMPORAL_ONE_SPATIAL".to_string()), + ), + // Aggregation of quantile_over_time (2-arg) + PromQLPattern::new( + Self::build_combined_quantile_pattern(), // Some("ONE_TEMPORAL_ONE_SPATIAL".to_string()), ), ]; // Insert in order from simple to complex to avoid panics - patterns.insert("ONLY_VECTOR".to_string(), spatial_patterns.clone()); + // ONLY_VECTOR is derived via disambiguation in test_query, not a separate entry patterns.insert("ONLY_SPATIAL".to_string(), spatial_patterns); patterns.insert("ONLY_TEMPORAL".to_string(), temporal_patterns); patterns.insert("ONE_TEMPORAL_ONE_SPATIAL".to_string(), combined_patterns); @@ -280,7 +266,20 @@ impl PatternTester { let args: Vec>> = vec![ms]; - PromQLPatternBuilder::function(vec!["rate", "increase"], args, Some("function"), None) + PromQLPatternBuilder::function( + vec![ + "rate", + "increase", + "avg_over_time", + "sum_over_time", + "count_over_time", + "min_over_time", + "max_over_time", + ], + args, + Some("function"), + None, + ) } fn build_quantile_over_time_pattern() -> Option> { @@ -327,7 +326,6 @@ impl PatternTester { let func = PromQLPatternBuilder::function( vec![ - "quantile_over_time", "sum_over_time", "count_over_time", "avg_over_time", @@ -351,6 +349,30 @@ impl PatternTester { ) } + fn build_combined_quantile_pattern() -> Option> { + let num = PromQLPatternBuilder::number(None, None); + let ms = PromQLPatternBuilder::matrix_selector( + PromQLPatternBuilder::metric(None, None, None, Some("metric")), + None, + Some("range_vector"), + ); + let func_args: Vec>> = vec![num, ms]; + let func = PromQLPatternBuilder::function( + vec!["quantile_over_time"], + func_args, + Some("function"), + None, + ); + PromQLPatternBuilder::aggregation( + vec!["sum", "count", "avg", "quantile", "min", "max"], + func, + None, + None, + None, + Some("aggregation"), + ) + } + fn build_sum_rate_pattern() -> Option> { let ms = PromQLPatternBuilder::matrix_selector( PromQLPatternBuilder::metric(None, None, None, Some("metric")), diff --git a/asap-common/tests/compare_matched_tokens/test_data/promql_queries.json b/asap-common/tests/compare_matched_tokens/test_data/promql_queries.json new file mode 100644 index 0000000..0d03af4 --- /dev/null +++ b/asap-common/tests/compare_matched_tokens/test_data/promql_queries.json @@ -0,0 +1,228 @@ +{ + "test_cases": [ + { + "id": "temporal_rate_basic", + "description": "Basic rate function over a time range", + "query": "rate(http_requests_total{job=\"api\"}[5m])", + "expected_pattern_type": "ONLY_TEMPORAL", + "expected_tokens": { + "metric": { + "name": "http_requests_total", + "labels": {"job": "api"}, + "at_modifier": null + }, + "function": { + "name": "rate" + }, + "range_vector": { + "range": "5m" + } + } + }, + { + "id": "temporal_increase_basic", + "description": "Increase function over a time range", + "query": "increase(http_requests_total[1h])", + "expected_pattern_type": "ONLY_TEMPORAL", + "expected_tokens": { + "metric": { + "name": "http_requests_total", + "labels": {}, + "at_modifier": null + }, + "function": { + "name": "increase" + }, + "range_vector": { + "range": "1h" + } + } + }, + { + "id": "temporal_quantile_over_time", + "description": "Quantile over time function", + "query": "quantile_over_time(0.95, cpu_usage{instance=\"host1\"}[10m])", + "expected_pattern_type": "ONLY_TEMPORAL", + "expected_tokens": { + "metric": { + "name": "cpu_usage", + "labels": {"instance": "host1"}, + "at_modifier": null + }, + "function": { + "name": "quantile_over_time" + }, + "range_vector": { + "range": "10m" + } + } + }, + { + "id": "temporal_avg_over_time", + "description": "Average over time function", + "query": "avg_over_time(memory_bytes[30m])", + "expected_pattern_type": "ONLY_TEMPORAL", + "expected_tokens": { + "metric": { + "name": "memory_bytes", + "labels": {}, + "at_modifier": null + }, + "function": { + "name": "avg_over_time" + }, + "range_vector": { + "range": "30m" + } + } + }, + { + "id": "spatial_sum_aggregation", + "description": "Sum aggregation across all series", + "query": "sum(http_requests_total{job=\"api\"})", + "expected_pattern_type": "ONLY_SPATIAL", + "expected_tokens": { + "metric": { + "name": "http_requests_total", + "labels": {"job": "api"}, + "at_modifier": null + }, + "aggregation": { + "op": "sum", + "modifier": null + } + } + }, + { + "id": "spatial_avg_aggregation", + "description": "Average aggregation by label", + "query": "avg by (instance) (cpu_usage)", + "expected_pattern_type": "ONLY_SPATIAL", + "expected_tokens": { + "metric": { + "name": "cpu_usage", + "labels": {}, + "at_modifier": null + }, + "aggregation": { + "op": "avg", + "modifier": null + } + } + }, + { + "id": "spatial_count_aggregation", + "description": "Count aggregation", + "query": "count(up{job=\"node\"})", + "expected_pattern_type": "ONLY_SPATIAL", + "expected_tokens": { + "metric": { + "name": "up", + "labels": {"job": "node"}, + "at_modifier": null + }, + "aggregation": { + "op": "count", + "modifier": null + } + } + }, + { + "id": "combined_sum_of_rate", + "description": "Sum aggregation of rate (temporal + spatial)", + "query": "sum(rate(http_requests_total{job=\"api\"}[5m]))", + "expected_pattern_type": "ONE_TEMPORAL_ONE_SPATIAL", + "expected_tokens": { + "metric": { + "name": "http_requests_total", + "labels": {"job": "api"}, + "at_modifier": null + }, + "function": { + "name": "rate" + }, + "aggregation": { + "op": "sum", + "modifier": null + }, + "range_vector": { + "range": "5m" + } + } + }, + { + "id": "combined_avg_of_quantile_over_time", + "description": "Avg aggregation of quantile_over_time (temporal + spatial)", + "query": "avg(quantile_over_time(0.99, response_time_seconds[15m]))", + "expected_pattern_type": "ONE_TEMPORAL_ONE_SPATIAL", + "expected_tokens": { + "metric": { + "name": "response_time_seconds", + "labels": {}, + "at_modifier": null + }, + "function": { + "name": "quantile_over_time" + }, + "aggregation": { + "op": "avg", + "modifier": null + }, + "range_vector": { + "range": "15m" + } + } + }, + { + "id": "combined_sum_of_avg_over_time", + "description": "Sum aggregation of avg_over_time", + "query": "sum by (job) (avg_over_time(memory_bytes{env=\"prod\"}[1h]))", + "expected_pattern_type": "ONE_TEMPORAL_ONE_SPATIAL", + "expected_tokens": { + "metric": { + "name": "memory_bytes", + "labels": {"env": "prod"}, + "at_modifier": null + }, + "function": { + "name": "avg_over_time" + }, + "aggregation": { + "op": "sum", + "modifier": null + }, + "range_vector": { + "range": "1h" + } + } + } + ], + "pattern_builder_tests": [ + { + "id": "builder_metric_no_labels", + "description": "Build a simple metric selector with no labels", + "builder_call": "metric", + "parameters": { + "collect_as": "metric" + }, + "expected_pattern": { + "type": "metric", + "collect_as": "metric" + } + }, + { + "id": "builder_function_rate", + "description": "Build a rate function pattern", + "builder_call": "function", + "parameters": { + "names": ["rate", "increase"], + "collect_as": "function" + }, + "expected_pattern": { + "type": "function", + "names": ["rate", "increase"], + "collect_as": "function" + } + } + ] +} diff --git a/asap-common/tests/compare_patterns/Cargo.toml b/asap-common/tests/compare_patterns/Cargo.toml index 8d04a6f..3f94a69 100644 --- a/asap-common/tests/compare_patterns/Cargo.toml +++ b/asap-common/tests/compare_patterns/Cargo.toml @@ -1,3 +1,5 @@ +[workspace] + [package] name = "compare_patterns_runner" version = "0.1.0" diff --git a/asap-common/tests/compare_patterns/src/main.rs b/asap-common/tests/compare_patterns/src/main.rs index 6c429f1..d75fa8b 100644 --- a/asap-common/tests/compare_patterns/src/main.rs +++ b/asap-common/tests/compare_patterns/src/main.rs @@ -25,7 +25,6 @@ fn main() { let pattern_1 = PromQLPatternBuilder::function(vec!["rate", "increase"], func_args1, Some("function"), None); let pattern1 = PromQLPattern::new( pattern_1, - vec!["metric".to_string(), "function".to_string(), "range_vector".to_string()], ); if let Some(ast) = pattern1.ast_pattern { only_temporal_patterns.push(serde_json::Value::Object(ast.into_iter().collect())); @@ -44,7 +43,6 @@ fn main() { let pattern_2 = PromQLPatternBuilder::function(vec!["quantile_over_time"], func_args2, Some("function"), Some("function_args")); let pattern2 = PromQLPattern::new( pattern_2, - vec!["metric".to_string(), "function".to_string(), "range_vector".to_string(), "function_args".to_string()], ); if let Some(ast) = pattern2.ast_pattern { only_temporal_patterns.push(serde_json::Value::Object(ast.into_iter().collect())); @@ -66,7 +64,6 @@ fn main() { ); let pattern3 = PromQLPattern::new( pattern_3, - vec!["metric".to_string(), "aggregation".to_string()], ); if let Some(ast) = pattern3.ast_pattern { only_spatial_patterns.push(serde_json::Value::Object(ast.into_iter().collect())); @@ -76,7 +73,6 @@ fn main() { let pattern_4 = PromQLPatternBuilder::metric(None, None, None, Some("metric")); let pattern4 = PromQLPattern::new( pattern_4, - vec!["metric".to_string()], ); if let Some(ast) = pattern4.ast_pattern { only_spatial_patterns.push(serde_json::Value::Object(ast.into_iter().collect())); @@ -108,7 +104,6 @@ fn main() { ); let pattern5 = PromQLPattern::new( pattern_5, - vec!["metric".to_string(), "range_vector".to_string(), "function".to_string(), "function_args".to_string(), "aggregation".to_string()], ); if let Some(ast) = pattern5.ast_pattern { one_temporal_one_spatial_patterns.push(serde_json::Value::Object(ast.into_iter().collect())); @@ -137,7 +132,6 @@ fn main() { ); let pattern6 = PromQLPattern::new( pattern_6, - vec!["metric".to_string(), "range_vector".to_string(), "function".to_string(), "aggregation".to_string()], ); if let Some(ast) = pattern6.ast_pattern { one_temporal_one_spatial_patterns.push(serde_json::Value::Object(ast.into_iter().collect())); diff --git a/asap-common/tests/rust_pattern_matching/Cargo.toml b/asap-common/tests/rust_pattern_matching/Cargo.toml index 14b2980..571641e 100644 --- a/asap-common/tests/rust_pattern_matching/Cargo.toml +++ b/asap-common/tests/rust_pattern_matching/Cargo.toml @@ -1,3 +1,5 @@ +[workspace] + [package] name = "promql_cross_lang_tests" version = "0.1.0" diff --git a/asap-common/tests/rust_pattern_matching/src/main.rs b/asap-common/tests/rust_pattern_matching/src/main.rs index 3a540af..4a29352 100644 --- a/asap-common/tests/rust_pattern_matching/src/main.rs +++ b/asap-common/tests/rust_pattern_matching/src/main.rs @@ -10,11 +10,6 @@ fn temporal_pattern( ) -> PromQLPattern { PromQLPattern::new( blocks[pattern_type].clone(), - vec![ - "metric".to_string(), - "function".to_string(), - "range_vector".to_string(), - ], ) } @@ -24,7 +19,6 @@ fn spatial_pattern( ) -> PromQLPattern { PromQLPattern::new( blocks[pattern_type].clone(), - vec!["metric".to_string(), "aggregation".to_string()], ) } @@ -39,12 +33,6 @@ fn spatial_of_temporal_pattern(temporal_block: &Option>) ); PromQLPattern::new( pattern, - vec![ - "metric".to_string(), - "function".to_string(), - "range_vector".to_string(), - "aggregation".to_string(), - ], ) }