diff --git a/CHANGELOG.md b/CHANGELOG.md index f9a15457..b116befa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Go server: Add support for `labeled_boolean` metrics with static labels ([AE-1250](https://mozilla-hub.atlassian.net/browse/AE-1250)) - BUGFIX: Go server: Serialize nil `string_list` metrics as empty arrays instead of null ([#837](https://github.com/mozilla/glean_parser/pull/837)) +- Add support to target `glean_sym` (the new experimental Glean Rust API) ([#841](https://github.com/mozilla/glean_parser/pull/841)) ## 19.0.0 diff --git a/glean_parser/rust_sym.py b/glean_parser/rust_sym.py new file mode 100644 index 00000000..6fa75943 --- /dev/null +++ b/glean_parser/rust_sym.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Outputter to generate Rust code for metrics. +""" + +from pathlib import Path +from typing import Any, Dict, Optional + +from . import __version__ +from . import metrics +from . import pings +from . import util +from .rust import ( + rust_datatypes_filter, + ctor, + type_name, + extra_type_name, + structure_type_name, + extra_keys, + Category, +) + + +def output_rust( + objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None +) -> None: + """ + Given a tree of objects, output Rust code to `output_dir`. + + :param objs: A tree of objects (metrics and pings) as returned from + `parser.parse_objects`. + :param output_dir: Path to an output directory to write to. + :param options: options dictionary, not currently used for Rust + """ + + if options is None: + options = {} + + template = util.get_jinja2_template( + "rust_sym.jinja2", + filters=( + ("rust", rust_datatypes_filter), + ("snake_case", util.snake_case), + ("camelize", util.camelize), + ("type_name", type_name), + ("extra_type_name", extra_type_name), + ("structure_type_name", structure_type_name), + ("ctor", ctor), + ("extra_keys", extra_keys), + ), + ) + + filename = "glean_metrics.rs" + filepath = output_dir / filename + categories = [] + + for category_key, category_val in objs.items(): + contains_pings = any( + isinstance(obj, pings.Ping) for obj in category_val.values() + ) + + cat = Category(category_key, category_val, contains_pings) + categories.append(cat) + + with filepath.open("w", encoding="utf-8") as fd: + fd.write( + template.render( + parser_version=__version__, + categories=categories, + extra_metric_args=util.extra_metric_args, + common_metric_args=util.common_metric_args, + ) + ) diff --git a/glean_parser/templates/rust_sym.jinja2 b/glean_parser/templates/rust_sym.jinja2 new file mode 100644 index 00000000..44676c0b --- /dev/null +++ b/glean_parser/templates/rust_sym.jinja2 @@ -0,0 +1,196 @@ +// -*- mode: Rust -*- + +// AUTOGENERATED BY glean_parser v{{ parser_version }}. DO NOT EDIT. DO NOT COMMIT. +{# The rendered source is autogenerated, but this +Jinja2 template is not. Please file bugs! #} + +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +{%- macro generate_structure(name, struct) %} +{%- if struct.type == "oneof" -%} + #[derive(Debug, Hash, Eq, PartialEq, Clone, ::glean_sym::__serde::Serialize, ::glean_sym::__serde::Deserialize)] + #[serde(crate = "::glean_sym::__serde")] + #[serde(untagged)] + pub enum {{ name }} { + {% for ty in struct.subtypes %} + {{ty|Camelize}}({{ty|structure_type_name}}), + {% endfor %} + } +{% elif struct.type == "array" %} + pub type {{ name }} = Vec<{{ name }}Item>; + + {{ generate_structure(name ~ "Item", struct["items"]) }} + +{% elif struct.type == "object" %} + #[derive(Debug, Hash, Eq, PartialEq, Clone, ::glean_sym::__serde::Serialize, ::glean_sym::__serde::Deserialize)] + #[serde(crate = "::glean_sym::__serde")] + #[serde(deny_unknown_fields)] + pub struct {{ name }} { + {% for itemname, val in struct.properties.items() %} + {% if val.type == "array" %} + #[serde(skip_serializing_if = "Vec::is_empty", default = "Vec::new", deserialize_with = "::glean_sym::__serde_helper::vec_null")] + pub {{itemname|snake_case}}: {{ name ~ itemname|Camelize }}, + {% elif val.type == "object" %} + #[serde(skip_serializing_if = "Option::is_none")] + pub {{itemname|snake_case}}: Option<{{ name ~ "Item" ~ itemname|Camelize ~ "Object" }}>, + {% elif val.type == "oneof" %} + #[serde(skip_serializing_if = "Option::is_none")] + pub {{itemname|snake_case}}: Option<{{ name ~ itemname|Camelize ~ "Enum" }}>, + {% else %} + #[serde(skip_serializing_if = "Option::is_none")] + pub {{itemname|snake_case}}: Option<{{val.type|structure_type_name}}>, + {% endif %} + {% endfor %} + } + + {% for itemname, val in struct.properties.items() %} + {% if val.type == "array" %} + {% set nested_name = name ~ itemname|Camelize %} + {{ generate_structure(nested_name, val) }} + {% elif val.type == "object" %} + {% set nested_name = name ~ "Item" ~ itemname|Camelize ~ "Object" %} + {{ generate_structure(nested_name, val) }} + {% elif val.type == "oneof" %} + {% set nested_name = name ~ itemname|Camelize ~ "Enum" %} + {{ generate_structure(nested_name, val) }} + {% endif %} + {% endfor %} + +{% else %} + +pub type {{ name }} = {{ struct.type|structure_type_name }}; + +{% endif %} + +{% endmacro %} + +{% macro generate_extra_keys(obj) %} +{% for name, _ in obj["_generate_enums"] %} +{# we always use the `extra` suffix, because we only expose the new event API #} +{% set suffix = "Extra" %} +{% if obj|attr(name)|length %} + {{ extra_keys_with_types(obj, name, suffix)|indent }} +{% endif %} +{% endfor %} +{% endmacro %} +{% macro extra_keys_with_types(obj, name, suffix) %} +#[derive(Default, Debug, Clone, Hash, Eq, PartialEq)] +pub struct {{ obj.name|Camelize }}{{ suffix }} { + {% for item, type in obj|attr(name) %} + pub {{ item|snake_case }}: Option<{{type|extra_type_name}}>, + {% endfor %} +} + +impl ExtraKeys for {{ obj.name|Camelize }}{{ suffix }} { + const ALLOWED_KEYS: &'static [&'static str] = {{ obj.allowed_extra_keys|extra_keys }}; + + fn into_ffi_extra(self) -> ::std::collections::HashMap<::std::string::String, ::std::string::String> { + let mut map = ::std::collections::HashMap::new(); + {% for key, _ in obj|attr(name) %} + self.{{key|snake_case}}.and_then(|val| map.insert("{{key}}".to_string(), val.to_string())); + {% endfor %} + map + } +} +{% endmacro %} +{% macro common_metric_data(obj) %} +CommonMetricData { + category: {{ obj.category|rust }}, + name: {{ obj.name|rust }}, + send_in_pings: {{ obj.send_in_pings|rust }}, + lifetime: {{ obj.lifetime|rust }}, + disabled: {{ obj.is_disabled()|rust }}, + ..Default::default() +} +{%- endmacro -%} +{% for category in categories %} +{% if category.contains_pings %} +{% for obj in category.objs.values() %} +#[allow(non_upper_case_globals, dead_code)] +/// {{ obj.description|wordwrap() | replace('\n', '\n/// ') }} +#[rustfmt::skip] +pub static {{ obj.name|snake_case }}: ::glean_sym::__export::Lazy<::glean_sym::PingType> = + ::glean_sym::__export::Lazy::new(|| ::glean_sym::PingType::new("{{ obj.name }}", {{ obj.include_client_id|rust }}, {{ obj.send_if_empty|rust }}, {{ obj.precise_timestamps|rust }}, {{ obj.include_info_sections|rust }}, {{ obj.enabled|rust }}, {{ obj.schedules_pings|rust }}, {{ obj.reason_codes|rust }}, {{ obj.follows_collection_enabled|rust }}, {{ obj.uploader_capabilities|rust }})); +{% endfor %} +{% else %} +pub mod {{ category.name|snake_case }} { + #[allow(unused_imports)] // HistogramType might be unusued, let's avoid warnings + use glean_sym::{metrics::*, types::*}; + {% for obj in category.objs.values() %} + + {% if obj|attr("_generate_structure") %} +{{ generate_structure(obj.name|Camelize ~ "Object", obj._generate_structure) }} + {%- endif %} + + {% if obj|attr("_generate_enums") %} +{{ generate_extra_keys(obj) }} + {%- endif %} + #[allow(non_upper_case_globals, dead_code)] + /// generated from {{ category.name }}.{{ obj.name }} + /// + /// {{ obj.description|wordwrap() | replace('\n', '\n /// ') }} + pub static {{ obj.name|snake_case }}: ::glean_sym::__export::Lazy<{{ obj|type_name }}> = ::glean_sym::__export::Lazy::new(|| { + let meta = + {% if obj.type == "labeled_custom_distribution" %} + LabeledMetricData::CustomDistribution { + cmd: {{ common_metric_data(obj)|indent(16) }} + {%- for arg_name in extra_metric_args if obj[arg_name] is defined and arg_name != 'allowed_extra_keys' -%} + , {{ arg_name }}: {{ obj[arg_name]|rust }} + {%- endfor -%} + }; + {% elif obj.type == "labeled_memory_distribution" %} + LabeledMetricData::MemoryDistribution { + cmd: {{ common_metric_data(obj)|indent(16) }} + {%- for arg_name in extra_metric_args if obj[arg_name] is defined and arg_name != 'allowed_extra_keys' -%} + , {{ "unit" if arg_name == "memory_unit" else arg_name }}: {{ obj[arg_name]|rust }} + {%- endfor -%} + }; + {% elif obj.type == "labeled_timing_distribution" %} + LabeledMetricData::TimingDistribution { + cmd: {{ common_metric_data(obj)|indent(16) }} + {%- for arg_name in extra_metric_args if obj[arg_name] is defined and arg_name != 'allowed_extra_keys' -%} + , {{ "unit" if arg_name == "time_unit" else arg_name }}: {{ obj[arg_name]|rust }} + {%- endfor -%} + }; + {% elif obj.labeled %} + LabeledMetricData::Common { + cmd: {{common_metric_data(obj)|indent(16) }}, + }; + {% else %} + {{ common_metric_data(obj)|indent(12) }}; + {% endif %} + {{ obj|ctor }}(meta + {%- for arg_name in extra_metric_args if not obj.labeled and obj[arg_name] is defined and arg_name != 'allowed_extra_keys' -%} + , {{ obj[arg_name]|rust }} + {%- endfor -%} + {{ ", " if obj.labeled or obj.dual_labeled else ")\n" }} + {%- if obj.labeled -%} + {%- if obj.labels -%} + Some({{ obj.labels|rust }}) + {%- else -%} + None + {%- endif -%}) + {% endif %} + {%- if obj.dual_labeled -%} + {%- if obj.keys -%} + Some({{ obj.keys|rust }}) + {%- else -%} + None + {%- endif -%} + {{ ", " }} + {%- if obj.categories -%} + Some({{ obj.categories|rust }}) + {%- else -%} + None + {%- endif -%}) + {% endif %} + }); + {% endfor %} +} +{% endif %} +{% endfor %} +{% if metric_by_type|length > 0 %} + +{% endif %} diff --git a/glean_parser/translate.py b/glean_parser/translate.py index 303ea15a..61fdc72b 100644 --- a/glean_parser/translate.py +++ b/glean_parser/translate.py @@ -26,6 +26,7 @@ from . import ruby_server from . import rust from . import rust_server +from . import rust_sym from . import swift from . import util @@ -68,6 +69,7 @@ def __init__( "swift": Outputter(swift.output_swift, ["*.swift"]), "rust": Outputter(rust.output_rust, []), "rust_server": Outputter(rust_server.output_rust, []), + "rust_sym": Outputter(rust_sym.output_rust, []), } diff --git a/tests/test_rust_sym.py b/tests/test_rust_sym.py new file mode 100644 index 00000000..c514bf65 --- /dev/null +++ b/tests/test_rust_sym.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +# Very minimal testing for `rust_sym` support. +# We're reusing most of the `rust` translator work, +# so we're only testing that the generated code is at least still valid Rust. + +from pathlib import Path +import shutil +import subprocess + +from glean_parser import translate + + +ROOT = Path(__file__).parent + + +def run_linters(files): + # Syntax check on the generated files. + # Only run this test if cargo is on the path. + if shutil.which("rustfmt"): + for filepath in files: + subprocess.check_call( + [ + "rustfmt", + "--check", + filepath, + ] + ) + if shutil.which("cargo"): + for filepath in files: + subprocess.check_call( + [ + "cargo", + "clippy", + "--all", + "--", + "-D", + "warnings", + filepath, + ] + ) + + +def test_parser(tmp_path): + """Test translating metrics to Rust files.""" + translate.translate( + ROOT / "data" / "core.yaml", "rust_sym", tmp_path, {}, {"allow_reserved": True} + ) + + assert set(x.name for x in tmp_path.iterdir()) == set(["glean_metrics.rs"]) + + # Make sure descriptions made it in + with (tmp_path / "glean_metrics.rs").open("r", encoding="utf-8") as fd: + content = fd.read() + + assert "True if the user has set Firefox as the default browser." in content + assert "جمع 搜集" in content + assert 'category: "telemetry"' in content