Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

- Go server: Add support for `labeled_boolean` metrics with static labels ([AE-1250](https://mozilla-hub.atlassian.net/browse/AE-1250))
- BUGFIX: Go server: Serialize nil `string_list` metrics as empty arrays instead of null ([#837](https://github.com/mozilla/glean_parser/pull/837))
- Add support to target `glean_sym` (the new experimental Glean Rust API) ([#841](https://github.com/mozilla/glean_parser/pull/841))

## 19.0.0

Expand Down
78 changes: 78 additions & 0 deletions glean_parser/rust_sym.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
# -*- coding: utf-8 -*-

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

"""
Outputter to generate Rust code for metrics.
"""

from pathlib import Path
from typing import Any, Dict, Optional

from . import __version__
from . import metrics
from . import pings
from . import util
from .rust import (
rust_datatypes_filter,
ctor,
type_name,
extra_type_name,
structure_type_name,
extra_keys,
Category,
)


def output_rust(
objs: metrics.ObjectTree, output_dir: Path, options: Optional[Dict[str, Any]] = None
) -> None:
"""
Given a tree of objects, output Rust code to `output_dir`.

:param objs: A tree of objects (metrics and pings) as returned from
`parser.parse_objects`.
:param output_dir: Path to an output directory to write to.
:param options: options dictionary, not currently used for Rust
"""

if options is None:
options = {}

template = util.get_jinja2_template(
"rust_sym.jinja2",
filters=(
("rust", rust_datatypes_filter),
("snake_case", util.snake_case),
("camelize", util.camelize),
("type_name", type_name),
("extra_type_name", extra_type_name),
("structure_type_name", structure_type_name),
("ctor", ctor),
("extra_keys", extra_keys),
),
)

filename = "glean_metrics.rs"
filepath = output_dir / filename
categories = []

for category_key, category_val in objs.items():
contains_pings = any(
isinstance(obj, pings.Ping) for obj in category_val.values()
)

cat = Category(category_key, category_val, contains_pings)
categories.append(cat)

with filepath.open("w", encoding="utf-8") as fd:
fd.write(
template.render(
parser_version=__version__,
categories=categories,
extra_metric_args=util.extra_metric_args,
common_metric_args=util.common_metric_args,
)
)
196 changes: 196 additions & 0 deletions glean_parser/templates/rust_sym.jinja2
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
// -*- mode: Rust -*-

// AUTOGENERATED BY glean_parser v{{ parser_version }}. DO NOT EDIT. DO NOT COMMIT.
{# The rendered source is autogenerated, but this
Jinja2 template is not. Please file bugs! #}

/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

{%- macro generate_structure(name, struct) %}
{%- if struct.type == "oneof" -%}
#[derive(Debug, Hash, Eq, PartialEq, Clone, ::glean_sym::__serde::Serialize, ::glean_sym::__serde::Deserialize)]
#[serde(crate = "::glean_sym::__serde")]
#[serde(untagged)]
pub enum {{ name }} {
{% for ty in struct.subtypes %}
{{ty|Camelize}}({{ty|structure_type_name}}),
{% endfor %}
}
{% elif struct.type == "array" %}
pub type {{ name }} = Vec<{{ name }}Item>;

{{ generate_structure(name ~ "Item", struct["items"]) }}

{% elif struct.type == "object" %}
#[derive(Debug, Hash, Eq, PartialEq, Clone, ::glean_sym::__serde::Serialize, ::glean_sym::__serde::Deserialize)]
#[serde(crate = "::glean_sym::__serde")]
#[serde(deny_unknown_fields)]
pub struct {{ name }} {
{% for itemname, val in struct.properties.items() %}
{% if val.type == "array" %}
#[serde(skip_serializing_if = "Vec::is_empty", default = "Vec::new", deserialize_with = "::glean_sym::__serde_helper::vec_null")]
pub {{itemname|snake_case}}: {{ name ~ itemname|Camelize }},
{% elif val.type == "object" %}
#[serde(skip_serializing_if = "Option::is_none")]
pub {{itemname|snake_case}}: Option<{{ name ~ "Item" ~ itemname|Camelize ~ "Object" }}>,
{% elif val.type == "oneof" %}
#[serde(skip_serializing_if = "Option::is_none")]
pub {{itemname|snake_case}}: Option<{{ name ~ itemname|Camelize ~ "Enum" }}>,
{% else %}
#[serde(skip_serializing_if = "Option::is_none")]
pub {{itemname|snake_case}}: Option<{{val.type|structure_type_name}}>,
{% endif %}
{% endfor %}
}

{% for itemname, val in struct.properties.items() %}
{% if val.type == "array" %}
{% set nested_name = name ~ itemname|Camelize %}
{{ generate_structure(nested_name, val) }}
{% elif val.type == "object" %}
{% set nested_name = name ~ "Item" ~ itemname|Camelize ~ "Object" %}
{{ generate_structure(nested_name, val) }}
{% elif val.type == "oneof" %}
{% set nested_name = name ~ itemname|Camelize ~ "Enum" %}
{{ generate_structure(nested_name, val) }}
{% endif %}
{% endfor %}

{% else %}

pub type {{ name }} = {{ struct.type|structure_type_name }};

{% endif %}

{% endmacro %}

{% macro generate_extra_keys(obj) %}
{% for name, _ in obj["_generate_enums"] %}
{# we always use the `extra` suffix, because we only expose the new event API #}
{% set suffix = "Extra" %}
{% if obj|attr(name)|length %}
{{ extra_keys_with_types(obj, name, suffix)|indent }}
{% endif %}
{% endfor %}
{% endmacro %}
{% macro extra_keys_with_types(obj, name, suffix) %}
#[derive(Default, Debug, Clone, Hash, Eq, PartialEq)]
pub struct {{ obj.name|Camelize }}{{ suffix }} {
{% for item, type in obj|attr(name) %}
pub {{ item|snake_case }}: Option<{{type|extra_type_name}}>,
{% endfor %}
}

impl ExtraKeys for {{ obj.name|Camelize }}{{ suffix }} {
const ALLOWED_KEYS: &'static [&'static str] = {{ obj.allowed_extra_keys|extra_keys }};

fn into_ffi_extra(self) -> ::std::collections::HashMap<::std::string::String, ::std::string::String> {
let mut map = ::std::collections::HashMap::new();
{% for key, _ in obj|attr(name) %}
self.{{key|snake_case}}.and_then(|val| map.insert("{{key}}".to_string(), val.to_string()));
{% endfor %}
map
}
}
{% endmacro %}
{% macro common_metric_data(obj) %}
CommonMetricData {
category: {{ obj.category|rust }},
name: {{ obj.name|rust }},
send_in_pings: {{ obj.send_in_pings|rust }},
lifetime: {{ obj.lifetime|rust }},
disabled: {{ obj.is_disabled()|rust }},
..Default::default()
}
{%- endmacro -%}
{% for category in categories %}
{% if category.contains_pings %}
{% for obj in category.objs.values() %}
#[allow(non_upper_case_globals, dead_code)]
/// {{ obj.description|wordwrap() | replace('\n', '\n/// ') }}
#[rustfmt::skip]
pub static {{ obj.name|snake_case }}: ::glean_sym::__export::Lazy<::glean_sym::PingType> =
::glean_sym::__export::Lazy::new(|| ::glean_sym::PingType::new("{{ obj.name }}", {{ obj.include_client_id|rust }}, {{ obj.send_if_empty|rust }}, {{ obj.precise_timestamps|rust }}, {{ obj.include_info_sections|rust }}, {{ obj.enabled|rust }}, {{ obj.schedules_pings|rust }}, {{ obj.reason_codes|rust }}, {{ obj.follows_collection_enabled|rust }}, {{ obj.uploader_capabilities|rust }}));
{% endfor %}
{% else %}
pub mod {{ category.name|snake_case }} {
#[allow(unused_imports)] // HistogramType might be unusued, let's avoid warnings
use glean_sym::{metrics::*, types::*};
{% for obj in category.objs.values() %}

{% if obj|attr("_generate_structure") %}
{{ generate_structure(obj.name|Camelize ~ "Object", obj._generate_structure) }}
{%- endif %}

{% if obj|attr("_generate_enums") %}
{{ generate_extra_keys(obj) }}
{%- endif %}
#[allow(non_upper_case_globals, dead_code)]
/// generated from {{ category.name }}.{{ obj.name }}
///
/// {{ obj.description|wordwrap() | replace('\n', '\n /// ') }}
pub static {{ obj.name|snake_case }}: ::glean_sym::__export::Lazy<{{ obj|type_name }}> = ::glean_sym::__export::Lazy::new(|| {
let meta =
{% if obj.type == "labeled_custom_distribution" %}
LabeledMetricData::CustomDistribution {
cmd: {{ common_metric_data(obj)|indent(16) }}
{%- for arg_name in extra_metric_args if obj[arg_name] is defined and arg_name != 'allowed_extra_keys' -%}
, {{ arg_name }}: {{ obj[arg_name]|rust }}
{%- endfor -%}
};
{% elif obj.type == "labeled_memory_distribution" %}
LabeledMetricData::MemoryDistribution {
cmd: {{ common_metric_data(obj)|indent(16) }}
{%- for arg_name in extra_metric_args if obj[arg_name] is defined and arg_name != 'allowed_extra_keys' -%}
, {{ "unit" if arg_name == "memory_unit" else arg_name }}: {{ obj[arg_name]|rust }}
{%- endfor -%}
};
{% elif obj.type == "labeled_timing_distribution" %}
LabeledMetricData::TimingDistribution {
cmd: {{ common_metric_data(obj)|indent(16) }}
{%- for arg_name in extra_metric_args if obj[arg_name] is defined and arg_name != 'allowed_extra_keys' -%}
, {{ "unit" if arg_name == "time_unit" else arg_name }}: {{ obj[arg_name]|rust }}
{%- endfor -%}
};
{% elif obj.labeled %}
LabeledMetricData::Common {
cmd: {{common_metric_data(obj)|indent(16) }},
};
{% else %}
{{ common_metric_data(obj)|indent(12) }};
{% endif %}
{{ obj|ctor }}(meta
{%- for arg_name in extra_metric_args if not obj.labeled and obj[arg_name] is defined and arg_name != 'allowed_extra_keys' -%}
, {{ obj[arg_name]|rust }}
{%- endfor -%}
{{ ", " if obj.labeled or obj.dual_labeled else ")\n" }}
{%- if obj.labeled -%}
{%- if obj.labels -%}
Some({{ obj.labels|rust }})
{%- else -%}
None
{%- endif -%})
{% endif %}
{%- if obj.dual_labeled -%}
{%- if obj.keys -%}
Some({{ obj.keys|rust }})
{%- else -%}
None
{%- endif -%}
{{ ", " }}
{%- if obj.categories -%}
Some({{ obj.categories|rust }})
{%- else -%}
None
{%- endif -%})
{% endif %}
});
{% endfor %}
}
{% endif %}
{% endfor %}
{% if metric_by_type|length > 0 %}

{% endif %}
2 changes: 2 additions & 0 deletions glean_parser/translate.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from . import ruby_server
from . import rust
from . import rust_server
from . import rust_sym
from . import swift
from . import util

Expand Down Expand Up @@ -68,6 +69,7 @@ def __init__(
"swift": Outputter(swift.output_swift, ["*.swift"]),
"rust": Outputter(rust.output_rust, []),
"rust_server": Outputter(rust_server.output_rust, []),
"rust_sym": Outputter(rust_sym.output_rust, []),
}


Expand Down
61 changes: 61 additions & 0 deletions tests/test_rust_sym.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
# -*- coding: utf-8 -*-

# Any copyright is dedicated to the Public Domain.
# http://creativecommons.org/publicdomain/zero/1.0/

# Very minimal testing for `rust_sym` support.
# We're reusing most of the `rust` translator work,
# so we're only testing that the generated code is at least still valid Rust.

from pathlib import Path
import shutil
import subprocess

from glean_parser import translate


ROOT = Path(__file__).parent


def run_linters(files):
# Syntax check on the generated files.
# Only run this test if cargo is on the path.
if shutil.which("rustfmt"):
for filepath in files:
subprocess.check_call(
[
"rustfmt",
"--check",
filepath,
]
)
if shutil.which("cargo"):
for filepath in files:
subprocess.check_call(
[
"cargo",
"clippy",
"--all",
"--",
"-D",
"warnings",
filepath,
]
)


def test_parser(tmp_path):
"""Test translating metrics to Rust files."""
translate.translate(
ROOT / "data" / "core.yaml", "rust_sym", tmp_path, {}, {"allow_reserved": True}
)

assert set(x.name for x in tmp_path.iterdir()) == set(["glean_metrics.rs"])

# Make sure descriptions made it in
with (tmp_path / "glean_metrics.rs").open("r", encoding="utf-8") as fd:
content = fd.read()

assert "True if the user has set Firefox as the default browser." in content
assert "جمع 搜集" in content
assert 'category: "telemetry"' in content