From bca1500993391e44f292f0f0a654c7b8c10b354c Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Wed, 29 Jul 2026 23:12:48 +0500 Subject: [PATCH 01/46] Add Apache Ossie <-> Cube converter (import direction) Scaffolds converters/cube/ following the osi-omni and osi-databricks converters: a pure offline YAML transform with no Cube deployment, API token, or network access required. This commit lands the import direction (Cube -> Ossie). Cubes become datasets, cube joins become relationships, cube measures are hoisted to model-level metrics, and the mapped view supplies the model's name, description, and AI context -- the view is the model boundary because Cube users are view-first and Cube's agent reads meta.ai_context only from views and members, not cubes. Design decisions worth calling out: - Fan-out. Cube corrects row multiplication at query time by deduplicating on declared primary keys, which a static Ossie expression cannot inherit. So a bare `type: count` maps to COUNT(DISTINCT ) -- exactly equal to both forms Cube renders, and correct in every join context -- and a non-idempotent aggregate on a dataset the graph fans out is refused by default, mirroring Cube's own refusal. --no-strict-fanout downgrades it to a recorded issue. - Calculated measures inline their {other_measure} references, because that is what Cube itself does; Ossie has no metric-to-metric reference. Cycles are rejected. - Measure filters fold into CASE WHEN ... END inside the aggregate, matching Cube's own applyMeasureFilters rendering. - Dimension `type: number` omits Ossie `datatype` rather than assert a precision the model does not carry; `type: geo` splits into two fields since an Ossie field holds one expression. - Losses that cannot be avoided surface as structured ConverterIssues rather than bare warnings, following the osi-dbt converter, so a pipeline can gate on them. Jinja-templated YAML, .js/.ts models, and `extends` are refused or preserved verbatim rather than half-converted. Everything Cube-only round-trips through custom_extensions[CUBE]. 49 tests pass; the fixture output validates against core-spec/osi-schema.json via validation/validate.py. Co-Authored-By: Claude Opus 5 --- .github/workflows/converter-cube-ci.yml | 63 ++ converters/cube/README.md | 237 +++++ converters/cube/pyproject.toml | 67 ++ converters/cube/src/ossie_cube/__init__.py | 37 + converters/cube/src/ossie_cube/_common.py | 593 +++++++++++++ converters/cube/src/ossie_cube/cli.py | 117 +++ .../cube/src/ossie_cube/converter_issues.py | 110 +++ converters/cube/src/ossie_cube/cube_to_osi.py | 839 ++++++++++++++++++ converters/cube/tests/_util.py | 98 ++ converters/cube/tests/conftest.py | 25 + .../fixtureA_cube/model/cubes/orders.yml | 68 ++ .../fixtureA_cube/model/cubes/users.yml | 48 + .../fixtureA_cube/model/views/sales.yml | 33 + converters/cube/tests/test_cube_to_osi.py | 518 +++++++++++ converters/cube/uv.lock | 216 +++++ 15 files changed, 3069 insertions(+) create mode 100644 .github/workflows/converter-cube-ci.yml create mode 100644 converters/cube/README.md create mode 100644 converters/cube/pyproject.toml create mode 100644 converters/cube/src/ossie_cube/__init__.py create mode 100644 converters/cube/src/ossie_cube/_common.py create mode 100644 converters/cube/src/ossie_cube/cli.py create mode 100644 converters/cube/src/ossie_cube/converter_issues.py create mode 100644 converters/cube/src/ossie_cube/cube_to_osi.py create mode 100644 converters/cube/tests/_util.py create mode 100644 converters/cube/tests/conftest.py create mode 100644 converters/cube/tests/fixtures/fixtureA_cube/model/cubes/orders.yml create mode 100644 converters/cube/tests/fixtures/fixtureA_cube/model/cubes/users.yml create mode 100644 converters/cube/tests/fixtures/fixtureA_cube/model/views/sales.yml create mode 100644 converters/cube/tests/test_cube_to_osi.py create mode 100644 converters/cube/uv.lock diff --git a/.github/workflows/converter-cube-ci.yml b/.github/workflows/converter-cube-ci.yml new file mode 100644 index 00000000..a8cafb84 --- /dev/null +++ b/.github/workflows/converter-cube-ci.yml @@ -0,0 +1,63 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Converters Cube CI + +on: + push: + branches: [ "main" ] + paths: + - 'converters/cube/**' + - '.github/workflows/converter-cube-ci.yml' + pull_request: + branches: [ "main" ] + paths: + - 'converters/cube/**' + - '.github/workflows/converter-cube-ci.yml' + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Checkout project + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "${HOME}/.local/bin" >> "${GITHUB_PATH}" + + - name: Sync dependencies + working-directory: converters/cube + run: | + uv sync + + - name: Unit Tests + working-directory: converters/cube + run: | + uv run pytest diff --git a/converters/cube/README.md b/converters/cube/README.md new file mode 100644 index 00000000..4fe445fe --- /dev/null +++ b/converters/cube/README.md @@ -0,0 +1,237 @@ + + +# Apache Ossie <-> Cube converter + +Bidirectional, offline conversion between an [Apache Ossie](https://github.com/apache/ossie) +semantic model and a [Cube](https://cube.dev/docs/product/data-modeling/overview) +data model. No Cube deployment, API token, or network access required. + +> **Status:** the **import** direction (Cube -> Ossie) is implemented. The +> **export** direction (Ossie -> Cube) is in progress; the mapping table below +> describes the agreed behavior for both. + +A Cube data model is a *directory* of YAML files rather than a single document, so +this converter maps one Ossie YAML document to/from the Cube model layout: + +``` +model/cubes/.yml # one per Ossie dataset +model/views/.yml # the view the Ossie model maps to +``` + +Import accepts any layout: `cubes:` and `views:` may live in any `.yml`/`.yaml` +file at any depth, several per file, and original file paths are preserved through +a round trip. + +- **Import** (`ossie-cube import`): Cube files -> Ossie. Cube features Ossie has + no native field for are preserved in `custom_extensions[CUBE]`, so + **Cube -> Ossie -> Cube is lossless**. +- **Export** (`ossie-cube export`): Ossie -> Cube files. Ossie features with no + Cube slot are parked under `meta.ossie` rather than dropped -- Cube has a `meta` + field at every level -- so **Ossie -> Cube -> Ossie is lossless too**. + +Any input that breaks a [requirement](#requirements) **raises a +`ConversionError`** -- the converter never silently drops a field or produces an +invalid result. Losses it *can* absorb are returned as structured +[issues](#conversion-issues) rather than printed and forgotten. + +## Installation + +```bash +pip install apache-ossie-cube # once published to PyPI +# or, from a checkout of this directory: +pip install -e . +``` + +The only runtime dependency is `PyYAML`. Python 3.11+. + +## Usage + +### Command line + +```bash +ossie-cube import -i model/ [-o model.yaml] [--name my_model] [--view sales] + [--no-strict-fanout] +``` + +With no `-o` the Ossie YAML goes to stdout; issues always go to stderr. `--view` +picks which view's name/description/AI context map onto the Ossie model when the +directory holds several. `--name` overrides the model name. + +### Python API + +```python +from ossie_cube import convert_cube_to_ossie + +ossie_yaml, issues = convert_cube_to_ossie(files) # {relative filename: YAML str} +for issue in issues: + print(issue) +``` + +## Mapping + +Each row maps in both directions; the **Notes** flag where a behavior is specific +to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). + +| Apache Ossie | Cube | Notes | +|---|---|---| +| `semantic_model` | a **view** | Cube users are view-first, and Cube's agent reads `meta.ai_context` only from views and members -- so the view, not any cube, is the model boundary. | +| `semantic_model.name` | view name | Import: the mapped view's name (override with `--name`). | +| `model.description` / `ai_context.instructions` | view `description` / `meta.ai_context` | Import: taken from the sole view, or `--view`. | +| dataset | `cubes[]` entry in `model/cubes/.yml` | Import: a non-canonical original path is stashed and restored on export. | +| `dataset.source` (dotted) | `sql_table` | Passed through verbatim; Cube interpolates it straight into `FROM`, so no catalog/schema split is needed. | +| `dataset.source` (`SELECT ...`) | `sql` | Cube requires exactly one of `sql` / `sql_table`. | +| `dataset.description` | cube `description` | | +| `dataset.ai_context` | cube `meta.ai_context` | Preserved for the round trip, but **inert in Cube** -- its agent ignores cube-level `ai_context`. Recorded as an issue. | +| `dataset.primary_key` | dimension(s) with `primary_key: true` | Composite = several. Export: a key column no field covers becomes a `public: false` dimension. | +| `dataset.unique_keys` | `meta.ossie.unique_keys` | No native Cube slot; parked rather than dropped. | +| field | `dimensions[]` entry | Export: a name that is not a valid Cube identifier is sanitized; a case-insensitive collision is an error, never a silent merge. | +| `field.expression` | dimension `sql` | Dataset-scoped, so `{CUBE}.col` <-> `col`. Export emits `{CUBE}.column` for a raw column and `{CUBE.member}` for a declared member, and never spells the cube's own name (which would break under `extends`). | +| `field.datatype` | dimension `type` (**required**) | `String`->`string`, `Boolean`->`boolean`, `Date`/`Time`/`DateTime`/`DateTimeTz`->`time`, `Integer`/`Decimal`/`Float`->`number`, `Opaque`->`string`. Import maps back except for `number`, where it **omits `datatype`** -- Cube collapses three Ossie types into one, and the spec says to omit rather than assert. The original `type` is stashed. | +| `field.dimension.is_time` | `type: time` | Import sets `is_time: true` for a time dimension. | +| `field.label` / `description` | dimension `title` / `description` | | +| `field.ai_context.instructions` | dimension `meta.ai_context` | Cube's documented AI-only context field. | +| — | `type: geo` dimension | An Ossie field holds one expression and a geo dimension has two, so it **splits** into `_latitude` / `_longitude` (`Float`). Reconstruction data rides on the latitude half. | +| relationship | `joins[]` on a cube | `many_to_one` on cube A -> `from: A`(many), `to: B`(one). `one_to_many` is flipped so Ossie's `from` is the many side; the declared side and type are stashed so export restores the original. | +| `from_columns` / `to_columns` | join `sql` | Only an AND-chain of equalities between two member references maps. Anything else (non-equi, range, literal, third cube) is preserved verbatim in the stash. | +| metric | `measures[]` on the cube its expression references | Import hoists cube-scoped measures to the model level, qualifying a colliding name as `__` and stashing the original name and owning cube. | +| `SUM`/`AVG`/`MIN`/`MAX(x)` | `type: sum`/`avg`/`min`/`max` + `sql` | | +| `COUNT(DISTINCT x)` | `type: count_distinct` | | +| `APPROX_COUNT_DISTINCT(x)` | `type: count_distinct_approx` | Cube resolves the warehouse-specific function itself. | +| `COUNT(DISTINCT )` | bare `type: count` | See [Fan-out](#fan-out) -- the primary key is load-bearing here. | +| anything else | `type: number` (calculated) | A `{other_measure}` reference is **inlined**, because that is what Cube itself does; Ossie has no metric-to-metric reference. | +| — | measure `filters` | Folded into `CASE WHEN … THEN … END` inside the aggregate, exactly as Cube's own `applyMeasureFilters` renders it. | +| `metric.datatype` | — | Import emits `Integer` for the count family, whose result type Cube does know, and omits it otherwise. | +| `metric.description` / `ai_context` | measure `description` / `meta.ai_context` | | +| `custom_extensions[CUBE]` | everything Cube-only | Import stashes; export restores -- keeping `Cube -> Ossie -> Cube` lossless. | +| foreign-vendor `custom_extensions` | `meta.ossie.custom_extensions` | Parked so a multi-vendor Ossie model survives the round trip. | + +**Stashed on import** (and restored on export): the views verbatim (minus the +natively mapped description/AI context), the mapped view's identity, original file +paths, cube extras (`title`, `sql_alias`, `data_source`, `public`, `refresh_key`, +`segments`, `pre_aggregations`, `hierarchies`, `access_policy`, `calendar`, ...), +dimension extras (`format`, `currency`, `granularities`, `case`, `sub_query`, +`order`, `aliases`, `meta`, ...), measure extras and any non-reconstructible +measure, joins with no Ossie form, Jinja-templated members, and files with no +Ossie form (`.js`/`.ts` models, non-model YAML). + +**Expression dialects**: Cube SQL is the SQL of the model's data source, and the +Ossie dialect enum has no `CUBE` entry -- so import emits `ANSI_SQL`, and export +prefers `ANSI_SQL` with `--dialect` prepending a warehouse dialect (e.g. +`SNOWFLAKE` for a Snowflake-backed Cube model). + +## Fan-out + +This is the one place where Cube carries semantics an Ossie expression cannot, and +it is handled deliberately rather than papered over. + +When a cube sits on the multiplied side of a join, Cube does **not** aggregate over +the flattened join. It builds `SELECT DISTINCT FROM `, joins +that key set back to the measure's own cube, and aggregates there -- so each source +row is counted once. If the measures themselves span cubes that fan out, Cube +refuses the query outright. Correctness comes from a *runtime rewrite keyed on +declared primary keys*, and a static SQL string has no way to inherit it. + +So the converter emits the fan-out-safe form wherever one exists, and refuses to +emit a silently-wrong one: + +| Cube measure | Ossie expression | Safe under fan-out? | +|---|---|---| +| bare `count` | `COUNT(DISTINCT )` | **Yes, exactly.** Cube renders `count(pk)` normally and `count(distinct pk)` when multiplied; `COUNT(DISTINCT pk)` equals both. A composite key is concatenated with `CAST` + `CONCAT`, as Cube does. | +| `count_distinct` | `COUNT(DISTINCT x)` | Yes, inherently | +| `count_distinct_approx` | `APPROX_COUNT_DISTINCT(x)` | Yes, inherently | +| `min` / `max` | `MIN(x)` / `MAX(x)` | Yes -- idempotent under duplication | +| `sum`, `avg`, `count` + `sql` | `SUM(x)`, `AVG(x)`, `COUNT(x)` | **No** | + +Only the last row is at risk, and only when its own cube is the `to` (one) side of +a relationship in the model. The converter computes that from the Ossie graph and, +**by default, refuses** -- mirroring Cube's own refusal. Pass +`--no-strict-fanout` to emit the metric with a `FANOUT_UNSAFE_METRIC` issue +instead, naming the metric, the dataset, and the relationship responsible. + +Because a bare `count` maps through the primary key, a cube carrying one **must** +declare `primary_key: true` on a dimension; its absence is an error, not a +different number. + +> Ossie has no additivity or grain declaration to record this properly -- dbt's +> `non_additive_dimension` is the nearest precedent, and this repo's dbt converter +> already loses the same information. Worth raising on `dev@`. + +## Conversion issues + +`convert_cube_to_ossie` returns `(yaml, IssueLog)`. Each issue carries a type, the +element it concerns, and a detail string. + +| Issue type | Meaning | +|---|---| +| `FANOUT_UNSAFE_METRIC` | A non-idempotent aggregate on a dataset the graph fans out; see [Fan-out](#fan-out) | +| `MULTI_STAGE_MEASURE_DROPPED` | A `multi_stage` measure (`group_by`/`reduce_by`/`time_shift`/`rank`) renders as a window function over another grain | +| `CUBE_LEVEL_AI_CONTEXT_INERT` | Cube's agent ignores cube-level `meta.ai_context` | +| `GEO_DIMENSION_SPLIT` | A `type: geo` dimension became two Ossie fields | +| `TEMPLATED_MEMBER_DROPPED` | Jinja templating, or a `.js`/`.ts` model file | +| `NO_USABLE_DIALECT` | Export: no `ANSI_SQL` or preferred-dialect expression | +| `PARKED_IN_META` | An element preserved in the stash with no native mapping | + +## Requirements + +Conversion raises a `ConversionError` (rather than guessing or emitting something +invalid) when an input breaks one of these: + +- a cube has neither or both of `sql` / `sql_table` (Cube requires exactly one); +- a cube uses `extends` -- resolving it means reproducing Cube's definition-merge + semantics exactly, so it is refused rather than half-applied; +- a bare `type: count` measure's cube declares no primary key; +- a join names a cube that is not in the model, or an unknown `relationship`; +- a measure has an unknown `type`, or a measure reference cycle; +- two cubes, two views, or two derived metric names collide; +- a dimension has an unknown `type`, or a `geo` dimension is missing + `latitude.sql` / `longitude.sql`; +- there are no convertible cubes at all; the input YAML is malformed. + +## Notes and limitations + +- **YAML data models only.** `.js`/`.ts` models and Jinja-templated YAML are + preserved verbatim for the round trip but no cube inside them is converted -- + matching what Cube's own `CubeSchemaConverter` does for the Rollup Designer. +- **camelCase is normalized.** Cube accepts `sqlTable` and `sql_table` alike; + import normalizes to snake_case and export always emits snake_case, so a + camelCase source file comes back snake_cased. +- A filter or computed operand written with bare column names (rather than + `{CUBE}.col`) cannot be qualified into `dataset.column` form, so it is emitted + as-is. Cube's own idiom uses the reference form, which converts fully. +- View curation (`prefix`, `alias`, `includes`/`excludes`, `folders`, + `default_filters`, `view_group`) is stash-and-restore only; Ossie field names + are always *cube* member names, so prefixed view members never leak into them. +- `type: switch` dimensions, `hierarchies`, `pre_aggregations`, `access_policy`, + and multiple `data_source`s have no Ossie semantics and round-trip via the stash. + +## Development + +```bash +uv sync +uv run pytest +``` + +## Future effort + +Both the Apache Ossie specification and Cube's data model are still evolving. As +either side adds or changes fields, this converter will be updated to track them. +Known next steps: the export direction, offline `extends` resolution, and a +first-class Ossie representation for measure additivity so the fan-out caveat can +be recorded in the model instead of an issue log. diff --git a/converters/cube/pyproject.toml b/converters/cube/pyproject.toml new file mode 100644 index 00000000..4b60853e --- /dev/null +++ b/converters/cube/pyproject.toml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "apache-ossie-cube" +version = "0.2.0.dev0" +description = "Bidirectional converter between Apache Ossie semantic models and Cube data models" +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] +requires-python = ">=3.11" +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "Cube", +] +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", +] +dependencies = [ + "PyYAML>=6.0", +] + +[dependency-groups] +dev = [ + "pytest>=8.0", + "hypothesis>=6.0", +] + +[project.scripts] +ossie-cube = "ossie_cube.cli:main" + +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_cube"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] + +[tool.uv] +required-version = ">=0.9.0" +default-groups = [ + "dev", +] diff --git a/converters/cube/src/ossie_cube/__init__.py b/converters/cube/src/ossie_cube/__init__.py new file mode 100644 index 00000000..fcf01fd5 --- /dev/null +++ b/converters/cube/src/ossie_cube/__init__.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Bidirectional converter between Apache Ossie semantic models and Cube data +models. Pure offline transforms: Ossie YAML string <-> {relative filename: YAML +string}. + + from ossie_cube import convert_cube_to_ossie + + ossie_yaml, issues = convert_cube_to_ossie(files) +""" + +from ._common import ConversionError +from .converter_issues import ConverterIssue, IssueLog, IssueType +from .cube_to_osi import convert_cube_to_ossie + +__all__ = [ + "ConversionError", + "ConverterIssue", + "IssueLog", + "IssueType", + "convert_cube_to_ossie", +] diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py new file mode 100644 index 00000000..9bfaee02 --- /dev/null +++ b/converters/cube/src/ossie_cube/_common.py @@ -0,0 +1,593 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared helpers for the Apache Ossie <-> Cube converters. + +Both directions are pure offline YAML transforms. The cross-cutting concerns +live here: version constants, the `custom_extensions` stash protocol, Cube +identifier rules, key-spelling normalization, the type/aggregate mapping tables, +and the member-reference translation between Cube's f-string SQL and the plain +column references Ossie expressions use. +""" + +import json +import re + +import yaml + +# Ossie semantic model spec version this converter targets (see core-spec). +OSSIE_VERSION = "0.2.0.dev0" + +# Vendor id used for the `custom_extensions` stash. +VENDOR = "CUBE" + +# Cube SQL is the SQL of the model's data source, so there is no CUBE entry in +# the Ossie dialect enum. Import emits ANSI_SQL; export prefers ANSI_SQL and lets +# the caller prepend a warehouse dialect the actual data source would accept. +DIALECT_ANSI = "ANSI_SQL" + +# Bump when the shape of a stashed `data` blob changes. +STASH_VERSION = 1 + +# Cube's default data model directory layout (`CUBEJS_SCHEMA_PATH` defaults to +# `model`, and `cube create` scaffolds these two subdirectories). +CUBE_DIR = "model/cubes" +VIEW_DIR = "model/views" + +# A valid Cube identifier -- `identifierRegex` in Cube's CubeValidator. +_CUBE_NAME_RE = re.compile(r"^[_a-zA-Z][_a-zA-Z0-9]*$") + +# A bare SQL identifier (single column reference), e.g. `c_name`. +_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + +# `cube.member` -- a dotted reference an Ossie expression uses to point into a +# dataset. Guarded so `a.b.c` and `1.5` do not match. +DOTTED_REF_RE = re.compile( + r"(? Cube -> Ossie` lossless for models carrying several vendors. + """ + return [ + ext + for ext in (obj or {}).get("custom_extensions") or [] + if ext.get("vendor_name") != VENDOR + ] + + +# --- expressions ---------------------------------------------------------------- + +def pick_expression(ossie_expression, preferred=None): + """Choose the SQL string for an Ossie expression. + + Preference order: the caller-chosen warehouse dialect (Cube passes SQL + through to the data source, so e.g. SNOWFLAKE SQL is valid on a + Snowflake-backed Cube model), then ANSI_SQL. Returns None if neither is + present (the caller records an issue and skips). + """ + dialects = { + d.get("dialect"): d.get("expression") + for d in (ossie_expression or {}).get("dialects") or [] + } + expr = None + if preferred: + expr = dialects.get(preferred) + if expr is None: + expr = dialects.get(DIALECT_ANSI) + if expr is not None and not isinstance(expr, str): + raise ConversionError( + f"expression must be a string, got {type(expr).__name__}") + return expr + + +def synonyms_of(ai_context): + """Extract the synonyms list from an Ossie ai_context (object form only).""" + if isinstance(ai_context, dict): + return list(ai_context.get("synonyms") or []) + return [] + + +def examples_of(ai_context): + if isinstance(ai_context, dict): + return list(ai_context.get("examples") or []) + return [] + + +def instructions_of(ai_context): + """The free-text part of an Ossie ai_context: the string itself, or the + object form's `instructions`.""" + if isinstance(ai_context, str) and ai_context.strip(): + return ai_context + if isinstance(ai_context, dict): + text = ai_context.get("instructions") + if isinstance(text, str) and text.strip(): + return text + return None + + +def cube_sql_to_ossie(sql, own_cube, resolve_ref=None, self_prefix=None): + """Translate Cube member references in a SQL string to the plain references + Ossie expressions use. Returns (translated, changed). + + - `{CUBE}.col` / `{TABLE}.col` -> `col` (a raw column of the own cube) + - `{CUBE.member}` -> `member` (own-cube member reference) + - `{member}` -> `member` (same, unqualified) + - `{other.member}` -> `other.member` + - `{own_cube.member}` -> `member` + + Ossie has no field-vs-column distinction, so both flavors flatten to names. + `\\{` / `\\}` (Cube's escape for a literal brace) survive as plain braces. + + `self_prefix`, when given, qualifies own-cube references with it instead of + reducing them to a bare name -- so `{CUBE}.col` becomes `orders.col`. Ossie + field expressions are dataset-scoped and want the bare form, but model-level + metric expressions address columns as `dataset.column`, so measure conversion + passes the owning cube's name here. + + `resolve_ref`, when given, is called with each raw reference body before the + rules above are applied; returning a string uses it verbatim instead, and + returning None falls through. Measure conversion uses this to inline a + `{other_measure}` reference, which Cube resolves to that measure's own + aggregate SQL and Ossie has no reference form for. + """ + if not isinstance(sql, str): + sql = str(sql) + changed = False + protected = sql.replace("\\{", _ESC_OPEN).replace("\\}", _ESC_CLOSE) + + def repl(m): + nonlocal changed + body = m.group(1).strip() + if resolve_ref is not None: + override = resolve_ref(body) + if override is not None: + changed = True + return override + changed = True + head, _, rest = body.partition(".") + if not rest: + # A lone `{name}`: either `{CUBE}`/`{TABLE}`, the cube's own name + # spelled out, or an unqualified member reference. The first two are + # an alias that a trailing `.column` attaches to, so they are marked + # for removal along with that dot; a member name is an own-cube + # reference. + if body in _SELF_REFS or (own_cube and body == own_cube): + return _SELF_MARK + return f"{self_prefix}.{body}" if self_prefix else body + if head in _SELF_REFS or (own_cube and head == own_cube): + return f"{self_prefix}.{rest}" if self_prefix else rest + return body + + out = _CUBE_REF_RE.sub(repl, protected) + # `{CUBE}.column` -- the alias marker plus the dot the column hangs off. + out = out.replace(f"{_SELF_MARK}.", f"{self_prefix}." if self_prefix else "") + out = out.replace(_SELF_MARK, "") + out = out.replace(_ESC_OPEN, "{").replace(_ESC_CLOSE, "}") + return out, changed + + +def ossie_expr_to_cube_sql(expr, own_cube, own_members=(), cube_names=()): + """Rewrite an Ossie expression into Cube member-reference form. + + Only *dotted* `cube.name` references are rewritten -- a bare identifier stays + bare, because in Ossie it is a physical column of the owning dataset and + rewriting it to `{CUBE.name}` would make a member's own `sql` self-referential. + + A dotted reference resolves to whichever form Cube expects: + - `own_cube.member` where `member` is declared -> `{CUBE.member}` + (compile-time checked, and inlines the member's own SQL) + - `own_cube.column` where it is not -> `{CUBE}.column` + (a raw physical column, passed through to the database) + - `other_cube.member` -> `{other_cube.member}` + (which is also what triggers the implicit join a cross-dataset metric needs) + + The own cube is always referenced as `{CUBE}` rather than by name, so the + model keeps working when the cube is extended. Literal braces in the incoming + expression are escaped. + """ + escaped = str(expr).replace("{", "\\{").replace("}", "\\}") + known = set(cube_names) + members = set(own_members) + + def repl(m): + head, name = m.group(1), m.group(2) + if head == own_cube: + return "{CUBE." + name + "}" if name in members else "{CUBE}." + name + if head in known: + return "{" + head + "." + name + "}" + # Not a dataset in this model -- a genuine schema-qualified table + # reference or an unrelated dotted token. Leave it alone. + return m.group(0) + + return DOTTED_REF_RE.sub(repl, escaped) + + +# --- source --------------------------------------------------------------------- + +def parse_source(source, dataset_name): + """Classify an Ossie dataset `source` for placement on a Cube cube. + + Returns ("sql", sql_text) for a SELECT/WITH subquery source, or + ("sql_table", table_ref) for a table reference. Cube's `sql_table` takes the + reference verbatim (it is interpolated straight into FROM), so no splitting + into catalog/schema/table is needed -- unlike Omni, Cube has no separate + `schema` key, which also means a bare one-part table name is fine. + """ + if not source or not str(source).strip(): + raise ConversionError(f"Dataset '{dataset_name}': missing/empty 'source'") + s = str(source).strip() + if re.match(r"(?i)(select|with)\b", s): + return ("sql", s) + return ("sql_table", s) + + +def join_source(cube, cube_name): + """Rebuild an Ossie dataset `source` string from a Cube cube dict. + + Cube's schema requires exactly one of `sql` / `sql_table` (an `xor` in + CubeValidator), so anything else is rejected rather than guessed at. + """ + sql = cube.get("sql") + table = cube.get("sql_table") + if sql is not None and table is not None: + raise ConversionError( + f"Cube '{cube_name}': has both 'sql' and 'sql_table'; Cube allows " + f"exactly one") + if table is not None: + return str(table).strip() + if sql is not None: + return str(sql).strip() + raise ConversionError( + f"Cube '{cube_name}': has neither 'sql' nor 'sql_table' (an `extends`-only " + f"cube?); Ossie datasets require a source") + + +# --- type mapping --------------------------------------------------------------- + +# Cube dimension `type` -> Ossie `datatype`. `number` is deliberately absent: +# Cube collapses Integer/Decimal/Float into one type, and Ossie says to omit +# `datatype` when it is unknown rather than assert a precision the model does not +# have. (Cube's SQL API reports `number` as Double, but that is a wire-protocol +# floor, not a claim about the column.) `geo` is absent because such a dimension +# is split into two numeric fields. +DIM_TYPE_TO_DATATYPE = { + "string": "String", + "boolean": "Boolean", + "time": "DateTime", + "switch": "String", +} + +# Ossie `datatype` -> Cube dimension `type`, which is required on every +# dimension. Lossy in the numeric and temporal directions by construction. +DATATYPE_TO_DIM_TYPE = { + "String": "string", + "Integer": "number", + "Decimal": "number", + "Float": "number", + "Boolean": "boolean", + "Date": "time", + "Time": "time", + "DateTime": "time", + "DateTimeTz": "time", + "Opaque": "string", +} + +# Ossie datatypes whose temporal role makes `is_time` default to true (spec.md, +# "DataType and is_time"). +TEMPORAL_DATATYPES = frozenset({"Date", "Time", "DateTime", "DateTimeTz"}) + +# Cube measure `type` -> the Ossie aggregate function that reproduces it. +# `count` is absent: it maps through the cube's primary key, see +# primary_key_count_expression(). +AGG_TO_OSSIE_FUNC = { + "sum": "SUM", + "avg": "AVG", + "min": "MIN", + "max": "MAX", + "count_distinct": "COUNT_DISTINCT", + "count_distinct_approx": "APPROX_COUNT_DISTINCT", +} + +OSSIE_FUNC_TO_AGG = { + "SUM": "sum", + "AVG": "avg", + "MIN": "min", + "MAX": "max", + "COUNT_DISTINCT": "count_distinct", + "APPROX_COUNT_DISTINCT": "count_distinct_approx", +} + +# Cube measure types whose aggregation is written out in the `sql` itself +# (CubeSymbols.isCalculatedMeasureType). Their sql is emitted verbatim. +CALCULATED_MEASURE_TYPES = frozenset({"number", "string", "boolean", "time"}) + +# Aggregates whose value is unaffected by duplicate input rows, so a static Ossie +# expression stays correct even when the relationship graph fans the dataset out. +# `count` belongs here only in its bare form, which maps to COUNT(DISTINCT ). +FANOUT_SAFE_AGGS = frozenset({ + "count_distinct", "count_distinct_approx", "min", "max", +}) + +# Aggregates that over-count under row multiplication. Cube corrects for these at +# query time by deduplicating on the primary key; an Ossie expression cannot. +FANOUT_UNSAFE_AGGS = frozenset({"sum", "avg"}) + +# The Ossie result datatype Cube itself declares for each aggregate. Only the +# count family is listed: those are exactly the aggregates whose result type does +# not depend on the operand. +AGG_TO_RESULT_DATATYPE = { + "count": "Integer", + "count_distinct": "Integer", + "count_distinct_approx": "Integer", +} + + +def primary_key_operand(cube_name, primary_keys): + """The single scalar expression standing for a cube's primary key. + + A composite key is concatenated the same way Cube does it (CAST + CONCAT, in + `primaryKeyCount`); both are REQUIRED functions in the Ossie expression + language, so the result stays portable. + """ + if not primary_keys: + raise ConversionError( + f"Cube '{cube_name}': a bare `type: count` measure needs the cube's " + f"primary key to convert safely, but no dimension declares " + f"`primary_key: true`") + if len(primary_keys) == 1: + return f"{cube_name}.{primary_keys[0]}" + parts = ", ".join(f"CAST({cube_name}.{pk} AS VARCHAR)" for pk in primary_keys) + return f"CONCAT({parts})" + + +def primary_key_count_expression(cube_name, primary_keys, filter_exprs=()): + """The Ossie expression for Cube's bare `type: count` measure. + + Cube renders such a measure as `count()` normally and + `count(distinct )` when the cube sits on the multiplied side of a join + (BaseQuery `primaryKeyCount`). `COUNT(DISTINCT )` equals both -- a primary + key is unique, so the DISTINCT is free when there is no fan-out and + load-bearing when there is -- making it the one static form that is correct in + every join context. + """ + operand = filtered_operand(primary_key_operand(cube_name, primary_keys), + filter_exprs) + return f"COUNT(DISTINCT {operand})" + + +def filtered_operand(operand, filter_sqls): + """Fold Cube measure `filters` into the operand, the way Cube itself does. + + Cube's `applyMeasureFilters` wraps the operand as + `CASE WHEN THEN END` inside the aggregate, + which is the filtered-aggregation idiom the Ossie expression language + endorses. The `ELSE` is omitted, matching Cube. + """ + if not filter_sqls: + return operand + where = " AND ".join(f"({f})" for f in filter_sqls) + return f"CASE WHEN {where} THEN {operand} END" diff --git a/converters/cube/src/ossie_cube/cli.py b/converters/cube/src/ossie_cube/cli.py new file mode 100644 index 00000000..51bc336c --- /dev/null +++ b/converters/cube/src/ossie_cube/cli.py @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Command-line interface for the Apache Ossie <-> Cube converter. + + ossie-cube import -i model/ [-o model.yaml] [--name my_model] [--view sales] + +`import` converts a Cube data model directory (any `.yml` holding `cubes:` / +`views:`) into an Apache Ossie semantic model; with no `-o` the Ossie YAML goes to +stdout. Conversions that could not carry something across print an issue list to +stderr. + +By default a metric whose value a static Ossie expression cannot keep correct +under row multiplication is refused, mirroring Cube's own refusal to answer such +a query; pass `--no-strict-fanout` to emit it with a recorded issue instead. +""" + +import argparse +import os +import sys + +from ._common import ConversionError +from .cube_to_osi import convert_cube_to_ossie + + +def _build_parser(): + parser = argparse.ArgumentParser( + prog="ossie-cube", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command") + sub.required = True + + imp = sub.add_parser( + "import", help="Cube data model directory -> Apache Ossie semantic model YAML") + imp.add_argument("-i", "--input", required=True, help="Cube model directory") + imp.add_argument("-o", "--output", + help="output Ossie YAML file (default: stdout)") + imp.add_argument("--name", + help="Ossie model name (default: the mapped view's name)") + imp.add_argument("--view", + help="view whose name/description/AI context map onto the " + "Ossie model (default: the sole view, if there is one)") + imp.add_argument("--no-strict-fanout", dest="strict_fanout", + action="store_false", default=True, + help="record fan-out-unsafe metrics as issues instead of " + "refusing the conversion") + return parser + + +def _read_model_dir(path): + """Collect every file under a Cube model directory as {relative path: text}. + + Everything is collected, not just YAML: a `.js` data model has no Ossie form, + but the converter preserves it so a round trip does not lose the file. Hidden + files and directories (including `node_modules`) are skipped. + """ + if not os.path.isdir(path): + raise ConversionError(f"'{path}' is not a directory") + files = {} + for dirpath, dirnames, filenames in os.walk(path): + dirnames[:] = [d for d in sorted(dirnames) + if not d.startswith(".") and d != "node_modules"] + for fname in sorted(filenames): + if fname.startswith("."): + continue + rel = os.path.relpath(os.path.join(dirpath, fname), path) + rel = rel.replace(os.sep, "/") + with open(os.path.join(dirpath, fname)) as fh: + files[rel] = fh.read() + if not files: + raise ConversionError(f"'{path}' holds no files") + return files + + +def _report(issues): + if not len(issues): + return + print(f"{len(issues)} conversion issue(s):", file=sys.stderr) + for issue in issues: + print(f" {issue}", file=sys.stderr) + + +def main(argv=None): + args = _build_parser().parse_args(argv) + try: + files = _read_model_dir(args.input) + out, issues = convert_cube_to_ossie( + files, model_name=args.name, view=args.view, + strict_fanout=args.strict_fanout) + if args.output: + with open(args.output, "w") as fh: + fh.write(out) + else: + sys.stdout.write(out) + _report(issues) + except (ConversionError, OSError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/converters/cube/src/ossie_cube/converter_issues.py b/converters/cube/src/ossie_cube/converter_issues.py new file mode 100644 index 00000000..64dfe4b1 --- /dev/null +++ b/converters/cube/src/ossie_cube/converter_issues.py @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Structured record of what a conversion could not carry across. + +A bare `warnings.warn` is fine for "this label was dropped", but Cube carries +semantics that an Apache Ossie expression string genuinely cannot hold -- most +importantly the row-multiplication correction Cube applies at query time (see +`FANOUT_UNSAFE_METRIC`). Those need to reach the caller as data, not as text on +stderr, so a pipeline can gate on them. Same approach as the osi-dbt converter's +`ConverterIssue`. +""" + +from dataclasses import dataclass, field +from enum import Enum + + +class IssueType(Enum): + """Identifies the kind of information loss that occurred during conversion.""" + + # A non-idempotent aggregate (sum/avg, or count over an expression) on a + # dataset that the relationship graph can fan out. Cube corrects for this at + # query time by deduplicating on the primary key; a static Ossie expression + # cannot, so a downstream consumer may over-count. See README "Fan-out". + FANOUT_UNSAFE_METRIC = "FANOUT_UNSAFE_METRIC" + + # A `multi_stage` measure (group_by / reduce_by / time_shift / rank). These + # render as window functions over a grain other than the query's, which an + # Ossie expression has no form for; the measure is preserved in the stash + # and omitted from `metrics`. + MULTI_STAGE_MEASURE_DROPPED = "MULTI_STAGE_MEASURE_DROPPED" + + # A cube-level `meta.ai_context`. Cube's own agent only consumes ai_context + # on views and on individual members, so this value is inert in Cube; it is + # preserved so the round trip stays lossless. + CUBE_LEVEL_AI_CONTEXT_INERT = "CUBE_LEVEL_AI_CONTEXT_INERT" + + # A `type: geo` dimension, split into two Ossie fields (latitude/longitude) + # because an Ossie field holds a single expression. + GEO_DIMENSION_SPLIT = "GEO_DIMENSION_SPLIT" + + # A dimension or measure whose `sql` uses Jinja templating, or a cube using + # `extends`: no static form, so it is preserved in the stash only. + TEMPLATED_MEMBER_DROPPED = "TEMPLATED_MEMBER_DROPPED" + + # An Ossie field or metric with no usable expression dialect (export). + NO_USABLE_DIALECT = "NO_USABLE_DIALECT" + + # An Ossie construct Cube has no slot for, parked under `meta.ossie`. + PARKED_IN_META = "PARKED_IN_META" + + +@dataclass(frozen=True) +class ConverterIssue: + """One instance of information loss, addressed to a named element.""" + + issue_type: IssueType + element_name: str + detail: str = "" + + def __str__(self): + suffix = f": {self.detail}" if self.detail else "" + return f"[{self.issue_type.value}] {self.element_name}{suffix}" + + +@dataclass +class IssueLog: + """Collects issues during a conversion. + + `strict_types` names the issue types that should abort the conversion + instead of being recorded. The CLI puts `FANOUT_UNSAFE_METRIC` in there by + default, mirroring Cube's own refusal to answer a query whose measures + reference cubes that lead to row multiplication. + """ + + issues: list = field(default_factory=list) + strict_types: frozenset = frozenset() + + def add(self, issue_type, element_name, detail=""): + issue = ConverterIssue(issue_type, element_name, detail) + if issue_type in self.strict_types: + # Imported here to avoid a circular import at module load. + from ._common import ConversionError + + raise ConversionError(f"{issue} (refused under strict mode)") + self.issues.append(issue) + return issue + + def of_type(self, issue_type): + return [i for i in self.issues if i.issue_type is issue_type] + + def __len__(self): + return len(self.issues) + + def __iter__(self): + return iter(self.issues) diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py new file mode 100644 index 00000000..6e165e8e --- /dev/null +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -0,0 +1,839 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert a Cube data model to an Apache Ossie semantic model. + +Pure offline conversion -- no Cube deployment required. Accepts a Cube model +directory as {relative filename: YAML string}: any `.yml`/`.yaml` file holding +top-level `cubes:` and/or `views:`. Cubes become Ossie datasets, cube joins +become relationships, cube measures are hoisted to model-level metrics, and the +mapped view supplies the model's name, description, and AI context. + +Cube features Ossie has no native field for (segments, pre-aggregations, +hierarchies, folders, view curation, formats, access policies, ...) are preserved +in `custom_extensions[CUBE]` so that converting back reproduces the original +files. See README.md. + +Usage (CLI): + ossie-cube import -i model/ [-o model.yaml] [--name NAME] [--view VIEW] +""" + +import re + +from ._common import ( + AGG_TO_OSSIE_FUNC, + AGG_TO_RESULT_DATATYPE, + CALCULATED_MEASURE_TYPES, + DIALECT_ANSI, + DIM_TYPE_TO_DATATYPE, + DOTTED_REF_RE, + FANOUT_UNSAFE_AGGS, + JINJA_RE, + OSSIE_VERSION, + ConversionError, + cube_file, + cube_sql_to_ossie, + dump_yaml, + filtered_operand, + is_simple_identifier, + join_source, + load_yaml, + primary_key_count_expression, + require_str, + snake, + snake_keys, + view_file, + write_stash, +) +from .converter_issues import IssueLog, IssueType + +# Cube keys the converter maps natively at the cube level; everything else is +# stashed verbatim in the dataset's `cube_extras` and restored on export. +_CUBE_NATIVE_KEYS = frozenset({ + "name", "sql", "sql_table", "description", "dimensions", "measures", + "joins", "meta", +}) + +# Dimension keys mapped natively; the rest stash flat on the field. +_DIM_NATIVE_KEYS = frozenset({ + "name", "sql", "type", "primary_key", "title", "description", "meta", + "latitude", "longitude", +}) + +# Measure keys an Ossie metric represents natively. Any other key forces the +# full-measure stash, because export could not rebuild the measure without it. +_MEASURE_NATIVE_KEYS = frozenset({ + "name", "sql", "type", "filters", "title", "description", "meta", +}) + +# `relationship` values, normalized. Cube accepts the legacy `belongsTo` / +# `hasMany` / `hasOne` spellings alongside the modern ones, in either case style. +_RELATIONSHIP_ALIASES = { + "belongs_to": "many_to_one", + "many_to_one": "many_to_one", + "has_many": "one_to_many", + "one_to_many": "one_to_many", + "has_one": "one_to_one", + "one_to_one": "one_to_one", +} + +_AND_SPLIT_RE = re.compile(r"\s+AND\s+", re.IGNORECASE) + + +def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=True): + """Convert Cube model files ({relative filename: YAML str}) to Ossie YAML. + + Returns (ossie_yaml_str, IssueLog). `model_name` overrides the Ossie model + name (default: the mapped view's name, else 'cube_model'). `view` names the + view whose name/description/AI context map onto the Ossie model when the + directory holds more than one. `strict_fanout` refuses metrics whose value a + static Ossie expression cannot keep correct under row multiplication -- see + README "Fan-out". + """ + if not isinstance(files, dict) or not files: + raise ConversionError("expected a non-empty mapping of {filename: YAML}") + + strict = {IssueType.FANOUT_UNSAFE_METRIC} if strict_fanout else set() + issues = IssueLog(strict_types=frozenset(strict)) + + cubes, cube_paths, views, view_paths, extra_files = _collect(files, issues) + if not cubes: + raise ConversionError( + "no convertible cubes found (a `.yml` file with a top-level `cubes:` " + "list); nothing to convert") + + # The mapped view supplies the Ossie model's identity. Cube users are + # view-first, and Cube's own agent reads `meta.ai_context` only from views and + # individual members -- so the view, not any cube, is the model boundary. + mapped_name = _pick_view(views, view, issues) + mapped_view = views.get(mapped_name) or {} + + model = {"name": model_name or mapped_name or "cube_model"} + if mapped_view.get("description"): + model["description"] = mapped_view["description"] + ai = _ai_context_from_meta(mapped_view.get("meta")) + if ai: + model["ai_context"] = ai + + # Joins are decomposed first: a join with no Ossie form is parked on its + # declaring cube's stash, which has to be known before the dataset is built. + relationships, extra_joins = _convert_joins(cubes, issues) + + datasets = [] + pk_by_cube = {} + for cname, cube in cubes.items(): + ds, primary_key = _convert_cube(cname, cube, extra_joins.get(cname), issues) + datasets.append(ds) + pk_by_cube[cname] = primary_key + model["datasets"] = datasets + if relationships: + model["relationships"] = relationships + + # A dataset on the `to` (one) side of a relationship can be fanned out by rows + # from the `from` (many) side. Derived entirely from the Ossie graph. + fanned_out = {rel["to"]: rel["name"] for rel in relationships} + + metrics = _convert_measures(cubes, pk_by_cube, fanned_out, issues) + if metrics: + model["metrics"] = metrics + + # Model-level stash: the views verbatim (minus natively mapped properties), + # the mapped view's identity, non-canonical file paths, and any file with no + # Ossie form. `views` is stashed even when empty, so a lossless re-export does + # not invent a view the original model never had. + stash = {"views": {}} + for vname, vdict in views.items(): + vdict = dict(vdict) + if vname == mapped_name: + vdict.pop("description", None) + leftover = _meta_without_ai_context(vdict.get("meta")) + vdict.pop("meta", None) + if leftover: + vdict["meta"] = leftover + stash["views"][vname] = vdict + off_layout_views = {v: p for v, p in view_paths.items() if p != view_file(v)} + if off_layout_views: + stash["view_files"] = off_layout_views + off_layout_cubes = {c: p for c, p in cube_paths.items() if p != cube_file(c)} + if off_layout_cubes: + stash["cube_files"] = off_layout_cubes + if mapped_name is not None: + stash["mapped_view"] = mapped_name + if extra_files: + stash["extra_files"] = extra_files + write_stash(model, stash) + + return dump_yaml({"version": OSSIE_VERSION, "semantic_model": [model]}), issues + + +# --- collection ----------------------------------------------------------------- + +def _collect(files, issues): + """Partition the input files into cubes, views, and everything else.""" + cubes, views = {}, {} + cube_paths, view_paths = {}, {} + extra_files = {} + for fname in sorted(files): + text = files[fname] + if not fname.lower().endswith((".yml", ".yaml")): + # A `.js`/`.ts` data model needs Cube's own transpiler and a `.py` one + # is Jinja-driven. Preserved verbatim so the round trip keeps the file, + # but no cube inside it is converted. + issues.add(IssueType.TEMPLATED_MEMBER_DROPPED, fname, + "not a YAML data model; preserved in custom_extensions only") + extra_files[fname] = text + continue + if JINJA_RE.search(text): + issues.add(IssueType.TEMPLATED_MEMBER_DROPPED, fname, + "uses Jinja templating, which has no static form; " + "preserved in custom_extensions only") + extra_files[fname] = text + continue + parsed = load_yaml(text, fname) + if not isinstance(parsed, dict) or not ("cubes" in parsed or "views" in parsed): + issues.add(IssueType.PARKED_IN_META, fname, + "no top-level `cubes:` or `views:`; preserved in " + "custom_extensions only") + extra_files[fname] = text + continue + for entry in _as_named_list(parsed.get("cubes"), f"'{fname}' cubes"): + name = require_str(entry, "name", f"'{fname}': cube") + if name in cubes: + raise ConversionError( + f"cube '{name}' is defined twice " + f"('{cube_paths[name]}' and '{fname}')") + if "extends" in entry: + # Resolving `extends` means reproducing Cube's definition-merge + # semantics exactly; refused rather than half-applied. + raise ConversionError( + f"cube '{name}' uses `extends`, which this converter does not " + f"resolve yet; flatten the cube or exclude the file") + cubes[name] = entry + cube_paths[name] = fname + for entry in _as_named_list(parsed.get("views"), f"'{fname}' views"): + name = require_str(entry, "name", f"'{fname}': view") + if name in views: + raise ConversionError( + f"view '{name}' is defined twice " + f"('{view_paths[name]}' and '{fname}')") + views[name] = entry + view_paths[name] = fname + return cubes, cube_paths, views, view_paths, extra_files + + +def _as_named_list(value, what): + """Normalize a Cube collection to a list of dicts carrying `name`. + + YAML data models write `cubes:` / `dimensions:` / `joins:` as lists whose + entries carry a `name`; the JavaScript form (and Cube's post-transpile schema) + uses a mapping keyed by name. Both are accepted, and keys are normalized to + snake_case so the mapping code only has to know one spelling. + """ + if value is None: + return [] + if isinstance(value, list): + out = [] + for entry in value: + if not isinstance(entry, dict): + raise ConversionError( + f"{what}: expected a mapping, got {type(entry).__name__}") + out.append(snake_keys(entry)) + return out + if isinstance(value, dict): + out = [] + for name, entry in value.items(): + entry = snake_keys(entry or {}) + entry.setdefault("name", name) + out.append(entry) + return out + raise ConversionError( + f"{what}: expected a list or mapping, got {type(value).__name__}") + + +def _pick_view(views, requested, issues): + if requested is not None: + if requested not in views: + raise ConversionError( + f"requested view '{requested}' not found; views present: " + f"{sorted(views) or 'none'}") + return requested + if len(views) == 1: + return next(iter(views)) + if len(views) > 1: + issues.add(IssueType.PARKED_IN_META, "model", + f"{len(views)} views found and none chosen with --view; view " + f"metadata is preserved in custom_extensions only") + return None + + +# --- ai_context ----------------------------------------------------------------- + +def _ai_context_from_meta(meta): + """Build an Ossie `ai_context` from a Cube `meta`. + + `meta.ai_context` is Cube's documented AI-only context field. A structured + copy parked by a previous export under `meta.ossie.ai_context` wins, since it + carries the synonyms/examples lists that the prose form flattens. + """ + if not isinstance(meta, dict): + return None + parked = (meta.get("ossie") or {}).get("ai_context") + if parked: + return parked + text = meta.get("ai_context") + if isinstance(text, str) and text.strip(): + return {"instructions": text.strip()} + return None + + +def _meta_without_ai_context(meta): + """The part of a Cube `meta` with no Ossie home, for the stash. + + `meta.ossie` is this converter's own parking spot; its contents are restored + into native Ossie fields, so it never rides in the stash. + """ + if not isinstance(meta, dict): + return {} + return {k: v for k, v in meta.items() if k not in ("ai_context", "ossie")} + + +# --- cubes ---------------------------------------------------------------------- + +def _convert_cube(cname, cube, extra_joins, issues): + """Build one Ossie dataset from a Cube cube. Returns (dataset, primary_key).""" + scope = f"cube '{cname}'" + ds = {"name": cname} + stash = {} + + ds["source"] = join_source(cube, cname) + if cube.get("description"): + ds["description"] = cube["description"] + + meta = cube.get("meta") if isinstance(cube.get("meta"), dict) else {} + parked = meta.get("ossie") or {} + ai = _ai_context_from_meta(meta) + if ai: + ds["ai_context"] = ai + if meta.get("ai_context"): + issues.add(IssueType.CUBE_LEVEL_AI_CONTEXT_INERT, scope, + "Cube's agent reads ai_context only on views and members, " + "so a cube-level value has no effect in Cube") + if parked.get("unique_keys"): + ds["unique_keys"] = [list(k) for k in parked["unique_keys"]] + + fields = [] + primary_key = [] + templated = {} + for dim in _as_named_list(cube.get("dimensions"), f"{scope} dimensions"): + dname = require_str(dim, "name", f"{scope}: dimension") + if JINJA_RE.search(str(dim.get("sql", ""))): + issues.add(IssueType.TEMPLATED_MEMBER_DROPPED, f"{cname}.{dname}", + "dimension sql uses Jinja templating; preserved in " + "custom_extensions only") + templated[dname] = dim + continue + if dim.get("primary_key"): + primary_key.append(dname) + fields.extend(_convert_dimension(cname, dname, dim, issues)) + if fields: + ds["fields"] = fields + if primary_key: + ds["primary_key"] = primary_key + if templated: + stash["extra_dimensions"] = templated + if extra_joins: + stash["extra_joins"] = extra_joins + + extras = {snake(k): v for k, v in cube.items() + if snake(k) not in _CUBE_NATIVE_KEYS} + leftover_meta = _meta_without_ai_context(cube.get("meta")) + if leftover_meta: + extras["meta"] = leftover_meta + if extras: + stash["cube_extras"] = extras + write_stash(ds, stash) + + # Foreign-vendor extensions parked by a previous export are restored after the + # stash is written, so the CUBE entry stays first and both survive. + if parked.get("custom_extensions"): + ds.setdefault("custom_extensions", []).extend(parked["custom_extensions"]) + return ds, primary_key + + +def _convert_dimension(cname, dname, dim, issues): + """Build the Ossie field(s) for one Cube dimension. + + Returns a list because a `type: geo` dimension carries two SQL expressions + (latitude and longitude) where an Ossie field holds one, so it splits into two + fields. Every other dimension yields exactly one. + """ + dtype = snake(dim.get("type") or "string") + if dtype == "geo": + return _convert_geo_dimension(cname, dname, dim, issues) + + stash = {} + sql = dim.get("sql") + if sql is None: + # No `sql` means the same-named physical column. + expr = dname + else: + expr, changed = cube_sql_to_ossie(sql, cname) + if changed or str(sql).strip() == dname: + # Stashed when the Ossie expression differs from the Cube sql, and also + # when the sql is an explicit same-named bare column -- which export + # would otherwise normalize away to the implicit form. + stash["sql"] = sql + + field = { + "name": dname, + "expression": {"dialects": [{"dialect": DIALECT_ANSI, "expression": expr}]}, + } + datatype = DIM_TYPE_TO_DATATYPE.get(dtype) + if datatype: + field["datatype"] = datatype + elif dtype == "number": + # Cube collapses Integer/Decimal/Float into `number`, so no Ossie datatype + # is asserted -- the spec says to omit it when unknown. The original type + # rides in the stash so export reproduces it. + stash["type"] = dtype + else: + raise ConversionError( + f"cube '{cname}': dimension '{dname}' has unknown type '{dtype}'") + if dtype == "time": + field["dimension"] = {"is_time": True} + if dim.get("title"): + field["label"] = dim["title"] + if dim.get("description"): + field["description"] = dim["description"] + ai = _ai_context_from_meta(dim.get("meta")) + if ai: + field["ai_context"] = ai + + for key, value in dim.items(): + skey = snake(key) + if skey not in _DIM_NATIVE_KEYS: + stash[skey] = value + leftover_meta = _meta_without_ai_context(dim.get("meta")) + if leftover_meta: + stash["meta"] = leftover_meta + write_stash(field, stash) + return [field] + + +def _convert_geo_dimension(cname, dname, dim, issues): + """Split a `type: geo` dimension into a latitude and a longitude field. + + The reconstruction data rides on the latitude half (`geo.host` holds the + dimension's other keys), so export can rebuild the single geo dimension. + """ + issues.add(IssueType.GEO_DIMENSION_SPLIT, f"{cname}.{dname}", + f"split into '{dname}_latitude' and '{dname}_longitude'; an Ossie " + f"field holds a single expression") + host_extras = { + snake(k): v for k, v in dim.items() + if snake(k) not in ("name", "type", "latitude", "longitude") + } + out = [] + for part in ("latitude", "longitude"): + sub = (dim.get(part) or {}).get("sql") + if sub is None: + raise ConversionError( + f"cube '{cname}': geo dimension '{dname}' is missing '{part}.sql'") + expr, _ = cube_sql_to_ossie(sub, cname) + field = { + "name": f"{dname}_{part}", + "expression": { + "dialects": [{"dialect": DIALECT_ANSI, "expression": expr}] + }, + "datatype": "Float", + } + geo = {"of": dname, "part": part, "sql": sub} + if part == "latitude" and host_extras: + geo["host"] = host_extras + write_stash(field, {"geo": geo}) + out.append(field) + return out + + +# --- joins ---------------------------------------------------------------------- + +def _convert_joins(cubes, issues): + """Turn every cube's `joins` into Ossie relationships. + + Ossie's `from` is always the many side. A `many_to_one` join declared on cube + A points A(many) -> B(one) directly; a `one_to_many` join is flipped, and the + declared side and type are stashed so export restores the original. + + Returns (relationships, {cube name: [unconvertible join, ...]}). + """ + relationships = [] + extra_joins = {} + taken = set() + for cname, cube in cubes.items(): + for index, join in enumerate( + _as_named_list(cube.get("joins"), f"cube '{cname}' joins")): + target = require_str(join, "name", f"cube '{cname}': join") + what = f"join '{cname}' -> '{target}'" + if target not in cubes: + raise ConversionError( + f"{what}: '{target}' is not a cube in this model") + raw_rel = snake(require_str(join, "relationship", what)) + rel_type = _RELATIONSHIP_ALIASES.get(raw_rel) + if rel_type is None: + raise ConversionError( + f"{what}: unknown relationship '{join['relationship']}'") + sql = require_str(join, "sql", what) + + pairs = _decompose_join_sql(sql, cname, target, what, issues) + if pairs is None: + extra_joins.setdefault(cname, []).append( + {"index": index, "join": join}) + continue + + from_cube, to_cube = cname, target + from_cols = [p[0] for p in pairs] + to_cols = [p[1] for p in pairs] + stash = {"declared_on": cname, "relationship": raw_rel} + if rel_type == "one_to_many": + from_cube, to_cube = to_cube, from_cube + from_cols, to_cols = to_cols, from_cols + elif rel_type == "one_to_one": + # Neither side multiplies, so Ossie's many/one orientation is not + # meaningful; the declared orientation is kept. + issues.add(IssueType.PARKED_IN_META, what, + "one_to_one has no Ossie orientation; the declared " + "orientation is kept and the type preserved") + if sql != _rebuild_join_sql(target, pairs): + stash["sql"] = sql + for key, value in join.items(): + if snake(key) not in ("name", "sql", "relationship"): + stash[snake(key)] = value + + # Ossie relationship names are unique per model; several joins between + # one cube pair would generate the same `_to_`, so repeats + # are suffixed. Export never reads the name, so this stays lossless. + name = f"{from_cube}_to_{to_cube}" + base, k = name, 2 + while name in taken: + name, k = f"{base}_{k}", k + 1 + taken.add(name) + + rel = {"name": name, "from": from_cube, "to": to_cube, + "from_columns": from_cols, "to_columns": to_cols} + write_stash(rel, stash) + relationships.append(rel) + return relationships, extra_joins + + +def _decompose_join_sql(sql, own_cube, target, what, issues): + """Split a Cube join `sql` into (own_column, target_column) pairs. + + Only an AND-chain of equalities between one own-cube reference and one + target-cube reference has an Ossie relationship form. Anything else -- a + range/non-equi condition, a comparison against a literal, a third cube -- + returns None, and the caller preserves the join in the stash instead. + """ + pairs = [] + for clause in _AND_SPLIT_RE.split(sql): + sides = clause.split("=") + if len(sides) != 2: + issues.add(IssueType.PARKED_IN_META, what, + f"join clause '{clause.strip()}' is not a single equality; " + f"preserved in custom_extensions only") + return None + left = _ref_target(sides[0], own_cube, target) + right = _ref_target(sides[1], own_cube, target) + if left is None or right is None: + issues.add(IssueType.PARKED_IN_META, what, + f"join clause '{clause.strip()}' is not between two member " + f"references; preserved in custom_extensions only") + return None + (lcube, lcol), (rcube, rcol) = left, right + if lcube == own_cube and rcube == target: + pairs.append((lcol, rcol)) + elif lcube == target and rcube == own_cube: + pairs.append((rcol, lcol)) + else: + issues.add(IssueType.PARKED_IN_META, what, + f"join clause '{clause.strip()}' references cubes other than " + f"'{own_cube}'/'{target}'; preserved in custom_extensions only") + return None + return pairs or None + + +def _ref_target(side, own_cube, target): + """Resolve one side of a join equality to (cube_name, column), or None.""" + translated, _ = cube_sql_to_ossie(side, own_cube) + translated = translated.strip() + if is_simple_identifier(translated): + # A bare name came from `{CUBE}.col`, `{CUBE.col}`, or `{col}` -- all of + # which address the cube the join is declared on. + return (own_cube, translated) + m = DOTTED_REF_RE.fullmatch(translated) + if m and m.group(1) in (own_cube, target): + return (m.group(1), m.group(2)) + return None + + +def _rebuild_join_sql(target, pairs): + """The canonical form export emits, used to decide whether the original has to + be stashed. The own side is always `{CUBE}` so the join keeps working when the + cube is extended.""" + return " AND ".join( + "{CUBE}." + own + " = {" + target + "." + other + "}" + for own, other in pairs + ) + + +# --- measures ------------------------------------------------------------------- + +class _MeasureResolver: + """Computes the Ossie expression for a Cube measure. + + Kept as a class because a calculated measure (`type: number`, and the other + types in `CALCULATED_MEASURE_TYPES`) can reference other measures, which Cube + resolves by inlining their full aggregate SQL -- so producing one measure's + expression may require producing another's first. Results are memoized and + reference cycles are rejected rather than recursed into. + """ + + def __init__(self, cubes, pk_by_cube, issues): + self._pk = pk_by_cube + self._issues = issues + self._raw = {} + self._dimensions = {} + for cname, cube in cubes.items(): + for m in _as_named_list(cube.get("measures"), f"cube '{cname}' measures"): + self._raw[(cname, require_str(m, "name", f"cube '{cname}': measure"))] = m + self._dimensions[cname] = { + d["name"] + for d in _as_named_list(cube.get("dimensions"), + f"cube '{cname}' dimensions") + } + + def measures(self): + return self._raw + + def is_measure(self, cube, name): + return (cube, name) in self._raw + + def aggregate_of(self, cname, mname): + """The normalized Cube `type` of a measure.""" + return snake(self._raw[(cname, mname)].get("type") or "") + + def expression(self, cname, mname, stack=()): + """The Ossie expression reproducing this measure, or None when the measure + has no static form (multi-stage, Jinja-templated).""" + key = (cname, mname) + if key in stack: + chain = " -> ".join(f"{c}.{m}" for c, m in stack + (key,)) + raise ConversionError(f"measure reference cycle: {chain}") + measure = self._raw[key] + scope = f"{cname}.{mname}" + mtype = snake(measure.get("type") or "") + if not mtype: + raise ConversionError(f"measure '{scope}': missing required 'type'") + + if measure.get("multi_stage"): + # group_by / reduce_by / time_shift / rank render as window functions + # over a grain other than the query's; Ossie has no form for that. + self._issues.add( + IssueType.MULTI_STAGE_MEASURE_DROPPED, scope, + f"multi_stage measure (type '{mtype}'); preserved in " + f"custom_extensions only") + return None + if JINJA_RE.search(str(measure.get("sql", ""))): + self._issues.add( + IssueType.TEMPLATED_MEMBER_DROPPED, scope, + "measure sql uses Jinja templating; preserved in " + "custom_extensions only") + return None + + sql = measure.get("sql") + filter_exprs = [ + self._translate(f["sql"], cname, stack + (key,)) + for f in (measure.get("filters") or []) + if isinstance(f, dict) and f.get("sql") + ] + + if mtype in CALCULATED_MEASURE_TYPES: + if sql is None: + raise ConversionError( + f"measure '{scope}': type '{mtype}' requires 'sql'") + expr = self._translate(sql, cname, stack + (key,)) + return filtered_operand(expr, filter_exprs) + if mtype == "count": + if sql is None: + return primary_key_count_expression( + cname, self._pk.get(cname) or [], filter_exprs) + operand = filtered_operand( + self._operand(cname, sql, stack + (key,)), filter_exprs) + return f"COUNT({operand})" + func = AGG_TO_OSSIE_FUNC.get(mtype) + if func is None: + raise ConversionError( + f"measure '{scope}': unknown aggregate type '{mtype}'") + if sql is None: + raise ConversionError( + f"measure '{scope}': type '{mtype}' requires 'sql'") + operand = filtered_operand( + self._operand(cname, sql, stack + (key,)), filter_exprs) + return (f"COUNT(DISTINCT {operand})" if func == "COUNT_DISTINCT" + else f"{func}({operand})") + + def _translate(self, sql, cname, stack): + """Translate a Cube SQL string, inlining any measure reference. + + `self_prefix` is the owning cube: Ossie metrics are model-level, so a + column reads as `dataset.column` here, unlike in a dataset-scoped field + expression. + """ + out, _ = cube_sql_to_ossie( + sql, cname, resolve_ref=lambda body: self._inline(body, cname, stack), + self_prefix=cname) + return out + + def _inline(self, body, cname, stack): + """Resolve one `{...}` body when it names a measure, else fall through. + + Cube inlines a measure reference to that measure's own aggregate SQL + (`isCalculatedMeasureType` emits the sql as-is), so `{revenue} / {count}` + becomes a complete ratio expression -- which is exactly the shape Ossie + metrics use. Parenthesized to keep the referenced measure's precedence. + """ + head, _, rest = body.partition(".") + if rest: + target_cube = cname if head in ("CUBE", "TABLE") else head + target_name = rest + else: + target_cube, target_name = cname, body + if not self.is_measure(target_cube, target_name): + return None + inner = self.expression(target_cube, target_name, stack) + if inner is None: + raise ConversionError( + f"measure '{cname}': references '{target_cube}.{target_name}', " + f"which has no static Ossie form") + return f"({inner})" + + def _operand(self, cname, sql, stack): + """Translate an aggregate's operand into an Ossie reference. + + A same-cube member or bare column becomes `cube.name` -- the qualified form + Ossie model-level metrics use. A computed operand keeps its own qualifiers + and is emitted as-is; the owning cube rides in the stash either way, so + export still puts the measure back on the right cube. + """ + translated = self._translate(sql, cname, stack).strip() + if is_simple_identifier(translated): + return f"{cname}.{translated}" + return translated + + +def _convert_measures(cubes, pk_by_cube, fanned_out, issues): + """Hoist every cube's measures into Ossie model-level metrics. + + A metric name is the measure name when globally unique, else + `__`; the original name and owning cube are stashed so export + puts the measure back where it came from. + """ + resolver = _MeasureResolver(cubes, pk_by_cube, issues) + + counts = {} + for (_cname, mname) in resolver.measures(): + counts[mname] = counts.get(mname, 0) + 1 + + metrics = [] + seen = set() + for cname, cube in cubes.items(): + for measure in _as_named_list(cube.get("measures"), + f"cube '{cname}' measures"): + mname = measure["name"] + metric_name = mname if counts[mname] == 1 else f"{cname}__{mname}" + if metric_name in seen: + raise ConversionError( + f"metric name '{metric_name}' derived twice; rename the " + f"colliding measures in Cube") + seen.add(metric_name) + metric = _convert_measure(cname, mname, metric_name, measure, resolver, + fanned_out, issues) + if metric is not None: + metrics.append(metric) + return metrics + + +def _convert_measure(cname, mname, metric_name, measure, resolver, fanned_out, + issues): + scope = f"{cname}.{mname}" + expr = resolver.expression(cname, mname) + if expr is None: + # No static form; the resolver already recorded why. + return None + mtype = resolver.aggregate_of(cname, mname) + sql = measure.get("sql") + + # Reconstructible = export can rebuild this measure from the Ossie expression + # alone. A calculated measure never is: export would re-parse its expression + # into a structured measure, and the inlined references cannot be un-inlined. + # Neither is a filtered one -- recovering `filters` would mean parsing the + # folded CASE back apart, so the original rides along instead. + reconstructible = ( + {snake(k) for k in measure} <= _MEASURE_NATIVE_KEYS + and mtype not in CALCULATED_MEASURE_TYPES + and not measure.get("filters") + ) + + # Fan-out: a non-idempotent aggregate on a dataset the graph can multiply. + # Cube fixes this at query time by deduplicating on the primary key; a static + # expression cannot, so the caller has to be told. + unsafe = mtype in FANOUT_UNSAFE_AGGS or (mtype == "count" and sql is not None) + if unsafe and cname in fanned_out: + issues.add( + IssueType.FANOUT_UNSAFE_METRIC, scope, + f"'{mtype}' over dataset '{cname}', which relationship " + f"'{fanned_out[cname]}' fans out; Cube deduplicates on the primary key " + f"at query time but a static Ossie expression cannot, so a consumer " + f"joining through that relationship may over-count") + + metric = { + "name": metric_name, + "expression": {"dialects": [{"dialect": DIALECT_ANSI, "expression": expr}]}, + } + datatype = AGG_TO_RESULT_DATATYPE.get(mtype) + if datatype: + metric["datatype"] = datatype + if measure.get("description"): + metric["description"] = measure["description"] + ai = _ai_context_from_meta(measure.get("meta")) + if ai: + metric["ai_context"] = ai + + stash = {"cube": cname} + if not reconstructible: + stash["measure"] = { + snake(k): v for k, v in measure.items() + if snake(k) not in ("description", "meta") + } + if metric_name != mname: + stash["name"] = mname + if measure.get("title"): + stash["title"] = measure["title"] + leftover_meta = _meta_without_ai_context(measure.get("meta")) + if leftover_meta: + stash["meta"] = leftover_meta + write_stash(metric, stash) + return metric diff --git a/converters/cube/tests/_util.py b/converters/cube/tests/_util.py new file mode 100644 index 00000000..feb7e253 --- /dev/null +++ b/converters/cube/tests/_util.py @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared test helpers: fixture loading and structural lookup.""" + +import copy +import json +import pathlib + +from ossie_cube._common import load_yaml # src is on sys.path via conftest.py + +FIXTURES = pathlib.Path(__file__).resolve().parent / "fixtures" +REPO_ROOT = pathlib.Path(__file__).resolve().parents[3] + + +def load_fixture(name): + with open(FIXTURES / name) as fh: + return fh.read() + + +def load_fixture_dir(name): + """Read a fixture Cube model directory as {relative posix path: text}.""" + root = FIXTURES / name + files = {} + for path in sorted(root.rglob("*")): + if path.is_file(): + files[path.relative_to(root).as_posix()] = path.read_text() + return files + + +def parse(yaml_str): + return load_yaml(yaml_str) + + +def model_of(ossie_yaml): + """The sole semantic model of an Ossie document.""" + doc = parse(ossie_yaml) + assert len(doc["semantic_model"]) == 1 + return doc["semantic_model"][0] + + +def by_name(items): + """Index a list of named Ossie objects by `name`.""" + return {item["name"]: item for item in items or []} + + +def expr_of(item, dialect="ANSI_SQL"): + """The expression string of an Ossie field or metric in a given dialect.""" + for entry in item["expression"]["dialects"]: + if entry["dialect"] == dialect: + return entry["expression"] + raise AssertionError(f"{item['name']} has no {dialect} expression") + + +def stash_of(item, vendor="CUBE"): + """The parsed vendor stash on an Ossie object, or {} when absent.""" + for ext in item.get("custom_extensions") or []: + if ext["vendor_name"] == vendor: + data = json.loads(ext["data"]) + data.pop("_v", None) + return data + return {} + + +def canon(obj): + """Deep-copy with every `custom_extensions[].data` JSON string parsed into a + dict, so comparisons are insensitive to JSON key order and whitespace.""" + obj = copy.deepcopy(obj) + + def walk(node): + if isinstance(node, dict): + for key, value in node.items(): + if key == "custom_extensions" and isinstance(value, list): + for ext in value: + if isinstance(ext, dict) and isinstance(ext.get("data"), str): + ext["data"] = json.loads(ext["data"]) + else: + walk(value) + elif isinstance(node, list): + for item in node: + walk(item) + + walk(obj) + return obj diff --git a/converters/cube/tests/conftest.py b/converters/cube/tests/conftest.py new file mode 100644 index 00000000..254b3d75 --- /dev/null +++ b/converters/cube/tests/conftest.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pathlib +import sys + +# Make the converter modules in ../src, and this directory's own helpers, +# importable from the tests. +_HERE = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(_HERE)) +sys.path.insert(0, str(_HERE.parent / "src")) diff --git a/converters/cube/tests/fixtures/fixtureA_cube/model/cubes/orders.yml b/converters/cube/tests/fixtures/fixtureA_cube/model/cubes/orders.yml new file mode 100644 index 00000000..a28395ab --- /dev/null +++ b/converters/cube/tests/fixtures/fixtureA_cube/model/cubes/orders.yml @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The fact cube: sits on the many side of the join, so its sum/avg measures are +# not exposed to row multiplication. Exercises a bare `count` (which maps through +# the primary key), a filtered sum, a calculated measure that references two other +# measures, and `meta.ai_context` on both a dimension and a measure. +cubes: + - name: orders + sql_table: public.orders + description: Customer orders + joins: + - name: users + sql: "{CUBE}.user_id = {users}.id" + relationship: many_to_one + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: user_id + sql: user_id + type: number + - name: status + sql: status + type: string + title: Order Status + description: Current order status + meta: + ai_context: Values are pending, shipped, and completed. + - name: created_at + sql: created_at + type: time + - name: is_large + sql: "{CUBE}.amount > 500" + type: boolean + measures: + - name: count + type: count + - name: total_amount + sql: "{CUBE}.amount" + type: sum + description: Total order amount + format: currency + meta: + ai_context: Use this for revenue questions. + - name: completed_amount + sql: "{CUBE}.amount" + type: sum + filters: + - sql: "{CUBE}.status = 'completed'" + - name: avg_order_value + sql: "{total_amount} / {count}" + type: number diff --git a/converters/cube/tests/fixtures/fixtureA_cube/model/cubes/users.yml b/converters/cube/tests/fixtures/fixtureA_cube/model/cubes/users.yml new file mode 100644 index 00000000..d7b59532 --- /dev/null +++ b/converters/cube/tests/fixtures/fixtureA_cube/model/cubes/users.yml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The dimension cube: sits on the one side of the join, so it is exposed to row +# multiplication -- both of its measures are deliberately fan-out-safe (a bare +# `count`, which maps to COUNT(DISTINCT ), and a count_distinct). Also +# exercises a `sql`-defined cube, a geo dimension (which splits into two Ossie +# fields), and a segment (which has no Ossie form and rides in the stash). +cubes: + - name: users + sql: SELECT * FROM public.users WHERE deleted_at IS NULL + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: city + sql: city + type: string + - name: location + type: geo + latitude: + sql: "{CUBE}.lat" + longitude: + sql: "{CUBE}.lon" + measures: + - name: count + type: count + - name: cities + sql: "{CUBE}.city" + type: count_distinct + segments: + - name: active + sql: "{CUBE}.status = 'active'" diff --git a/converters/cube/tests/fixtures/fixtureA_cube/model/views/sales.yml b/converters/cube/tests/fixtures/fixtureA_cube/model/views/sales.yml new file mode 100644 index 00000000..b6f0d584 --- /dev/null +++ b/converters/cube/tests/fixtures/fixtureA_cube/model/views/sales.yml @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The sole view, so it is the mapped one: its name, description, and +# meta.ai_context become the Ossie model's. The `cubes:` curation has no Ossie +# form and round-trips through the model-level stash. +views: + - name: sales + description: Sales overview + meta: + ai_context: > + Primary view for revenue analysis. Use it for any question about + sales, orders, or customer spend. + cubes: + - join_path: orders + includes: "*" + - join_path: orders.users + includes: + - city diff --git a/converters/cube/tests/test_cube_to_osi.py b/converters/cube/tests/test_cube_to_osi.py new file mode 100644 index 00000000..15e07a95 --- /dev/null +++ b/converters/cube/tests/test_cube_to_osi.py @@ -0,0 +1,518 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Cube data model -> Apache Ossie semantic model.""" + +import pytest +from _util import by_name, expr_of, load_fixture_dir, model_of, stash_of + +from ossie_cube import ConversionError, IssueType, convert_cube_to_ossie +from ossie_cube._common import OSSIE_VERSION, cube_sql_to_ossie + + +@pytest.fixture +def fixture_a(): + return load_fixture_dir("fixtureA_cube") + + +@pytest.fixture +def model_a(fixture_a): + out, issues = convert_cube_to_ossie(fixture_a) + return model_of(out), issues + + +# --- model identity ------------------------------------------------------------- + +def test_version_and_single_model(fixture_a): + out, _ = convert_cube_to_ossie(fixture_a) + from _util import parse + + doc = parse(out) + assert doc["version"] == OSSIE_VERSION + assert len(doc["semantic_model"]) == 1 + + +def test_mapped_view_supplies_model_identity(model_a): + model, _ = model_a + assert model["name"] == "sales" + assert model["description"] == "Sales overview" + assert "revenue analysis" in model["ai_context"]["instructions"] + + +def test_model_name_override(fixture_a): + out, _ = convert_cube_to_ossie(fixture_a, model_name="custom") + assert model_of(out)["name"] == "custom" + + +def test_unknown_view_is_rejected(fixture_a): + with pytest.raises(ConversionError, match="not found"): + convert_cube_to_ossie(fixture_a, view="nope") + + +def test_view_curation_rides_in_the_stash(model_a): + model, _ = model_a + stash = stash_of(model) + assert stash["mapped_view"] == "sales" + # The natively mapped description/ai_context are stripped; the curation stays. + view = stash["views"]["sales"] + assert "description" not in view + assert "meta" not in view + assert view["cubes"][0]["join_path"] == "orders" + + +# --- datasets ------------------------------------------------------------------- + +def test_cubes_become_datasets(model_a): + model, _ = model_a + datasets = by_name(model["datasets"]) + assert set(datasets) == {"orders", "users"} + assert datasets["orders"]["source"] == "public.orders" + assert datasets["orders"]["description"] == "Customer orders" + # A `sql`-defined cube keeps its query as the source. + assert datasets["users"]["source"].startswith("SELECT * FROM public.users") + + +def test_primary_key_from_dimension_flag(model_a): + model, _ = model_a + datasets = by_name(model["datasets"]) + assert datasets["orders"]["primary_key"] == ["id"] + assert datasets["users"]["primary_key"] == ["id"] + + +def test_segments_have_no_ossie_form_and_are_stashed(model_a): + model, _ = model_a + users = by_name(model["datasets"])["users"] + segments = stash_of(users)["cube_extras"]["segments"] + assert segments[0]["name"] == "active" + + +# --- fields --------------------------------------------------------------------- + +def test_dimension_types_map_to_datatypes(model_a): + model, _ = model_a + fields = by_name(by_name(model["datasets"])["orders"]["fields"]) + assert fields["status"]["datatype"] == "String" + assert fields["is_large"]["datatype"] == "Boolean" + assert fields["created_at"]["datatype"] == "DateTime" + assert fields["created_at"]["dimension"]["is_time"] is True + + +def test_number_dimension_asserts_no_datatype(model_a): + """Cube collapses Integer/Decimal/Float into `number`, so the converter omits + `datatype` rather than assert a precision the model does not carry.""" + model, _ = model_a + fields = by_name(by_name(model["datasets"])["orders"]["fields"]) + assert "datatype" not in fields["id"] + assert stash_of(fields["id"])["type"] == "number" + + +def test_dimension_title_becomes_label_and_ai_context_maps(model_a): + model, _ = model_a + status = by_name(by_name(model["datasets"])["orders"]["fields"])["status"] + assert status["label"] == "Order Status" + assert status["description"] == "Current order status" + assert status["ai_context"]["instructions"].startswith("Values are pending") + + +def test_cube_reference_is_stripped_in_a_field_expression(model_a): + """Field expressions are dataset-scoped, so `{CUBE}.amount` reads as `amount`.""" + model, _ = model_a + is_large = by_name(by_name(model["datasets"])["orders"]["fields"])["is_large"] + assert expr_of(is_large) == "amount > 500" + + +def test_geo_dimension_splits_into_two_fields(model_a): + model, issues = model_a + fields = by_name(by_name(model["datasets"])["users"]["fields"]) + assert "location" not in fields + assert expr_of(fields["location_latitude"]) == "lat" + assert expr_of(fields["location_longitude"]) == "lon" + assert fields["location_latitude"]["datatype"] == "Float" + assert stash_of(fields["location_latitude"])["geo"]["of"] == "location" + assert issues.of_type(IssueType.GEO_DIMENSION_SPLIT) + + +# --- relationships -------------------------------------------------------------- + +def test_many_to_one_join_becomes_a_relationship(model_a): + model, _ = model_a + rel = by_name(model["relationships"])["orders_to_users"] + assert rel["from"] == "orders" + assert rel["to"] == "users" + assert rel["from_columns"] == ["user_id"] + assert rel["to_columns"] == ["id"] + # The declaring side and the exact Cube spelling round-trip via the stash. + assert stash_of(rel)["declared_on"] == "orders" + assert stash_of(rel)["relationship"] == "many_to_one" + + +def test_one_to_many_join_is_flipped_to_many_side_first(): + """Ossie's `from` is always the many side, so a join declared as one_to_many on + the one side is flipped -- and the declared orientation stashed.""" + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: users\n" + " sql_table: public.users\n" + " joins:\n" + " - name: orders\n" + " sql: \"{CUBE}.id = {orders}.user_id\"\n" + " relationship: one_to_many\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: user_id\n" + " sql: user_id\n" + " type: number\n" + ) + } + out, _ = convert_cube_to_ossie(files) + rel = model_of(out)["relationships"][0] + assert rel["from"] == "orders" + assert rel["to"] == "users" + assert rel["from_columns"] == ["user_id"] + assert rel["to_columns"] == ["id"] + assert stash_of(rel)["relationship"] == "one_to_many" + assert stash_of(rel)["declared_on"] == "users" + + +def test_non_equi_join_is_preserved_not_guessed_at(): + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: rates\n" + " sql: \"{CUBE}.day >= {rates}.valid_from\"\n" + " relationship: many_to_one\n" + " dimensions:\n" + " - name: day\n" + " sql: day\n" + " type: time\n" + " - name: rates\n" + " sql_table: public.rates\n" + " dimensions:\n" + " - name: valid_from\n" + " sql: valid_from\n" + " type: time\n" + ) + } + out, issues = convert_cube_to_ossie(files) + model = model_of(out) + assert "relationships" not in model + orders = by_name(model["datasets"])["orders"] + assert stash_of(orders)["extra_joins"][0]["join"]["name"] == "rates" + assert issues.of_type(IssueType.PARKED_IN_META) + + +def test_join_to_unknown_cube_is_rejected(): + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: ghosts\n" + " sql: \"{CUBE}.id = {ghosts}.id\"\n" + " relationship: many_to_one\n" + ) + } + with pytest.raises(ConversionError, match="not a cube in this model"): + convert_cube_to_ossie(files) + + +# --- metrics -------------------------------------------------------------------- + +def test_measures_are_hoisted_and_disambiguated(model_a): + """`count` exists on both cubes, so both are qualified and the original names + stashed; a globally unique measure keeps its own name.""" + model, _ = model_a + metrics = by_name(model["metrics"]) + assert "orders__count" in metrics + assert "users__count" in metrics + assert stash_of(metrics["orders__count"])["name"] == "count" + assert stash_of(metrics["orders__count"])["cube"] == "orders" + assert "total_amount" in metrics + + +def test_bare_count_maps_through_the_primary_key(model_a): + """Cube renders a bare `count` as count(pk), and count(distinct pk) when the + cube is fanned out. COUNT(DISTINCT pk) equals both, so it is the one static + form that stays correct in every join context.""" + model, _ = model_a + metrics = by_name(model["metrics"]) + assert expr_of(metrics["orders__count"]) == "COUNT(DISTINCT orders.id)" + assert expr_of(metrics["users__count"]) == "COUNT(DISTINCT users.id)" + assert metrics["orders__count"]["datatype"] == "Integer" + + +def test_bare_count_without_a_primary_key_is_rejected(): + """The primary key is load-bearing for a correct `count`, so its absence is an + error rather than a silently-different number.""" + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: status\n" + " sql: status\n" + " type: string\n" + " measures:\n" + " - name: count\n" + " type: count\n" + ) + } + with pytest.raises(ConversionError, match="primary key"): + convert_cube_to_ossie(files) + + +def test_aggregate_measures_become_qualified_expressions(model_a): + model, _ = model_a + metrics = by_name(model["metrics"]) + assert expr_of(metrics["total_amount"]) == "SUM(orders.amount)" + assert expr_of(metrics["cities"]) == "COUNT(DISTINCT users.city)" + assert metrics["total_amount"]["description"] == "Total order amount" + assert metrics["total_amount"]["ai_context"]["instructions"].startswith("Use this") + + +def test_measure_filters_fold_into_a_case_expression(model_a): + """Cube's own applyMeasureFilters wraps the operand as + CASE WHEN THEN END inside the aggregate.""" + model, _ = model_a + metric = by_name(model["metrics"])["completed_amount"] + assert expr_of(metric) == ( + "SUM(CASE WHEN (orders.status = 'completed') THEN orders.amount END)") + + +def test_filtered_and_calculated_measures_keep_the_original(model_a): + """Export cannot recover `filters` from the folded CASE, nor un-inline a + calculated measure's references, so both keep the original measure verbatim -- + which is what makes Cube -> Ossie -> Cube lossless.""" + model, _ = model_a + metrics = by_name(model["metrics"]) + assert stash_of(metrics["completed_amount"])["measure"]["filters"] + assert stash_of(metrics["avg_order_value"])["measure"]["sql"] == ( + "{total_amount} / {count}") + # A plain aggregate needs no such copy. + assert "measure" not in stash_of(metrics["orders__count"]) + + +def test_calculated_measure_inlines_its_measure_references(model_a): + """Cube resolves `{total_amount} / {count}` to the referenced measures' own + aggregate SQL; Ossie has no metric-to-metric reference, so it is inlined.""" + model, _ = model_a + metric = by_name(model["metrics"])["avg_order_value"] + assert expr_of(metric) == "(SUM(orders.amount)) / (COUNT(DISTINCT orders.id))" + + +def test_measure_reference_cycle_is_rejected(): + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " measures:\n" + " - name: a\n" + " sql: \"{b} + 1\"\n" + " type: number\n" + " - name: b\n" + " sql: \"{a} + 1\"\n" + " type: number\n" + ) + } + with pytest.raises(ConversionError, match="cycle"): + convert_cube_to_ossie(files) + + +def test_multi_stage_measure_is_dropped_with_an_issue(): + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " measures:\n" + " - name: rolling\n" + " sql: amount\n" + " type: sum\n" + " multi_stage: true\n" + ) + } + out, issues = convert_cube_to_ossie(files) + assert "metrics" not in model_of(out) + assert issues.of_type(IssueType.MULTI_STAGE_MEASURE_DROPPED) + + +# --- fan-out -------------------------------------------------------------------- + +_FANOUT_MODEL = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {users}.id\"\n" + " relationship: many_to_one\n" + " dimensions:\n" + " - name: user_id\n" + " sql: user_id\n" + " type: number\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: lifetime_value\n" + " sql: \"{CUBE}.ltv\"\n" + " type: sum\n" + ) +} + + +def test_fanout_unsafe_metric_is_refused_by_default(): + """`users` is the one side of a many-to-one join, so summing over it after the + join over-counts. Cube deduplicates on the primary key at query time; a static + Ossie expression cannot, so the default is to refuse rather than emit a number + that silently disagrees with Cube.""" + with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): + convert_cube_to_ossie(_FANOUT_MODEL) + + +def test_fanout_unsafe_metric_is_recorded_when_not_strict(): + out, issues = convert_cube_to_ossie(_FANOUT_MODEL, strict_fanout=False) + metric = by_name(model_of(out)["metrics"])["lifetime_value"] + assert expr_of(metric) == "SUM(users.ltv)" + recorded = issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) + assert len(recorded) == 1 + assert recorded[0].element_name == "users.lifetime_value" + assert "over-count" in recorded[0].detail + + +def test_idempotent_aggregates_are_never_flagged(fixture_a): + """count / count_distinct / min / max are unaffected by duplicate rows, so a + fanned-out dataset carrying only those converts cleanly under strict mode.""" + _, issues = convert_cube_to_ossie(fixture_a) + assert not issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) + + +# --- rejections and preservation ------------------------------------------------ + +def test_jinja_templated_file_is_preserved_not_parsed(): + files = { + "model/cubes/dyn.yml": "cubes:\n - name: o{{ suffix }}\n sql_table: t\n", + "model/cubes/ok.yml": ( + "cubes:\n - name: orders\n sql_table: public.orders\n"), + } + out, issues = convert_cube_to_ossie(files) + model = model_of(out) + assert by_name(model["datasets"]).keys() == {"orders"} + assert "model/cubes/dyn.yml" in stash_of(model)["extra_files"] + assert issues.of_type(IssueType.TEMPLATED_MEMBER_DROPPED) + + +def test_javascript_model_is_preserved_not_parsed(): + files = { + "model/cubes/orders.js": "cube(`orders`, { sql_table: `public.orders` });", + "model/cubes/ok.yml": ( + "cubes:\n - name: orders_yaml\n sql_table: public.orders\n"), + } + out, issues = convert_cube_to_ossie(files) + assert "model/cubes/orders.js" in stash_of(model_of(out))["extra_files"] + assert issues.of_type(IssueType.TEMPLATED_MEMBER_DROPPED) + + +def test_extends_is_refused_rather_than_half_resolved(): + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: base\n" + " sql_table: public.orders\n" + " - name: derived\n" + " extends: base\n" + ) + } + with pytest.raises(ConversionError, match="extends"): + convert_cube_to_ossie(files) + + +def test_cube_without_a_source_is_rejected(): + files = {"model/cubes/m.yml": "cubes:\n - name: orders\n description: x\n"} + with pytest.raises(ConversionError, match="neither 'sql' nor 'sql_table'"): + convert_cube_to_ossie(files) + + +def test_cube_with_both_sources_is_rejected(): + files = { + "model/cubes/m.yml": ( + "cubes:\n - name: orders\n sql: SELECT 1\n sql_table: t\n") + } + with pytest.raises(ConversionError, match="exactly one"): + convert_cube_to_ossie(files) + + +def test_duplicate_cube_name_is_rejected(): + files = { + "model/cubes/a.yml": "cubes:\n - name: orders\n sql_table: a\n", + "model/cubes/b.yml": "cubes:\n - name: orders\n sql_table: b\n", + } + with pytest.raises(ConversionError, match="defined twice"): + convert_cube_to_ossie(files) + + +def test_model_with_no_cubes_is_rejected(): + with pytest.raises(ConversionError, match="no convertible cubes"): + convert_cube_to_ossie({"README.md": "not a model"}) + + +# --- reference translation ------------------------------------------------------ + +@pytest.mark.parametrize("sql,expected", [ + ("{CUBE}.status", "status"), + ("{TABLE}.status", "status"), + ("{CUBE.status}", "status"), + ("{status}", "status"), + ("{orders.status}", "status"), + ("{users.city}", "users.city"), + ("${CUBE}.status", "status"), + ("LOWER({CUBE}.email)", "LOWER(email)"), + (r"'\{literal\}'", "'{literal}'"), +]) +def test_reference_translation_in_a_field_context(sql, expected): + """A field expression is dataset-scoped, so own-cube references reduce to a + bare name. `\\{` stays a literal brace.""" + assert cube_sql_to_ossie(sql, "orders")[0] == expected + + +@pytest.mark.parametrize("sql,expected", [ + ("{CUBE}.amount", "orders.amount"), + ("{CUBE.amount}", "orders.amount"), + ("{amount}", "orders.amount"), + ("{users.city}", "users.city"), +]) +def test_reference_translation_in_a_metric_context(sql, expected): + """A metric expression is model-level, so own-cube references are qualified.""" + assert cube_sql_to_ossie(sql, "orders", self_prefix="orders")[0] == expected diff --git a/converters/cube/uv.lock b/converters/cube/uv.lock new file mode 100644 index 00000000..c213eb6b --- /dev/null +++ b/converters/cube/uv.lock @@ -0,0 +1,216 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "apache-ossie-cube" +version = "0.2.0.dev0" +source = { editable = "." } +dependencies = [ + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "hypothesis" }, + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [{ name = "pyyaml", specifier = ">=6.0" }] + +[package.metadata.requires-dev] +dev = [ + { name = "hypothesis", specifier = ">=6.0" }, + { name = "pytest", specifier = ">=8.0" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "hypothesis" +version = "6.163.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/08/4cbfa0327e9df00f57fc67f91847add2f0dd6c23408935273b095ee9a9f1/hypothesis-6.163.0.tar.gz", hash = "sha256:520480d4bd3a17557616c25923640953e360332c89d012fffcebd69857e674a9", size = 490145, upload-time = "2026-07-28T07:16:46.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/b1/3eea7a422de342bd0095a9672c3e03fe0466e4561a55499a1e01b4a9f098/hypothesis-6.163.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:331906cb029b6b360b8ebac3ec00c3cfa720037fe2efb294a503a1979c9a9a8f", size = 769898, upload-time = "2026-07-28T07:15:19.323Z" }, + { url = "https://files.pythonhosted.org/packages/bc/24/45b5c948c76c16ecf1e4ad1ca4a4a3fce55b3317ee172bc8230ff2956ae1/hypothesis-6.163.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:b4ad2134405d5345434c22dea96bbc12c85abcfc3c253a8063dbc9ff01164555", size = 765438, upload-time = "2026-07-28T07:16:04.69Z" }, + { url = "https://files.pythonhosted.org/packages/0f/72/7725039a75b3679dc445a169026b11860a0e67a600c4ffed49a42040a000/hypothesis-6.163.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50073f8e63c1e7d3403899755657a990d8bba7b5b5bff66b1c56796d4969bb28", size = 1094699, upload-time = "2026-07-28T07:15:56.668Z" }, + { url = "https://files.pythonhosted.org/packages/dd/fa/bcfa3879f303a302ec6e5f4d35b583924867a0821b42fa275f440f3b8dab/hypothesis-6.163.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8c5d1e6bad47edf6fb1d7406cf6d67314ac08325c63a49550d782a4596ea302b", size = 1123315, upload-time = "2026-07-28T07:16:40.381Z" }, + { url = "https://files.pythonhosted.org/packages/5f/7f/e7fe2f0658db5182bb1d4b266d17f8f81cf3db82eeba27677ddfea13ac09/hypothesis-6.163.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ec3b709508ccd835d8ded1db025b7800618f2289a22a6bfd4927da5f4eb33c", size = 1144220, upload-time = "2026-07-28T07:15:20.617Z" }, + { url = "https://files.pythonhosted.org/packages/90/14/c26f93a4693bfd83d7ba14b7043d986458026f6635d1b157bc2f1c56a59e/hypothesis-6.163.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:7cb3d927360fe73f9a06d646e6082237142ee39c24679c7133d22bf06dd03b45", size = 1099527, upload-time = "2026-07-28T07:16:17.955Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f5/4c1dde28d2e18e169dd6c471ef4bcf12abf3c6a3f4a1744bdabdbdf411ba/hypothesis-6.163.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3f3cceb4720a39127622fbf3bcebe1775b894372c53b5edddfdef10bbdeef9ec", size = 1136272, upload-time = "2026-07-28T07:16:38.182Z" }, + { url = "https://files.pythonhosted.org/packages/3a/5b/54153509ed42cc17e1b65efe34a0c781f42fb04c902dc5f25486c537ec88/hypothesis-6.163.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:59f5fdb8addb44c17520a60d50542d9db6ceba577bbf54efefa9c10ee20be140", size = 1268518, upload-time = "2026-07-28T07:15:42.991Z" }, + { url = "https://files.pythonhosted.org/packages/81/86/f86f0d15d91b9cbff3546c1b8891950b2b08c0527e7494133fb2180e1b8b/hypothesis-6.163.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:f1fe222f50a1898e87a1e7323ab35f9e956278efabe4dd55a1342808206d05ad", size = 1396357, upload-time = "2026-07-28T07:15:26.696Z" }, + { url = "https://files.pythonhosted.org/packages/46/3a/c096b6b272f15e17e8e65c6ccfb2e1d167456ff5bbd9db33dca3185199f6/hypothesis-6.163.0-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:56ed585baab75cb98462c57ca88bbdc6a9d935a14118dd572fb476c3ecec2a06", size = 1269128, upload-time = "2026-07-28T07:16:11.902Z" }, + { url = "https://files.pythonhosted.org/packages/bb/9f/0807445874b3083a22c9a14a0ffc31bd0060ef3b408ce4cb31c20779cdf8/hypothesis-6.163.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2849c23b2e0fe2eef4c1ec336b01eac7ad7397c49fca43c264f59ec1e6046eac", size = 1311189, upload-time = "2026-07-28T07:16:08.545Z" }, + { url = "https://files.pythonhosted.org/packages/d6/de/3319aa8fcffd1641defc1c5eea1ad646a33d1b62528ef487a89752bc8079/hypothesis-6.163.0-cp310-abi3-win32.whl", hash = "sha256:b2ddcdaf6691101e06dc4a5add7b8c8fdf1e68daba599255a281f3f3550d3331", size = 655743, upload-time = "2026-07-28T07:16:30.966Z" }, + { url = "https://files.pythonhosted.org/packages/8e/48/36bc72910451e6e88b75e59a6ddbf0db34ff61a3b11c9439801dfd5fec20/hypothesis-6.163.0-cp310-abi3-win_amd64.whl", hash = "sha256:4ab0dadc09c537d4ac57e564039dfe7daf09c98375306d54bfc0fd6c218efcca", size = 661902, upload-time = "2026-07-28T07:16:34.407Z" }, + { url = "https://files.pythonhosted.org/packages/63/80/796ac61dddb3ede550ea127e28e027a57a4ea481581c0bb27701fefd655a/hypothesis-6.163.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:213527755f0fc2b1f3721e73fd60023e2752a48f914e3e2df8d35111956ae5c8", size = 770366, upload-time = "2026-07-28T07:15:22.003Z" }, + { url = "https://files.pythonhosted.org/packages/eb/87/53924f322922bcfc05e40c79d432978333827b335fa9efa5fd1e8cbf3c90/hypothesis-6.163.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ca1b48bde68c528a79dec2a2859e05035802e5b1c9c3579f388c9de6ed6d0148", size = 766150, upload-time = "2026-07-28T07:15:16.151Z" }, + { url = "https://files.pythonhosted.org/packages/f3/fb/b62480e6510052139d7b2a0219d4ae8bd52ccad0ed74db15dd61330a6962/hypothesis-6.163.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a3db868a943c814cc557104712d43bf609adfe5ea9f708f38377d366b4855f8", size = 1095046, upload-time = "2026-07-28T07:15:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/4d/d7/fa8aac7b47f41d0fe30fa270c5026f37a0c0c60d7b9a1a93dcc8fcfc50ce/hypothesis-6.163.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4159a1c2560e10de51b1c14956e277eb1b37526c9abef9e87c1e531760486448", size = 1144516, upload-time = "2026-07-28T07:16:32.662Z" }, + { url = "https://files.pythonhosted.org/packages/4f/26/c543c76d8a8b8f58f2d7adf0cb42e4928be3464e95f4fa9d7221b42ea9ce/hypothesis-6.163.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ffdda3006a383a48f71a23b4f2b3fae3fe1b09af67925d885985f7ec34d66bcb", size = 1268886, upload-time = "2026-07-28T07:16:36.102Z" }, + { url = "https://files.pythonhosted.org/packages/d9/54/3613ef980cfa60f5c6bfbc533989d6c67b6b5f4e9e638fc6d010a5dd852f/hypothesis-6.163.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3b6cee2afe6c67b31a4a64b63a876e0b020befdc61daabea80f7a0e14f19203a", size = 1311472, upload-time = "2026-07-28T07:16:13.707Z" }, + { url = "https://files.pythonhosted.org/packages/af/0c/cbaebc49fd5807a4b4286dea6e60d5330951e43115d002371bdfc99e70f8/hypothesis-6.163.0-cp311-cp311-win_amd64.whl", hash = "sha256:0a933aca9ebf9daf951d07cf01200c94c321b6ee0b42cc7b67675c9686d914c2", size = 661580, upload-time = "2026-07-28T07:15:32.939Z" }, + { url = "https://files.pythonhosted.org/packages/8f/2c/74e989557efc429b28282cbe754c17fc74495467a72a1c94b5eb734fe374/hypothesis-6.163.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a57352efa938889ea9992667a5014c0fc870d03945de71918574d1cf28276378", size = 771445, upload-time = "2026-07-28T07:16:44.2Z" }, + { url = "https://files.pythonhosted.org/packages/fd/08/3ed2089d8cbeae125ea92879e82b99ea3a8b676c837710019249ccab379f/hypothesis-6.163.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0a0c396244c13805edcb73ff467c4c8178ccefc41c4ef5ed00a68e612fd773e9", size = 763070, upload-time = "2026-07-28T07:15:34.318Z" }, + { url = "https://files.pythonhosted.org/packages/e5/1e/4e85a11f15e8ebd730f3b3e4a5d83653f7da482a4e81fa36958951d89b4a/hypothesis-6.163.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28a6cc1c25a6cc9b6ec079eaabd32ac769994831ecddd57123ce43c9056dcf34", size = 1093500, upload-time = "2026-07-28T07:15:49.539Z" }, + { url = "https://files.pythonhosted.org/packages/1f/29/c4790d2a5e6f48e6be5f868124a0d3c7e1d0102e2ca50503a0718e51289c/hypothesis-6.163.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd312b15044b1c1a0920a5827a830559b2d1fa380851cedf509f8b835309c5b9", size = 1143541, upload-time = "2026-07-28T07:15:44.393Z" }, + { url = "https://files.pythonhosted.org/packages/da/a8/01b72694e758449e9b530b72ecaf5f3625c80a3968de0d12ee6e784de22e/hypothesis-6.163.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b839dfd1342bb50570cb0c66b80322307cdb468abf14faf5df4dab022bc1b9ce", size = 1266326, upload-time = "2026-07-28T07:15:12.604Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a3/b3997add991e2fc6123b3c8dde3631f9f633db67ba23f5b2567736b50b9e/hypothesis-6.163.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c4f5be1482189c7b0a1dcac269fffe97a7d18cc04ac9a9a4d6613212dd87f38b", size = 1310536, upload-time = "2026-07-28T07:15:17.798Z" }, + { url = "https://files.pythonhosted.org/packages/5d/fe/36f576185d63ee0b4d94ed9564415de08baed4937e785b3695c8dd665c9c/hypothesis-6.163.0-cp312-cp312-win_amd64.whl", hash = "sha256:7ca7b20bf38d51e15f7808b0239791c4792b1709ce0c63093acaff56a09c31e6", size = 659021, upload-time = "2026-07-28T07:15:53.143Z" }, + { url = "https://files.pythonhosted.org/packages/58/69/c474a3fa1c33d9a6e059e820221275d9c60eb10085f6164212c291856157/hypothesis-6.163.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:a16ebce774755a7a652bd44c62101dc914372ed1a98935969624848c9627b4a4", size = 771335, upload-time = "2026-07-28T07:15:14.988Z" }, + { url = "https://files.pythonhosted.org/packages/54/27/8951688de58314780ba0af5bf2675709009dcc1fb568d244f2a03c4de8e9/hypothesis-6.163.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40dfab6fe6a02a80abef81aebf88e53cd529e3f2f6ba3486b674a67b1f4a3512", size = 763020, upload-time = "2026-07-28T07:15:25.218Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d7/9eb7451400507f4b5c972c81c2bc57d0009597ea37295519a9645963a50e/hypothesis-6.163.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f28ad27193c1fbcfb52ef2ee63d2b721563525089e80962b4268b306dac45507", size = 1093415, upload-time = "2026-07-28T07:16:27.408Z" }, + { url = "https://files.pythonhosted.org/packages/24/cc/c12c780676c7a4a4051d05e7b278a94bf1bb496ef31882881160f16866c9/hypothesis-6.163.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8aac96db8a6c7ee43aba2ee0d3c43893da1fb7c38ed54790c1be2b6d8fd87b96", size = 1143356, upload-time = "2026-07-28T07:16:01.571Z" }, + { url = "https://files.pythonhosted.org/packages/4b/c1/61c5ebdb77a3f259803e60a12f56de35f8b6d641a6219f798dcfa69dece3/hypothesis-6.163.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9d23f0f3a14bb6e6f99c793d340196dba4af95ba25bfcab624d1794f540f5e27", size = 1266375, upload-time = "2026-07-28T07:15:11.353Z" }, + { url = "https://files.pythonhosted.org/packages/a2/53/f3a89b4d21d89098d1dec749632aa0fece04f5d17a9e7e91c7f10596d55a/hypothesis-6.163.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b123b4995a7612f1130e2b2362c9a5d0568df887bf7e7bdb45c23af8cd5423c9", size = 1310257, upload-time = "2026-07-28T07:16:42.213Z" }, + { url = "https://files.pythonhosted.org/packages/06/e7/88e71c4cec0df68aa7a2fb251083c544f30697bd90fb4b1ed19de493ab81/hypothesis-6.163.0-cp313-cp313-win_amd64.whl", hash = "sha256:b268211e625cd550e361fc387bf1db5deb1e9cae0ce4041116f0a0aafeef7c06", size = 658983, upload-time = "2026-07-28T07:16:10.289Z" }, + { url = "https://files.pythonhosted.org/packages/71/df/e3b2f0419cebcc86bba96a357d7ef37790538ed6b09f3183ae01f1fc3d23/hypothesis-6.163.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:9105c66ea8dbc108adc42058bb7b65bd953f53ee178bf63bf9ebb0cded6c8c96", size = 771564, upload-time = "2026-07-28T07:15:28.017Z" }, + { url = "https://files.pythonhosted.org/packages/f1/f8/a6f75e61ecd983029f8463bf007498155c4ed33114e6931e7aa3fbaf651e/hypothesis-6.163.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e165f6cc2075059b7c95dac1612bfb25494f72d90f56880e84c288b089f8a896", size = 763164, upload-time = "2026-07-28T07:15:29.973Z" }, + { url = "https://files.pythonhosted.org/packages/5e/07/5193812567f6ca46c1f0cd02dabfd47c85d7c9e3287b8a870fc8150ee462/hypothesis-6.163.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cba5202f74e7e4cdb676d86f26e8cc1b4fdc88f7f58ba73c8ac45b6b22f3070", size = 1093916, upload-time = "2026-07-28T07:16:25.626Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0c/9d61f0e306499d734dc63ebe374b01c5f50be7072fd5e76eed25cc0b86ac/hypothesis-6.163.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7f706df6839dcc53f20833f2933cbcd126fd2fdee7c312e053de49df4b64e44", size = 1143547, upload-time = "2026-07-28T07:15:07.311Z" }, + { url = "https://files.pythonhosted.org/packages/60/b4/2a9eb04c9847ddf7e6b30bba8bc27b7efe5511d368335b6a41657b3f02dd/hypothesis-6.163.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:487ab8ec2f01a225d6a1e2ceadc5290cde2c691952bd2e7f76199cf82e06fb25", size = 1266755, upload-time = "2026-07-28T07:15:41.457Z" }, + { url = "https://files.pythonhosted.org/packages/d2/f3/8d1903fbfcbf48b90bb5590826821afc9b4fc494391ab1fdfab9df23e928/hypothesis-6.163.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ae63dec6d1d467b7f4737455f81a7a82f14a41c14510937fcfbc726a085b5f8", size = 1310560, upload-time = "2026-07-28T07:15:58.419Z" }, + { url = "https://files.pythonhosted.org/packages/47/2b/1a8c0457b44775d0aad369f21cbae8026b772a71aa917d1b956b7349b2a4/hypothesis-6.163.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:31dc46c48aa53c3ec92d03120978ca7f19b9cf96d195ed3fc93503f1433c94a6", size = 603067, upload-time = "2026-07-28T07:16:06.697Z" }, + { url = "https://files.pythonhosted.org/packages/10/56/a9cd947064043035457dec0124ac2437ca72acf358b30cf203a229ad831b/hypothesis-6.163.0-cp314-cp314-win_amd64.whl", hash = "sha256:320b076bf6436f971f1c73ee651e60001226d1b4e341f2c4a1ca87248261ca03", size = 658931, upload-time = "2026-07-28T07:15:37.174Z" }, + { url = "https://files.pythonhosted.org/packages/77/37/2d16317fda0ecd915cb094be9a9e8911106e693ed03a4d680de314711854/hypothesis-6.163.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:ab34c61d9249f1a8129cb4276062c04e3e47b5be8de6446e7c7fe11362d6fe43", size = 770142, upload-time = "2026-07-28T07:16:21.827Z" }, + { url = "https://files.pythonhosted.org/packages/3f/65/d80a9bfb7868f2c6c072a684993548391c56e0afa1972f79ee1fe168d91b/hypothesis-6.163.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f2f1b67a48da86d3e41c9445367b49a49f7efdb60fc8b5e3593f05e6afb2efbe", size = 761694, upload-time = "2026-07-28T07:15:31.36Z" }, + { url = "https://files.pythonhosted.org/packages/a7/01/c1f2515c638d2637300bb2fd6af129319eae75cdf2ed7804644535f64fb2/hypothesis-6.163.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c084749c115ea7918cf7efa144682783da17eec70d1276689182b871126e715", size = 1092511, upload-time = "2026-07-28T07:15:51.503Z" }, + { url = "https://files.pythonhosted.org/packages/cd/f8/b3cd946308b5a09e52b075792610da310c0b7972bb1e8aa673400b86540c/hypothesis-6.163.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21e72e8d5818e5ef8cd6a2191c386e3fd1a6d9e3739cf97289b4d9b5dbc8e38d", size = 1142425, upload-time = "2026-07-28T07:16:15.626Z" }, + { url = "https://files.pythonhosted.org/packages/85/96/b122859e6f7335b54aff759f573a813158d2249488450a75e76462ddaf61/hypothesis-6.163.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5a3ac6c62d49f7fe518dfe7fa924fa03aac839993702207802b0e45f9e1b0dab", size = 1264946, upload-time = "2026-07-28T07:15:23.848Z" }, + { url = "https://files.pythonhosted.org/packages/ba/2e/e0226e8c904b8b4788eb52dacd303c91acbd260904abf64e8f9bad03c88a/hypothesis-6.163.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e568a3d766b7ba8df00e0c33efc4c6530cde14fbc72daabe4824eed211ed7596", size = 1309320, upload-time = "2026-07-28T07:16:29.218Z" }, + { url = "https://files.pythonhosted.org/packages/8c/48/e8bd29fed17c9608524b6a39db4a27b6eec7ce85bda78bba3ee0deebd80e/hypothesis-6.163.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b8f22fb8218ba6a452bf9000fc656e1ed57625d17cc8a3871a0fcea3b1b69ebf", size = 659064, upload-time = "2026-07-28T07:16:00.075Z" }, + { url = "https://files.pythonhosted.org/packages/39/db/d4e877b8639bbebedeff4a0511f5fc459c06a31d79a79bf75584eddda8da/hypothesis-6.163.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:002a9709345892279fb0e81b5a05b72d08cfe81f937339827be0d588607ca9b0", size = 771252, upload-time = "2026-07-28T07:15:08.694Z" }, + { url = "https://files.pythonhosted.org/packages/ea/28/3492490e997e5c8c9244a56728a623ca817577af3462fca5d1def060a066/hypothesis-6.163.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:52f16840add2eb02c2416f3b83cec4f527b6c19699f2d31eff4859233c715526", size = 767161, upload-time = "2026-07-28T07:15:54.972Z" }, + { url = "https://files.pythonhosted.org/packages/51/73/37a4d4a6f3f0789fb2fddc851da957075fbe223706d1148ccc4dda825171/hypothesis-6.163.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34fc895691a2420595506eb17f3a104f2fa9039f013c0770a6cc2743ccaf6fed", size = 1096016, upload-time = "2026-07-28T07:15:09.821Z" }, + { url = "https://files.pythonhosted.org/packages/48/46/20bc7801f8b539334dc0439c28753632a078c96d6732eccf1bd4880644d2/hypothesis-6.163.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ef8954e37c80e0c46e6161eef1c72c71059b95250e620a77bd646f6c7a52a2d", size = 1145797, upload-time = "2026-07-28T07:15:39.864Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/4d5c8feb78ed86e698d4e0665d3179eb657ae66d3606ffe096d8055aaa82/hypothesis-6.163.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d0838a28e9943d5b834ebae59b02adda76e2cd1e65caa808104c72102052057d", size = 662696, upload-time = "2026-07-28T07:15:38.519Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] From 2fc45bc167582b4d31e3312f97a4731180cf1dfd Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 00:15:47 +0500 Subject: [PATCH 02/46] Add the Ossie -> Cube export direction, completing the converter Both directions are now implemented and losslessly round-trip. Export emits one `model/cubes/.yml` per dataset plus a `model/views/.yml` for the model itself -- the view is always emitted, not optional, because it is the model boundary for view-first Cube users. A model imported from Cube restores its original file paths, view curation, segments, pre-aggregations, hierarchies, and every other Cube-only construct from the stash; a hand-authored Ossie model gets a view generated at the FK sink with each cube addressed by its join path. Because Cube has a `meta` field at every level, Ossie constructs Cube has no slot for (`unique_keys`, foreign-vendor `custom_extensions`, the structured form of `ai_context`) are parked under `meta.ossie` instead of being dropped. So Ossie -> Cube -> Ossie is lossless too, not just Cube -> Ossie -> Cube. Reference forms follow Cube's semantics rather than being uniform: `{CUBE.member}` when the dataset declares a field of that name (reusing the member's SQL, compile-time checked), `{CUBE}.column` for a raw physical column, and `{other_cube.member}` across cubes -- which is also what gives a cross-dataset metric its implicit join. The cube's own name is never spelled out, so the model survives `extends`. COUNT(DISTINCT ) converts back to Cube's bare `type: count`, closing the loop on the fan-out mapping in both directions. Tests (160): per-direction unit tests, fixture round-trips including a TPC-DS model generated from examples/tpcds_semantic_model.yaml as the guide asks, core-spec JSON Schema validation of every emitted Ossie document, and Hypothesis property-based round-trips over generated Cube models with a seeded fallback when hypothesis is unavailable. The property tests found two real defects: a `{{` anywhere in a file disqualifies it as Jinja (matching Cube's own file-level check), which could leave a join pointing at a cube that was never converted -- the error now names the skipped file instead of just reporting a missing cube. Co-Authored-By: Claude Opus 5 --- converters/cube/README.md | 33 +- converters/cube/pyproject.toml | 3 + converters/cube/src/ossie_cube/__init__.py | 5 +- converters/cube/src/ossie_cube/_common.py | 4 - converters/cube/src/ossie_cube/cli.py | 37 +- converters/cube/src/ossie_cube/cube_to_osi.py | 58 +- converters/cube/src/ossie_cube/osi_to_cube.py | 711 ++++++++++++++++++ converters/cube/tests/_roundtrip_helpers.py | 210 ++++++ converters/cube/tests/_util.py | 14 + .../tpcds_cube/model/cubes/customer.yml | 83 ++ .../tpcds_cube/model/cubes/date_dim.yml | 84 +++ .../fixtures/tpcds_cube/model/cubes/item.yml | 98 +++ .../fixtures/tpcds_cube/model/cubes/store.yml | 97 +++ .../tpcds_cube/model/cubes/store_sales.yml | 211 ++++++ .../model/views/tpcds_retail_model.yml | 60 ++ converters/cube/tests/test_cube_to_osi.py | 21 + converters/cube/tests/test_osi_to_cube.py | 429 +++++++++++ converters/cube/tests/test_roundtrip.py | 194 +++++ .../cube/tests/test_roundtrip_properties.py | 84 +++ converters/cube/uv.lock | 184 +++++ 20 files changed, 2594 insertions(+), 26 deletions(-) create mode 100644 converters/cube/src/ossie_cube/osi_to_cube.py create mode 100644 converters/cube/tests/_roundtrip_helpers.py create mode 100644 converters/cube/tests/fixtures/tpcds_cube/model/cubes/customer.yml create mode 100644 converters/cube/tests/fixtures/tpcds_cube/model/cubes/date_dim.yml create mode 100644 converters/cube/tests/fixtures/tpcds_cube/model/cubes/item.yml create mode 100644 converters/cube/tests/fixtures/tpcds_cube/model/cubes/store.yml create mode 100644 converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml create mode 100644 converters/cube/tests/fixtures/tpcds_cube/model/views/tpcds_retail_model.yml create mode 100644 converters/cube/tests/test_osi_to_cube.py create mode 100644 converters/cube/tests/test_roundtrip.py create mode 100644 converters/cube/tests/test_roundtrip_properties.py diff --git a/converters/cube/README.md b/converters/cube/README.md index 4fe445fe..7a5c4a4a 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -23,10 +23,6 @@ Bidirectional, offline conversion between an [Apache Ossie](https://github.com/a semantic model and a [Cube](https://cube.dev/docs/product/data-modeling/overview) data model. No Cube deployment, API token, or network access required. -> **Status:** the **import** direction (Cube -> Ossie) is implemented. The -> **export** direction (Ossie -> Cube) is in progress; the mapping table below -> describes the agreed behavior for both. - A Cube data model is a *directory* of YAML files rather than a single document, so this converter maps one Ossie YAML document to/from the Cube model layout: @@ -68,18 +64,23 @@ The only runtime dependency is `PyYAML`. Python 3.11+. ```bash ossie-cube import -i model/ [-o model.yaml] [--name my_model] [--view sales] [--no-strict-fanout] +ossie-cube export -i model.yaml -o model/ [--dialect SNOWFLAKE] [--base-cube orders] ``` -With no `-o` the Ossie YAML goes to stdout; issues always go to stderr. `--view` -picks which view's name/description/AI context map onto the Ossie model when the -directory holds several. `--name` overrides the model name. +`import` with no `-o` writes the Ossie YAML to stdout; `export` always needs `-o` +(a directory). Issues always go to stderr. `--view` picks which view's +name/description/AI context map onto the Ossie model when the directory holds +several; `--name` overrides the model name. `--base-cube` picks the cube a +*generated* view is rooted at, and is only consulted for a hand-authored Ossie +model with no stashed views. ### Python API ```python -from ossie_cube import convert_cube_to_ossie +from ossie_cube import convert_cube_to_ossie, convert_ossie_to_cube -ossie_yaml, issues = convert_cube_to_ossie(files) # {relative filename: YAML str} +ossie_yaml, issues = convert_cube_to_ossie(files) # {relative filename: YAML str} +files, issues = convert_ossie_to_cube(ossie_yaml) # -> {relative filename: YAML str} for issue in issues: print(issue) ``` @@ -228,10 +229,18 @@ uv sync uv run pytest ``` +Example-based unit tests per direction, fixture round-trip tests (including the +[TPC-DS model](../../examples/tpcds_semantic_model.yaml) the converter guide asks +for as a baseline), core-spec JSON Schema validation of every emitted Ossie +document, and Hypothesis property-based round-trip tests over generated Cube +models -- which fall back to a seeded sweep when `hypothesis` is unavailable, so +the properties still run. + ## Future effort Both the Apache Ossie specification and Cube's data model are still evolving. As either side adds or changes fields, this converter will be updated to track them. -Known next steps: the export direction, offline `extends` resolution, and a -first-class Ossie representation for measure additivity so the fan-out caveat can -be recorded in the model instead of an issue log. +Known next steps: offline `extends` resolution, `.js`/`.ts` model support (which +needs Cube's own transpiler, so most likely a Cube-side exporter feeding this +converter), and a first-class Ossie representation for measure additivity so the +fan-out caveat can be recorded in the model instead of an issue log. diff --git a/converters/cube/pyproject.toml b/converters/cube/pyproject.toml index 4b60853e..a78e12b8 100644 --- a/converters/cube/pyproject.toml +++ b/converters/cube/pyproject.toml @@ -44,6 +44,9 @@ dependencies = [ dev = [ "pytest>=8.0", "hypothesis>=6.0", + # So the core-spec schema validation in test_roundtrip.py runs rather than + # skipping; the converter itself needs neither. + "jsonschema>=4.0", ] [project.scripts] diff --git a/converters/cube/src/ossie_cube/__init__.py b/converters/cube/src/ossie_cube/__init__.py index fcf01fd5..037f4162 100644 --- a/converters/cube/src/ossie_cube/__init__.py +++ b/converters/cube/src/ossie_cube/__init__.py @@ -19,14 +19,16 @@ models. Pure offline transforms: Ossie YAML string <-> {relative filename: YAML string}. - from ossie_cube import convert_cube_to_ossie + from ossie_cube import convert_cube_to_ossie, convert_ossie_to_cube ossie_yaml, issues = convert_cube_to_ossie(files) + files, issues = convert_ossie_to_cube(ossie_yaml) """ from ._common import ConversionError from .converter_issues import ConverterIssue, IssueLog, IssueType from .cube_to_osi import convert_cube_to_ossie +from .osi_to_cube import convert_ossie_to_cube __all__ = [ "ConversionError", @@ -34,4 +36,5 @@ "IssueLog", "IssueType", "convert_cube_to_ossie", + "convert_ossie_to_cube", ] diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index 9bfaee02..1468ab4c 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -496,10 +496,6 @@ def join_source(cube, cube_name): "Opaque": "string", } -# Ossie datatypes whose temporal role makes `is_time` default to true (spec.md, -# "DataType and is_time"). -TEMPORAL_DATATYPES = frozenset({"Date", "Time", "DateTime", "DateTimeTz"}) - # Cube measure `type` -> the Ossie aggregate function that reproduces it. # `count` is absent: it maps through the cube's primary key, see # primary_key_count_expression(). diff --git a/converters/cube/src/ossie_cube/cli.py b/converters/cube/src/ossie_cube/cli.py index 51bc336c..8616a2a7 100644 --- a/converters/cube/src/ossie_cube/cli.py +++ b/converters/cube/src/ossie_cube/cli.py @@ -18,15 +18,17 @@ """Command-line interface for the Apache Ossie <-> Cube converter. ossie-cube import -i model/ [-o model.yaml] [--name my_model] [--view sales] + ossie-cube export -i model.yaml -o model/ [--dialect SNOWFLAKE] [--base-cube orders] `import` converts a Cube data model directory (any `.yml` holding `cubes:` / `views:`) into an Apache Ossie semantic model; with no `-o` the Ossie YAML goes to -stdout. Conversions that could not carry something across print an issue list to -stderr. +stdout. `export` does the reverse and always needs `-o` (a directory). +Conversions that could not carry something across print an issue list to stderr. By default a metric whose value a static Ossie expression cannot keep correct -under row multiplication is refused, mirroring Cube's own refusal to answer such -a query; pass `--no-strict-fanout` to emit it with a recorded issue instead. +under row multiplication is refused on import, mirroring Cube's own refusal to +answer such a query; pass `--no-strict-fanout` to emit it with a recorded issue +instead. """ import argparse @@ -35,6 +37,7 @@ from ._common import ConversionError from .cube_to_osi import convert_cube_to_ossie +from .osi_to_cube import convert_ossie_to_cube def _build_parser(): @@ -58,6 +61,18 @@ def _build_parser(): action="store_false", default=True, help="record fan-out-unsafe metrics as issues instead of " "refusing the conversion") + + exp = sub.add_parser( + "export", help="Apache Ossie semantic model -> Cube data model directory") + exp.add_argument("-i", "--input", required=True, help="Ossie YAML file") + exp.add_argument("-o", "--output", required=True, + help="output directory for the Cube model files") + exp.add_argument("-d", "--dialect", + help="preferred Ossie expression dialect (e.g. SNOWFLAKE); " + "ANSI_SQL is always the fallback") + exp.add_argument("-b", "--base-cube", + help="dataset a generated view is rooted at (only used for a " + "model with no stashed views; default: the FK-sink dataset)") return parser @@ -97,6 +112,20 @@ def _report(issues): def main(argv=None): args = _build_parser().parse_args(argv) try: + if args.command == "export": + with open(args.input) as fh: + ossie_yaml = fh.read() + files, issues = convert_ossie_to_cube( + ossie_yaml, dialect=args.dialect, base_cube=args.base_cube) + for rel, text in files.items(): + dest = os.path.join(args.output, *rel.split("/")) + os.makedirs(os.path.dirname(dest) or ".", exist_ok=True) + with open(dest, "w") as fh: + fh.write(text) + print(f"Wrote {len(files)} file(s) to {args.output}", file=sys.stderr) + _report(issues) + return 0 + files = _read_model_dir(args.input) out, issues = convert_cube_to_ossie( files, model_name=args.name, view=args.view, diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 6e165e8e..35800a2f 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -121,6 +121,7 @@ def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=True) # individual members -- so the view, not any cube, is the model boundary. mapped_name = _pick_view(views, view, issues) mapped_view = views.get(mapped_name) or {} + cubes = _order_by_view(cubes, mapped_view) model = {"name": model_name or mapped_name or "cube_model"} if mapped_view.get("description"): @@ -131,7 +132,7 @@ def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=True) # Joins are decomposed first: a join with no Ossie form is parked on its # declaring cube's stash, which has to be known before the dataset is built. - relationships, extra_joins = _convert_joins(cubes, issues) + relationships, extra_joins = _convert_joins(cubes, sorted(extra_files), issues) datasets = [] pk_by_cube = {} @@ -177,6 +178,13 @@ def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=True) stash["extra_files"] = extra_files write_stash(model, stash) + # Foreign-vendor extensions a previous export parked on the mapped view are + # restored after the stash is written, so the CUBE entry stays first. + parked_exts = ((mapped_view.get("meta") or {}).get("ossie") or {}).get( + "custom_extensions") + if parked_exts: + model.setdefault("custom_extensions", []).extend(parked_exts) + return dump_yaml({"version": OSSIE_VERSION, "semantic_model": [model]}), issues @@ -264,6 +272,29 @@ def _as_named_list(value, what): f"{what}: expected a list or mapping, got {type(value).__name__}") +def _order_by_view(cubes, mapped_view): + """Order the datasets the way the mapped view presents them. + + The view is the model boundary, so its `cubes:` order is the order a Cube user + sees -- and carrying it over means the Ossie dataset order is meaningful rather + than an artifact of how the files happened to be named. A cube the view does + not include keeps its file position, after the ones it does. + """ + ranks = {} + for entry in mapped_view.get("cubes") or []: + if not isinstance(entry, dict): + continue + path = entry.get("join_path") + if not isinstance(path, str) or not path: + continue + leaf = path.split(".")[-1] + ranks.setdefault(leaf, len(ranks)) + if not ranks: + return cubes + order = sorted(cubes, key=lambda name: (ranks.get(name, len(ranks)),)) + return {name: cubes[name] for name in order} + + def _pick_view(views, requested, issues): if requested is not None: if requested not in views: @@ -296,7 +327,10 @@ def _ai_context_from_meta(meta): return parked text = meta.get("ai_context") if isinstance(text, str) and text.strip(): - return {"instructions": text.strip()} + # Kept verbatim rather than stripped: a folded block scalar carries a + # trailing newline, and normalizing it away here would make the round trip + # lossy for the sake of cosmetics. + return {"instructions": text} return None @@ -471,13 +505,17 @@ def _convert_geo_dimension(cname, dname, dim, issues): # --- joins ---------------------------------------------------------------------- -def _convert_joins(cubes, issues): +def _convert_joins(cubes, skipped_files, issues): """Turn every cube's `joins` into Ossie relationships. Ossie's `from` is always the many side. A `many_to_one` join declared on cube A points A(many) -> B(one) directly; a `one_to_many` join is flipped, and the declared side and type are stashed so export restores the original. + `skipped_files` names the input files that held no convertible cube, so a join + pointing into one of them explains itself rather than just reporting a missing + cube. + Returns (relationships, {cube name: [unconvertible join, ...]}). """ relationships = [] @@ -489,8 +527,13 @@ def _convert_joins(cubes, issues): target = require_str(join, "name", f"cube '{cname}': join") what = f"join '{cname}' -> '{target}'" if target not in cubes: + hint = "" + if skipped_files: + hint = (f"; note that no cube was converted from " + f"{', '.join(repr(f) for f in skipped_files)} -- if " + f"'{target}' is defined there, that is why") raise ConversionError( - f"{what}: '{target}' is not a cube in this model") + f"{what}: '{target}' is not a cube in this model{hint}") raw_rel = snake(require_str(join, "relationship", what)) rel_type = _RELATIONSHIP_ALIASES.get(raw_rel) if rel_type is None: @@ -754,7 +797,7 @@ def _convert_measures(cubes, pk_by_cube, fanned_out, issues): resolver = _MeasureResolver(cubes, pk_by_cube, issues) counts = {} - for (_cname, mname) in resolver.measures(): + for (_, mname) in resolver.measures(): counts[mname] = counts.get(mname, 0) + 1 metrics = [] @@ -828,6 +871,11 @@ def _convert_measure(cname, mname, metric_name, measure, resolver, fanned_out, snake(k): v for k, v in measure.items() if snake(k) not in ("description", "meta") } + elif sql is not None: + # The operand's exact Cube spelling: `{CUBE}.city` and `{CUBE.city}` are + # equivalent but not interchangeable byte-for-byte, and export cannot tell + # which one the author wrote from the Ossie expression alone. + stash["sql"] = sql if metric_name != mname: stash["name"] = mname if measure.get("title"): diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py new file mode 100644 index 00000000..0dc4d4a5 --- /dev/null +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -0,0 +1,711 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert an Apache Ossie semantic model to a Cube data model. + +Pure offline conversion. Produces the Cube model-directory layout: one +`model/cubes/.yml` per dataset and a `model/views/.yml` for the model +itself, plus -- when a prior import stashed them -- the original file paths and +every Cube-only construct restored verbatim. + +Ossie features Cube has no field for (`unique_keys`, foreign-vendor +`custom_extensions`, the structured form of `ai_context`) are parked under +`meta.ossie` rather than dropped, since Cube has a `meta` field at every level. +That keeps `Ossie -> Cube -> Ossie` lossless as well. + +Usage (CLI): + ossie-cube export -i model.yaml -o model/ [--dialect SNOWFLAKE] [--base-cube orders] +""" + +import re + +from ._common import ( + DATATYPE_TO_DIM_TYPE, + OSSIE_FUNC_TO_AGG, + OSSIE_VERSION, + ConversionError, + cube_file, + dump_yaml, + examples_of, + foreign_vendor_extensions, + instructions_of, + is_simple_identifier, + load_yaml, + ossie_expr_to_cube_sql, + parse_source, + pick_expression, + primary_key_operand, + read_stash, + require_str, + sanitize_name, + synonyms_of, + view_file, +) +from .converter_issues import IssueLog, IssueType + +# An aggregate call the exporter can turn back into a structured Cube measure. +_AGG_CALL_RE = re.compile( + r"^\s*(SUM|AVG|MIN|MAX|COUNT|APPROX_COUNT_DISTINCT)\s*\((.*)\)\s*$", + re.IGNORECASE | re.DOTALL, +) +_DISTINCT_RE = re.compile(r"^DISTINCT\s+(.+)$", re.IGNORECASE | re.DOTALL) + +# The order Cube's own YAML documentation and generators use, so exported files +# read the way a hand-authored model does. +_CUBE_KEY_ORDER = [ + "name", "sql_table", "sql", "title", "description", "meta", "joins", + "dimensions", "measures", "segments", +] +_DIM_KEY_ORDER = [ + "name", "sql", "type", "primary_key", "title", "description", "meta", +] +_MEASURE_KEY_ORDER = [ + "name", "sql", "type", "filters", "title", "description", "meta", +] + + +def convert_ossie_to_cube(ossie_yaml_str, dialect=None, base_cube=None): + """Parse Ossie YAML and return Cube model files as {relative filename: YAML str}. + + Returns (files, IssueLog). `dialect` prepends a warehouse dialect (e.g. + SNOWFLAKE) to the expression preference order; ANSI_SQL is always the fallback. + `base_cube` names the dataset a generated view is rooted at, and is only + consulted for a hand-authored Ossie model with no stashed views. + """ + root = load_yaml(ossie_yaml_str, "Ossie model") + if not isinstance(root, dict): + raise ConversionError("Invalid Ossie YAML: expected a mapping at the root") + version = str(root.get("version", "")) + if version != OSSIE_VERSION: + raise ConversionError( + f"Unsupported Ossie version '{version}'. Supported: {OSSIE_VERSION}") + models = root.get("semantic_model") + if not isinstance(models, list) or not models: + raise ConversionError("'semantic_model' must be a non-empty list") + + issues = IssueLog() + if len(models) > 1: + issues.add(IssueType.PARKED_IN_META, "model", + f"{len(models)} semantic models found; converting only the first") + return _convert_model(models[0], dialect, base_cube, issues) + + +def _convert_model(model, dialect, base_cube, issues): + name = model.get("name", "") + dataset_list = model.get("datasets") or [] + if not dataset_list: + raise ConversionError(f"Model '{name}' has no datasets") + + # Dataset -> cube names. A collision (including a case-insensitive duplicate, + # which sanitizes identically) fails loudly rather than merging. + cube_names = {} + taken = set() + for ds in dataset_list: + ds_name = require_str(ds, "name", f"Model '{name}': dataset") + cube_names[ds_name] = sanitize_name( + ds_name, f"Model '{name}': dataset", taken) + taken.add(cube_names[ds_name].lower()) + datasets = {ds["name"]: ds for ds in dataset_list} + + relationships = model.get("relationships") or [] + for rel in relationships: + scope = f"Model '{name}': relationship '{rel.get('name', '')}'" + if (require_str(rel, "from", scope) not in datasets + or require_str(rel, "to", scope) not in datasets): + raise ConversionError(f"{scope} references an unknown dataset") + + model_stash = read_stash(model) + + # Per-cube facts the join and measure stages need. + members_by_cube = {} + pk_by_cube = {} + for ds_name, ds in datasets.items(): + cname = cube_names[ds_name] + members_by_cube[cname] = { + sanitize_name(f["name"], f"dataset '{ds_name}': field", set()) + for f in (ds.get("fields") or []) + } + pk_by_cube[cname] = [str(c) for c in (ds.get("primary_key") or [])] + + joins_by_cube = _build_joins(relationships, cube_names, issues) + measures_by_cube = _build_measures( + model, cube_names, members_by_cube, pk_by_cube, datasets, relationships, + base_cube, dialect, issues) + + # Cubes, grouped by the file they belong in: several datasets can share one + # stashed original path, in which case they go back into the same file. + stashed_paths = model_stash.get("cube_files") or {} + files_content = {} + for ds_name, ds in datasets.items(): + cname = cube_names[ds_name] + cube = _build_cube(ds, cname, members_by_cube[cname], + joins_by_cube.get(cname), measures_by_cube.get(cname), + dialect, issues) + path = stashed_paths.get(cname) or cube_file(cname) + files_content.setdefault(path, {}).setdefault("cubes", []).append(cube) + + for vpath, view in _build_views(model, model_stash, cube_names, relationships, + datasets, base_cube, issues).items(): + files_content.setdefault(vpath, {}).setdefault("views", []).append(view) + + files = {path: dump_yaml(content) for path, content in files_content.items()} + + # Files a prior import could not convert (`.js` models, Jinja-templated YAML, + # non-model YAML) restore verbatim. + for fname, text in (model_stash.get("extra_files") or {}).items(): + files[fname] = text + return files, issues + + +# --- ai_context ----------------------------------------------------------------- + +def _ai_context_to_meta(ai_context): + """Split an Ossie `ai_context` into (Cube prose, parked original). + + Cube's `meta.ai_context` is free text, so the instructions go there verbatim + and any synonyms are appended as prose -- which is how Cube's own + documentation expresses them ("Common acronyms: LC = Lucky Charms"). The + structured original is parked under `meta.ossie.ai_context` whenever the prose + alone would not restore it, so the Ossie round trip stays exact. + """ + if not ai_context: + return None, None + instructions = instructions_of(ai_context) + synonyms = synonyms_of(ai_context) + examples = examples_of(ai_context) + + parts = [instructions] if instructions else [] + if synonyms: + parts.append("Also known as: " + ", ".join(str(s) for s in synonyms) + ".") + if examples: + parts.append("Example questions: " + + " ".join(str(e) for e in examples)) + prose = "\n".join(parts) if parts else None + + # Import reads a bare prose value back as {"instructions": prose}. Anything + # else -- a plain string, synonyms, examples, extra keys -- needs the original. + round_trips = (isinstance(ai_context, dict) + and set(ai_context) == {"instructions"} + and ai_context.get("instructions") == prose) + return prose, (None if round_trips else ai_context) + + +def _build_meta(ai_context, stashed_meta, parked_extra): + """Assemble a Cube `meta` from the Ossie AI context, a stashed original meta, + and anything Ossie-only that needs parking.""" + prose, parked_ai = _ai_context_to_meta(ai_context) + meta = {} + if prose: + meta["ai_context"] = prose + for key, value in (stashed_meta or {}).items(): + meta[key] = value + parked = dict(parked_extra or {}) + if parked_ai is not None: + parked["ai_context"] = parked_ai + if parked: + meta["ossie"] = parked + return meta + + +def _ordered(obj, order): + """Re-key a dict so the well-known Cube keys come first, in their documented + order, with anything restored from the stash following.""" + out = {k: obj[k] for k in order if k in obj} + for key, value in obj.items(): + if key not in out: + out[key] = value + return out + + +# --- cubes ---------------------------------------------------------------------- + +def _build_cube(ds, cname, members, joins, measures, dialect, issues): + ds_name = ds["name"] + scope = f"dataset '{ds_name}'" + stash = read_stash(ds) + cube = {"name": cname} + + kind, value = parse_source(ds.get("source"), ds_name) + cube[kind] = value + if ds.get("description"): + cube["description"] = ds["description"] + + parked = {} + if ds.get("unique_keys"): + parked["unique_keys"] = [list(k) for k in ds["unique_keys"]] + issues.add(IssueType.PARKED_IN_META, scope, + "unique_keys have no Cube field; parked under meta.ossie") + foreign = foreign_vendor_extensions(ds) + if foreign: + parked["custom_extensions"] = foreign + cube_extras = dict(stash.get("cube_extras") or {}) + stashed_meta = cube_extras.pop("meta", None) + meta = _build_meta(ds.get("ai_context"), stashed_meta, parked) + if meta: + cube["meta"] = meta + if "ai_context" in meta: + issues.add(IssueType.CUBE_LEVEL_AI_CONTEXT_INERT, scope, + "Cube's agent reads ai_context only on views and members, " + "so this cube-level value has no effect in Cube") + + dimensions, covered = _build_dimensions(ds, cname, members, dialect, issues) + # A primary-key column no field covers still has to exist as a dimension for + # Cube to join or roll up the cube. + pk_names = [] + for col in (ds.get("primary_key") or []): + col = str(col) + if col in covered: + pk_names.append(covered[col]) + continue + issues.add(IssueType.PARKED_IN_META, scope, + f"primary key column '{col}' has no field; emitted as a " + f"non-public dimension with type 'string' (Cube requires a type " + f"and Ossie carries none here)") + synth = {"name": col, "sql": col, "type": "string", + "primary_key": True, "public": False} + dimensions.append(synth) + covered[col] = col + pk_names.append(col) + for dim in dimensions: + if dim["name"] in pk_names: + dim["primary_key"] = True + + dimensions.extend( + _ordered(dict(d, name=n), _DIM_KEY_ORDER) + for n, d in (stash.get("extra_dimensions") or {}).items() + ) + if dimensions: + cube["dimensions"] = [_ordered(d, _DIM_KEY_ORDER) for d in dimensions] + + joins = list(joins or []) + # Joins a prior import could not represent go back at their original indices. + for item in sorted(stash.get("extra_joins") or [], key=lambda x: x.get("index", 0)): + joins.insert(min(item.get("index", 0), len(joins)), item["join"]) + if joins: + cube["joins"] = joins + if measures: + cube["measures"] = [_ordered(m, _MEASURE_KEY_ORDER) for m in measures] + + for key, value in cube_extras.items(): + cube[key] = value + return _ordered(cube, _CUBE_KEY_ORDER) + + +def _build_dimensions(ds, cname, members, dialect, issues): + """Build a cube's dimensions from an Ossie dataset's fields. + + Returns (dimensions, {column or field name: dimension name}) -- the second + value is what primary-key resolution matches against. Fields carrying a `geo` + stash are re-merged into the single Cube dimension they were split from. + """ + ds_name = ds["name"] + dimensions = [] + covered = {} + taken = set() + geo_parts = {} + for field in (ds.get("fields") or []): + fname = require_str(field, "name", f"dataset '{ds_name}': field") + stash = read_stash(field) + if "geo" in stash: + geo = stash["geo"] + slot = geo_parts.setdefault(geo["of"], {"index": len(dimensions)}) + slot[geo["part"]] = geo["sql"] + if "host" in geo: + slot["host"] = geo["host"] + if geo["part"] == "latitude": + dimensions.append(None) # placeholder, filled in below + continue + + dname = sanitize_name(fname, f"dataset '{ds_name}': field", taken) + taken.add(dname.lower()) + expr = pick_expression(field.get("expression"), dialect) + if expr is None: + issues.add(IssueType.NO_USABLE_DIALECT, f"{ds_name}.{fname}", + "no ANSI_SQL or preferred-dialect expression; field dropped") + continue + + dim = {"name": dname} + if "sql" in stash: + # The exact Cube spelling a prior import saw. + dim["sql"] = stash["sql"] + else: + dim["sql"] = ossie_expr_to_cube_sql(expr, cname, members, ()) + dim["type"] = _dimension_type(field, stash, f"{ds_name}.{fname}", issues) + if field.get("label"): + dim["title"] = field["label"] + if field.get("description"): + dim["description"] = field["description"] + parked = {} + foreign = foreign_vendor_extensions(field) + if foreign: + parked["custom_extensions"] = foreign + extras = {k: v for k, v in stash.items() if k not in ("sql", "type", "meta")} + meta = _build_meta(field.get("ai_context"), stash.get("meta"), parked) + if meta: + dim["meta"] = meta + for key, value in extras.items(): + dim[key] = value + + dimensions.append(dim) + covered[dname] = dname + if is_simple_identifier(expr): + covered[expr.strip()] = dname + + for of, slot in geo_parts.items(): + if "latitude" not in slot or "longitude" not in slot: + raise ConversionError( + f"dataset '{ds_name}': geo dimension '{of}' is missing its " + f"{'longitude' if 'latitude' in slot else 'latitude'} half") + dim = {"name": of, "type": "geo", + "latitude": {"sql": slot["latitude"]}, + "longitude": {"sql": slot["longitude"]}} + for key, value in (slot.get("host") or {}).items(): + dim[key] = value + dimensions[slot["index"]] = dim + covered[of] = of + return [d for d in dimensions if d is not None], covered + + +def _dimension_type(field, stash, scope, issues): + """Choose the Cube `type`, which every dimension must declare.""" + if "type" in stash: + # Cube collapses Integer/Decimal/Float into `number`, so import parks the + # original rather than asserting an Ossie datatype; restore it here. + return stash["type"] + datatype = field.get("datatype") + explicit_is_time = (field.get("dimension") or {}).get("is_time") + if datatype: + ctype = DATATYPE_TO_DIM_TYPE.get(datatype) + if ctype is None: + raise ConversionError(f"{scope}: unknown datatype '{datatype}'") + if explicit_is_time is True and ctype != "time": + issues.add(IssueType.PARKED_IN_META, scope, + f"is_time is true but datatype '{datatype}' maps to Cube " + f"type '{ctype}'; Cube marks time dimensions by type, so " + f"the temporal role is not carried") + elif explicit_is_time is False and ctype == "time": + issues.add(IssueType.PARKED_IN_META, scope, + f"is_time is false but datatype '{datatype}' maps to Cube " + f"type 'time', which Cube always treats as a time dimension") + return ctype + if explicit_is_time: + return "time" + issues.add(IssueType.PARKED_IN_META, scope, + "no datatype; emitted as Cube type 'string', which Cube requires") + return "string" + + +# --- joins ---------------------------------------------------------------------- + +def _build_joins(relationships, cube_names, issues): + """Group Ossie relationships into per-cube `joins` lists. + + A stashed `declared_on`/`relationship` restores the original declaring side and + type. A hand-authored relationship is declared on its `from` (many) cube as + `many_to_one`, which is the orientation Ossie already guarantees. + """ + joins_by_cube = {} + for rel in relationships: + rname = rel.get("name", "") + from_cols = rel.get("from_columns") or [] + to_cols = rel.get("to_columns") or [] + if not isinstance(from_cols, list) or not isinstance(to_cols, list) \ + or not from_cols or not to_cols: + raise ConversionError( + f"Relationship '{rname}': from_columns and to_columns are required " + f"lists") + if len(from_cols) != len(to_cols): + raise ConversionError( + f"Relationship '{rname}': from_columns ({len(from_cols)}) and " + f"to_columns ({len(to_cols)}) must have the same length") + + stash = read_stash(rel) + from_cube = cube_names[rel["from"]] + to_cube = cube_names[rel["to"]] + declared_on = stash.get("declared_on") + relationship = stash.get("relationship", "many_to_one") + + if declared_on == to_cube: + # The import flipped a one_to_many (or kept a one_to_one) declared on + # the other side; flip back to the original orientation. + own, other = to_cube, from_cube + own_cols, other_cols = to_cols, from_cols + else: + own, other = from_cube, to_cube + own_cols, other_cols = from_cols, to_cols + + join = {"name": other, "relationship": relationship} + if "sql" in stash: + join["sql"] = stash["sql"] + else: + join["sql"] = " AND ".join( + "{CUBE}." + str(a) + " = {" + other + "." + str(b) + "}" + for a, b in zip(own_cols, other_cols)) + for key, value in stash.items(): + if key not in ("declared_on", "relationship", "sql"): + join[key] = value + if rel.get("ai_context"): + issues.add(IssueType.PARKED_IN_META, f"relationship '{rname}'", + "Cube joins carry no metadata, so relationship ai_context " + "has no home; dropped") + joins_by_cube.setdefault(own, []).append( + _ordered(join, ["name", "sql", "relationship"])) + return joins_by_cube + + +# --- measures ------------------------------------------------------------------- + +def _build_measures(model, cube_names, members_by_cube, pk_by_cube, datasets, + relationships, base_cube, dialect, issues): + """Group Ossie metrics into per-cube `measures` lists.""" + name = model.get("name", "") + sanitized = set(cube_names.values()) + base_cache = [] + + def resolve_base(): + if not base_cache: + base_cache.append(cube_names[_pick_base_cube( + name, datasets, relationships, base_cube)]) + return base_cache[0] + + measures_by_cube = {} + for metric in (model.get("metrics") or []): + mname_raw = require_str(metric, "name", "metric") + scope = f"metric '{mname_raw}'" + stash = read_stash(metric) + mname = stash.get("name") or sanitize_name(mname_raw, scope, set()) + + if "measure" in stash: + # A prior import stashed the original measure (a filtered, calculated, + # or otherwise non-reconstructible one); restore it verbatim and + # re-inject the natively mapped metadata. + measure = dict(stash["measure"]) + measure["name"] = mname + _apply_measure_metadata(metric, measure, stash) + target = stash.get("cube") or resolve_base() + _place(measures_by_cube, target, measure, name) + continue + + expr = pick_expression(metric.get("expression"), dialect) + if expr is None: + issues.add(IssueType.NO_USABLE_DIALECT, scope, + "no ANSI_SQL or preferred-dialect expression; metric dropped") + continue + + referenced = { + m.group(1) for m in re.finditer( + r"(?)` is Cube's bare `type: count` -- + which is how import renders it, precisely because that form stays correct + whether or not the cube is fanned out. A recognized aggregate over a single + operand becomes the matching `type` plus `sql`; anything else becomes a + calculated `type: number` measure carrying the whole expression. + """ + measure = {"name": mname} + m = _AGG_CALL_RE.match(expr) + if m and _balanced(m.group(2)): + func, inner = m.group(1).upper(), m.group(2).strip() + distinct = _DISTINCT_RE.match(inner) + if func == "COUNT" and distinct: + inner = distinct.group(1).strip() + if primary_key and inner == primary_key_operand(target, primary_key): + measure["type"] = "count" + return measure + func = "COUNT_DISTINCT" + if func == "COUNT" and inner == "*": + measure["type"] = "count" + return measure + agg = OSSIE_FUNC_TO_AGG.get(func) or ("count" if func == "COUNT" else None) + if agg is not None: + measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( + inner, target, members, sanitized) + measure["type"] = agg + return measure + + # A ratio, a window expression, or a multi-dataset aggregate: Cube expresses + # these as a calculated measure whose sql carries the aggregation. + measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( + expr, target, members, sanitized) + measure["type"] = "number" + if len({ + ref for ref in re.findall( + r"(? 1: + issues.add(IssueType.PARKED_IN_META, scope, + f"expression spans several datasets; emitted as a calculated " + f"measure on cube '{target}' -- verify the join path") + return measure + + +def _apply_measure_metadata(metric, measure, stash): + if stash.get("title"): + measure["title"] = stash["title"] + if metric.get("description"): + measure["description"] = metric["description"] + parked = {} + foreign = foreign_vendor_extensions(metric) + if foreign: + parked["custom_extensions"] = foreign + meta = _build_meta(metric.get("ai_context"), stash.get("meta"), parked) + if meta: + measure["meta"] = meta + + +def _balanced(s): + depth = 0 + for ch in s: + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth < 0: + return False + return depth == 0 + + +# --- views ---------------------------------------------------------------------- + +def _build_views(model, model_stash, cube_names, relationships, datasets, + base_cube, issues): + """Return {file path: view dict}. + + Stashed views restore verbatim, with the natively mapped description and AI + context re-injected on the mapped one. The `views` stash key being *present* -- + even empty -- means the original Cube model's view set is known, so a view is + only generated for hand-authored Ossie. + """ + # The model's foreign-vendor extensions have no Cube field, so they ride on the + # view that represents the model -- the mapped one, or the generated one. + parked = {} + foreign = foreign_vendor_extensions(model) + if foreign: + parked["custom_extensions"] = foreign + + out = {} + if "views" in model_stash: + mapped = model_stash.get("mapped_view") + paths = model_stash.get("view_files") or {} + if foreign and mapped is None: + issues.add(IssueType.PARKED_IN_META, "model", + "no mapped view to park foreign-vendor custom_extensions on; " + "they have no Cube home and are dropped") + for vname, view in (model_stash["views"] or {}).items(): + view = dict(view) + if vname == mapped: + if model.get("description"): + view["description"] = model["description"] + meta = _build_meta(model.get("ai_context"), view.get("meta"), parked) + if meta: + view["meta"] = meta + out[paths.get(vname) or view_file(vname)] = view + return out + + vname = sanitize_name(model.get("name", "model"), "Model", set()) + view = {"name": vname} + if model.get("description"): + view["description"] = model["description"] + meta = _build_meta(model.get("ai_context"), None, parked) + if meta: + view["meta"] = meta + view["cubes"] = _view_cubes( + cube_names, relationships, + cube_names[_pick_base_cube(model.get("name", ""), datasets, + relationships, base_cube)]) + out[view_file(vname)] = view + return out + + +def _view_cubes(cube_names, relationships, base): + """Build a generated view's `cubes:` list: the base cube plus every cube + reachable from it, each addressed by its full `join_path`.""" + adjacency = {} + for rel in relationships: + a, b = cube_names[rel["from"]], cube_names[rel["to"]] + adjacency.setdefault(a, []).append(b) + adjacency.setdefault(b, []).append(a) + + entries = [{"join_path": base, "includes": "*"}] + paths = {base: base} + queue = [base] + while queue: + current = queue.pop(0) + for neighbor in adjacency.get(current, []): + if neighbor in paths: + continue + paths[neighbor] = f"{paths[current]}.{neighbor}" + entries.append({"join_path": paths[neighbor], "includes": "*"}) + queue.append(neighbor) + # A cube no relationship reaches cannot be addressed by a join path, so it is + # simply not part of the generated view; it is still exported and joinable. + return entries + + +def _pick_base_cube(model_name, datasets, relationships, hint): + """Choose the cube a generated view is rooted at: an explicit hint, else the + dataset that is never a relationship `to` (the FK sink of a many-to-one star).""" + if hint is not None: + if hint not in datasets: + raise ConversionError( + f"Model '{model_name}': requested base cube '{hint}' is not a dataset") + return hint + if len(datasets) == 1: + return next(iter(datasets)) + if not relationships: + raise ConversionError( + f"Model '{model_name}': {len(datasets)} datasets but no relationships; " + f"name the view's base cube with --base-cube.") + incoming = {name: 0 for name in datasets} + for rel in relationships: + incoming[rel["to"]] += 1 + roots = [n for n in datasets if incoming[n] == 0] + if not roots: + raise ConversionError( + f"Model '{model_name}': every dataset is a relationship target (the " + f"graph has a cycle); name the view's base cube with --base-cube.") + if len(roots) > 1: + raise ConversionError( + f"Model '{model_name}': multiple candidate base cubes {sorted(roots)}; " + f"name the view's base cube with --base-cube.") + return roots[0] diff --git a/converters/cube/tests/_roundtrip_helpers.py b/converters/cube/tests/_roundtrip_helpers.py new file mode 100644 index 00000000..f7d20f16 --- /dev/null +++ b/converters/cube/tests/_roundtrip_helpers.py @@ -0,0 +1,210 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared model builders and round-trip assertions for property-based tests. + +This module is deliberately free of any third-party test dependency (no +hypothesis, no pytest) so the generation and assertion logic can run two ways: + + - driven by Hypothesis strategies (see test_roundtrip_properties.py), and + - driven by a plain seeded `random.Random` (RandomRnd below), which is how the + logic is exercised when hypothesis is not installed. + +Both drivers implement the small `Rnd` interface (chance/count/pick/text); the +builders depend only on that interface, so the generated model space is identical +either way. + +The builders generate within the *round-trippable subset* -- the shapes the +converter reproduces exactly. Known normalizations are avoided by construction: + + - names are generated already valid as Cube identifiers, so the sanitizer never + renames anything; + - the topology is a star with a single fact, so there is one unambiguous FK sink + and no cycle; + - every cube declares a primary key, which a bare `type: count` needs; + - `sum`/`avg` measures are only placed on the fact cube, which is never the + `to` side of a join -- a non-idempotent aggregate on a fanned-out cube is + refused by design, and that refusal has its own targeted tests; + - a view lists every cube, so dataset ordering is pinned by the view rather + than by file names. + +Name fuzzing (collisions, reserved words) and the fan-out refusal are left to the +targeted unit tests, which assert the converter *rejects* or *reports* those. +""" + +import random +import string + +from ossie_cube import convert_cube_to_ossie, convert_ossie_to_cube +from ossie_cube._common import dump_yaml, load_yaml + +# Aggregates whose value survives duplicate rows, so they are safe on any cube. +IDEMPOTENT_AGGS = ["count_distinct", "count_distinct_approx", "min", "max"] +# Aggregates only placed on the fact cube; see the module docstring. +FACT_ONLY_AGGS = ["sum", "avg"] + +DIM_TYPES = ["string", "number", "boolean", "time"] + + +class RandomRnd: + """The `Rnd` interface backed by a seeded `random.Random`.""" + + def __init__(self, seed): + self.r = random.Random(seed) + + def chance(self, p=0.5): + return self.r.random() < p + + def count(self, lo, hi): + return self.r.randint(lo, hi) + + def pick(self, seq): + return self.r.choice(list(seq)) + + def text(self): + # Alphanumeric with optional interior spaces; no leading/trailing space and + # no YAML-special characters, so the value survives a dump/load cycle + # verbatim. + alnum = string.ascii_letters + string.digits + words = [] + for _ in range(self.r.randint(1, 3)): + words.append("".join( + self.r.choice(alnum) for _ in range(self.r.randint(1, 6)))) + return " ".join(words) + + +def build_cube_model(rnd): + """Generate a Cube model as {relative filename: YAML str}.""" + dim_count = rnd.count(1, 3) + dim_names = [f"dim_{i}" for i in range(dim_count)] + fact = "fact" + + cubes = {} + cubes[fact] = _build_cube(rnd, fact, is_fact=True, dim_names=dim_names) + for name in dim_names: + cubes[name] = _build_cube(rnd, name, is_fact=False, dim_names=()) + + files = {} + for name, cube in cubes.items(): + files[f"model/cubes/{name}.yml"] = dump_yaml({"cubes": [cube]}) + + view = {"name": "main"} + if rnd.chance(0.6): + view["description"] = rnd.text() + if rnd.chance(0.6): + view["meta"] = {"ai_context": rnd.text()} + view["cubes"] = ( + [{"join_path": fact, "includes": "*"}] + + [{"join_path": f"{fact}.{d}", "includes": "*"} for d in dim_names] + ) + files["model/views/main.yml"] = dump_yaml({"views": [view]}) + return files + + +def _build_cube(rnd, name, is_fact, dim_names): + cube = {"name": name} + if rnd.chance(0.3): + cube["sql"] = f"SELECT * FROM raw.{name}" + else: + cube["sql_table"] = f"public.{name}" + if rnd.chance(0.5): + cube["description"] = rnd.text() + + if is_fact and dim_names: + cube["joins"] = [ + {"name": d, "sql": "{CUBE}." + f"{d}_id" + " = {" + f"{d}.id" + "}", + "relationship": "many_to_one"} + for d in dim_names + ] + + dimensions = [{"name": "id", "sql": "id", "type": "number", + "primary_key": True}] + for d in dim_names: + dimensions.append({"name": f"{d}_id", "sql": f"{d}_id", "type": "number"}) + for i in range(rnd.count(0, 3)): + dimensions.append(_build_dimension(rnd, f"attr_{i}")) + if rnd.chance(0.25): + dimensions.append({ + "name": "place", "type": "geo", + "latitude": {"sql": "{CUBE}.lat"}, + "longitude": {"sql": "{CUBE}.lon"}, + }) + cube["dimensions"] = dimensions + + # Every cube carries a bare `count`, which collides across cubes and so + # exercises the `__` qualification on import. + measures = [{"name": "count", "type": "count"}] + aggs = IDEMPOTENT_AGGS + (FACT_ONLY_AGGS if is_fact else []) + for i in range(rnd.count(0, 2)): + measure = {"name": f"m_{i}", "sql": "{CUBE}.value", "type": rnd.pick(aggs)} + if rnd.chance(0.4): + measure["description"] = rnd.text() + if rnd.chance(0.3): + measure["meta"] = {"ai_context": rnd.text()} + if rnd.chance(0.25): + measure["format"] = "currency" + measures.append(measure) + cube["measures"] = measures + return cube + + +def _build_dimension(rnd, name): + dtype = rnd.pick(DIM_TYPES) + dim = {"name": name, "type": dtype} + if rnd.chance(0.3): + # A computed expression, which import translates and stashes verbatim. + dim["sql"] = "LOWER({CUBE}." + name + ")" if dtype == "string" \ + else "{CUBE}." + name + else: + dim["sql"] = name + if rnd.chance(0.4): + dim["title"] = rnd.text() + if rnd.chance(0.4): + dim["description"] = rnd.text() + if rnd.chance(0.3): + dim["meta"] = {"ai_context": rnd.text()} + if rnd.chance(0.2): + dim["format"] = "percent" if dtype == "number" else None + if dim["format"] is None: + del dim["format"] + return dim + + +def _parse_files(files): + return {name: load_yaml(text, name) for name, text in files.items()} + + +def assert_cube_roundtrip_is_lossless(files): + """Cube -> Ossie -> Cube reproduces the model structurally.""" + ossie, _ = convert_cube_to_ossie(files) + files2, _ = convert_ossie_to_cube(ossie) + assert _parse_files(files2) == _parse_files(files), ( + "Cube -> Ossie -> Cube changed the model") + + +def assert_ossie_roundtrip_is_lossless(files): + """Ossie -> Cube -> Ossie reproduces the model too.""" + ossie, _ = convert_cube_to_ossie(files) + files2, _ = convert_ossie_to_cube(ossie) + ossie2, _ = convert_cube_to_ossie(files2) + assert load_yaml(ossie2) == load_yaml(ossie), ( + "Ossie -> Cube -> Ossie changed the model") + + +def check_model(files): + assert_cube_roundtrip_is_lossless(files) + assert_ossie_roundtrip_is_lossless(files) diff --git a/converters/cube/tests/_util.py b/converters/cube/tests/_util.py index feb7e253..bf11b481 100644 --- a/converters/cube/tests/_util.py +++ b/converters/cube/tests/_util.py @@ -46,6 +46,20 @@ def parse(yaml_str): return load_yaml(yaml_str) +def parse_files(files): + """Parse every file of a Cube model dict for structural comparison. + + Comments and key order are not part of the data model, so round-trip fidelity + is asserted on the parsed structures. A non-YAML file (a `.js` model preserved + verbatim) is compared as text. + """ + out = {} + for name, text in files.items(): + out[name] = (load_yaml(text, name) if name.lower().endswith((".yml", ".yaml")) + else text) + return out + + def model_of(ossie_yaml): """The sole semantic model of an Ossie document.""" doc = parse(ossie_yaml) diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/customer.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/customer.yml new file mode 100644 index 00000000..206ea8d8 --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/customer.yml @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then +# kept as a fixture: the converter guide asks every converter to use the TPC-DS +# model as its baseline. Exercises a five-cube star, cross-cube calculated +# measures, a synthesized primary-key dimension, and meta.ossie parking. + +cubes: +- name: customer + sql_table: tpcds.public.customer + description: Customer dimension with demographic information + meta: + ai_context: 'Also known as: customers, shoppers, buyers.' + ossie: + unique_keys: + - - c_customer_sk + ai_context: + synonyms: + - customers + - shoppers + - buyers + dimensions: + - name: c_customer_sk + sql: c_customer_sk + type: number + primary_key: true + description: Surrogate key for customer + - name: c_customer_id + sql: c_customer_id + type: string + description: Business key for customer + meta: + ai_context: 'Also known as: customer ID, customer number.' + ossie: + ai_context: + synonyms: + - customer ID + - customer number + - name: c_first_name + sql: c_first_name + type: string + description: Customer first name + - name: c_last_name + sql: c_last_name + type: string + description: Customer last name + - name: customer_full_name + sql: c_first_name || ' ' || c_last_name + type: string + description: Customer full name (computed field) + meta: + ai_context: 'Also known as: full name, customer name.' + ossie: + ai_context: + synonyms: + - full name + - customer name + - name: c_email_address + sql: c_email_address + type: string + description: Customer email address + meta: + ai_context: 'Also known as: email, contact.' + ossie: + ai_context: + synonyms: + - email + - contact diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/date_dim.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/date_dim.yml new file mode 100644 index 00000000..8855e811 --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/date_dim.yml @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then +# kept as a fixture: the converter guide asks every converter to use the TPC-DS +# model as its baseline. Exercises a five-cube star, cross-cube calculated +# measures, a synthesized primary-key dimension, and meta.ossie parking. + +cubes: +- name: date_dim + sql_table: tpcds.public.date_dim + description: Date dimension with calendar attributes + meta: + ai_context: 'Also known as: calendar, dates, time periods.' + ossie: + unique_keys: + - - d_date_sk + ai_context: + synonyms: + - calendar + - dates + - time periods + dimensions: + - name: d_date_sk + sql: d_date_sk + type: number + primary_key: true + description: Surrogate key for date + - name: d_date + sql: d_date + type: time + description: Actual date value + meta: + ai_context: 'Also known as: date, calendar date.' + ossie: + ai_context: + synonyms: + - date + - calendar date + - name: d_year + sql: d_year + type: number + description: Year + meta: + ai_context: 'Also known as: year.' + ossie: + ai_context: + synonyms: + - year + - name: d_quarter_name + sql: d_quarter_name + type: time + description: Quarter name (e.g., 2024Q1) + meta: + ai_context: 'Also known as: quarter, fiscal quarter.' + ossie: + ai_context: + synonyms: + - quarter + - fiscal quarter + - name: d_month_name + sql: d_month_name + type: time + description: Month name + meta: + ai_context: 'Also known as: month.' + ossie: + ai_context: + synonyms: + - month diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/item.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/item.yml new file mode 100644 index 00000000..79cf11de --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/item.yml @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then +# kept as a fixture: the converter guide asks every converter to use the TPC-DS +# model as its baseline. Exercises a five-cube star, cross-cube calculated +# measures, a synthesized primary-key dimension, and meta.ossie parking. + +cubes: +- name: item + sql_table: tpcds.public.item + description: Item/Product dimension with product attributes + meta: + ai_context: 'Also known as: products, items, merchandise.' + ossie: + unique_keys: + - - i_item_sk + ai_context: + synonyms: + - products + - items + - merchandise + dimensions: + - name: i_item_sk + sql: i_item_sk + type: number + primary_key: true + description: Surrogate key for item + - name: i_item_id + sql: i_item_id + type: string + description: Business key for item + meta: + ai_context: 'Also known as: item ID, product ID, SKU.' + ossie: + ai_context: + synonyms: + - item ID + - product ID + - SKU + - name: i_item_desc + sql: i_item_desc + type: string + description: Item description + meta: + ai_context: 'Also known as: product description, item name.' + ossie: + ai_context: + synonyms: + - product description + - item name + - name: i_brand + sql: i_brand + type: string + description: Brand name + meta: + ai_context: 'Also known as: brand, manufacturer.' + ossie: + ai_context: + synonyms: + - brand + - manufacturer + - name: i_category + sql: i_category + type: string + description: Item category + meta: + ai_context: 'Also known as: product category, department.' + ossie: + ai_context: + synonyms: + - product category + - department + - name: i_current_price + sql: i_current_price + type: number + description: Current price of the item + meta: + ai_context: 'Also known as: price, list price.' + ossie: + ai_context: + synonyms: + - price + - list price diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store.yml new file mode 100644 index 00000000..c3d59832 --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store.yml @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then +# kept as a fixture: the converter guide asks every converter to use the TPC-DS +# model as its baseline. Exercises a five-cube star, cross-cube calculated +# measures, a synthesized primary-key dimension, and meta.ossie parking. + +cubes: +- name: store + sql_table: tpcds.public.store + description: Store dimension with location and store attributes + meta: + ai_context: 'Also known as: stores, retail locations, branches.' + ossie: + unique_keys: + - - s_store_id + ai_context: + synonyms: + - stores + - retail locations + - branches + dimensions: + - name: s_store_sk + sql: s_store_sk + type: number + primary_key: true + description: Surrogate key for store + - name: s_store_id + sql: s_store_id + type: string + description: Business key for store + meta: + ai_context: 'Also known as: store ID, store number.' + ossie: + ai_context: + synonyms: + - store ID + - store number + - name: s_store_name + sql: s_store_name + type: string + description: Store name + meta: + ai_context: 'Also known as: store name, location name.' + ossie: + ai_context: + synonyms: + - store name + - location name + - name: s_city + sql: s_city + type: string + description: City where store is located + meta: + ai_context: 'Also known as: city, location.' + ossie: + ai_context: + synonyms: + - city + - location + - name: s_state + sql: s_state + type: string + description: State where store is located + meta: + ai_context: 'Also known as: state, region.' + ossie: + ai_context: + synonyms: + - state + - region + - name: s_number_employees + sql: s_number_employees + type: number + description: Number of employees at the store + meta: + ai_context: 'Also known as: employee count, staff size.' + ossie: + ai_context: + synonyms: + - employee count + - staff size diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml new file mode 100644 index 00000000..48f2b70f --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml @@ -0,0 +1,211 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then +# kept as a fixture: the converter guide asks every converter to use the TPC-DS +# model as its baseline. Exercises a five-cube star, cross-cube calculated +# measures, a synthesized primary-key dimension, and meta.ossie parking. + +cubes: +- name: store_sales + sql_table: tpcds.public.store_sales + description: Fact table containing all store sales transactions + meta: + ai_context: 'Also known as: sales transactions, store purchases, retail sales, + POS data.' + ossie: + unique_keys: + - - ss_item_sk + - ss_ticket_number + ai_context: + synonyms: + - sales transactions + - store purchases + - retail sales + - POS data + joins: + - name: date_dim + sql: '{CUBE}.ss_sold_date_sk = {date_dim.d_date_sk}' + relationship: many_to_one + - name: customer + sql: '{CUBE}.ss_customer_sk = {customer.c_customer_sk}' + relationship: many_to_one + - name: item + sql: '{CUBE}.ss_item_sk = {item.i_item_sk}' + relationship: many_to_one + - name: store + sql: '{CUBE}.ss_store_sk = {store.s_store_sk}' + relationship: many_to_one + dimensions: + - name: ss_sold_date_sk + sql: ss_sold_date_sk + type: number + description: Foreign key to date dimension + meta: + ai_context: 'Also known as: sale date, transaction date.' + ossie: + ai_context: + synonyms: + - sale date + - transaction date + - name: ss_item_sk + sql: ss_item_sk + type: number + primary_key: true + description: Foreign key to item dimension + meta: + ai_context: 'Also known as: product, item.' + ossie: + ai_context: + synonyms: + - product + - item + - name: ss_customer_sk + sql: ss_customer_sk + type: number + description: Foreign key to customer dimension + meta: + ai_context: 'Also known as: customer, buyer.' + ossie: + ai_context: + synonyms: + - customer + - buyer + - name: ss_store_sk + sql: ss_store_sk + type: number + description: Foreign key to store dimension + meta: + ai_context: 'Also known as: store, location.' + ossie: + ai_context: + synonyms: + - store + - location + - name: ss_quantity + sql: ss_quantity + type: number + description: Quantity of items sold + meta: + ai_context: 'Also known as: units sold, quantity.' + ossie: + ai_context: + synonyms: + - units sold + - quantity + - name: ss_sales_price + sql: ss_sales_price + type: number + description: Sales price per unit + meta: + ai_context: 'Also known as: unit price, price.' + ossie: + ai_context: + synonyms: + - unit price + - price + - name: ss_ext_sales_price + sql: ss_ext_sales_price + type: number + description: Extended sales price (quantity * price) + meta: + ai_context: 'Also known as: total price, line total.' + ossie: + ai_context: + synonyms: + - total price + - line total + - name: ss_net_profit + sql: ss_net_profit + type: number + description: Net profit from the sale + meta: + ai_context: 'Also known as: profit, margin.' + ossie: + ai_context: + synonyms: + - profit + - margin + - name: ss_ticket_number + sql: ss_ticket_number + type: string + primary_key: true + public: false + measures: + - name: total_sales + sql: '{CUBE.ss_ext_sales_price}' + type: sum + description: Total sales revenue across all transactions + meta: + ai_context: 'Also known as: total revenue, gross sales, sales amount.' + ossie: + ai_context: + synonyms: + - total revenue + - gross sales + - sales amount + - name: total_profit + sql: '{CUBE.ss_net_profit}' + type: sum + description: Total net profit from store sales + meta: + ai_context: 'Also known as: net profit, total earnings, profit.' + ossie: + ai_context: + synonyms: + - net profit + - total earnings + - profit + - name: customer_lifetime_value + sql: SUM({CUBE.ss_ext_sales_price}) / COUNT(DISTINCT {customer.c_customer_sk}) + type: number + description: Average lifetime sales value per customer + meta: + ai_context: 'Also known as: CLV, LTV, customer value, lifetime revenue.' + ossie: + ai_context: + synonyms: + - CLV + - LTV + - customer value + - lifetime revenue + - name: sales_by_brand + sql: '{CUBE.ss_ext_sales_price}' + type: sum + description: Total sales by brand (requires grouping by item.i_brand) + meta: + ai_context: 'Also known as: brand sales, brand performance, brand revenue.' + ossie: + ai_context: + synonyms: + - brand sales + - brand performance + - brand revenue + - name: store_productivity + sql: SUM({CUBE.ss_ext_sales_price}) / NULLIF(SUM({store.s_number_employees}), + 0) + type: number + description: Sales per employee across stores + meta: + ai_context: 'Also known as: sales per employee, employee productivity, revenue + per employee.' + ossie: + ai_context: + synonyms: + - sales per employee + - employee productivity + - revenue per employee diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/views/tpcds_retail_model.yml b/converters/cube/tests/fixtures/tpcds_cube/model/views/tpcds_retail_model.yml new file mode 100644 index 00000000..b4694965 --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_cube/model/views/tpcds_retail_model.yml @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then +# kept as a fixture: the converter guide asks every converter to use the TPC-DS +# model as its baseline. Exercises a five-cube star, cross-cube calculated +# measures, a synthesized primary-key dimension, and meta.ossie parking. + +views: +- name: tpcds_retail_model + description: TPC-DS retail semantic model for sales and customer analytics + meta: + ai_context: Use this semantic model for retail analytics. It provides comprehensive + sales, customer, product, and store data from the TPC-DS benchmark. The model + supports time-based analysis, customer segmentation, product performance, and + store operations metrics. + ossie: + custom_extensions: + - vendor_name: SALESFORCE + data: | + { + "tableau_workbook_id": "tpcds_retail_dashboard", + "einstein_enabled": true, + "crm_sync": { + "enabled": true, + "sync_frequency": "daily", + "customer_mapping": "customer.c_customer_id -> Account.AccountNumber" + }, + "tableau_semantics": { + "published": true, + "version": "0.1.1" + } + } + - vendor_name: DBT + data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' + cubes: + - join_path: store_sales + includes: '*' + - join_path: store_sales.date_dim + includes: '*' + - join_path: store_sales.customer + includes: '*' + - join_path: store_sales.item + includes: '*' + - join_path: store_sales.store + includes: '*' diff --git a/converters/cube/tests/test_cube_to_osi.py b/converters/cube/tests/test_cube_to_osi.py index 15e07a95..44fdb14a 100644 --- a/converters/cube/tests/test_cube_to_osi.py +++ b/converters/cube/tests/test_cube_to_osi.py @@ -434,6 +434,27 @@ def test_jinja_templated_file_is_preserved_not_parsed(): assert issues.of_type(IssueType.TEMPLATED_MEMBER_DROPPED) +def test_join_into_a_skipped_file_explains_itself(): + """A file the converter had to skip whole (Jinja, `.js`) can leave a join + pointing at a cube that is no longer there. The error says so, rather than just + reporting a missing cube.""" + files = { + "model/cubes/dyn.yml": ( + "cubes:\n - name: users\n sql_table: t{{ suffix }}\n"), + "model/cubes/orders.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {users}.id\"\n" + " relationship: many_to_one\n" + ), + } + with pytest.raises(ConversionError, match="model/cubes/dyn.yml"): + convert_cube_to_ossie(files) + + def test_javascript_model_is_preserved_not_parsed(): files = { "model/cubes/orders.js": "cube(`orders`, { sql_table: `public.orders` });", diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py new file mode 100644 index 00000000..87099837 --- /dev/null +++ b/converters/cube/tests/test_osi_to_cube.py @@ -0,0 +1,429 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Apache Ossie semantic model -> Cube data model.""" + +import pytest +from _util import by_name, parse + +from ossie_cube import ConversionError, IssueType, convert_ossie_to_cube +from ossie_cube._common import OSSIE_VERSION + + +def _ossie(datasets, relationships="", metrics="", model_extra=""): + return (f"version: {OSSIE_VERSION}\n" + "semantic_model:\n" + "- name: shop\n" + f"{model_extra}" + " datasets:\n" + f"{datasets}" + f"{relationships}" + f"{metrics}") + + +_ORDERS = ( + " - name: orders\n" + " source: sales.public.orders\n" + " primary_key:\n" + " - id\n" + " fields:\n" + " - name: id\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: id\n" + " datatype: Integer\n" + " - name: amount\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: amount\n" + " datatype: Decimal\n" +) + + +def _cubes(files, path="model/cubes/orders.yml"): + return by_name(parse(files[path])["cubes"]) + + +# --- layout --------------------------------------------------------------------- + +def test_emits_one_file_per_cube_plus_a_view(): + files, _ = convert_ossie_to_cube(_ossie(_ORDERS)) + assert set(files) == {"model/cubes/orders.yml", "model/views/shop.yml"} + + +def test_version_is_enforced(): + with pytest.raises(ConversionError, match="Unsupported Ossie version"): + convert_ossie_to_cube("version: 9.9.9\nsemantic_model: []\n") + + +def test_model_without_datasets_is_rejected(): + with pytest.raises(ConversionError, match="no datasets"): + convert_ossie_to_cube( + f"version: {OSSIE_VERSION}\nsemantic_model:\n- name: shop\n datasets: []\n") + + +def test_relationship_to_unknown_dataset_is_rejected(): + rel = (" relationships:\n" + " - name: r\n from: orders\n to: ghosts\n" + " from_columns: [x]\n to_columns: [y]\n") + with pytest.raises(ConversionError, match="unknown dataset"): + convert_ossie_to_cube(_ossie(_ORDERS, rel)) + + +def test_mismatched_relationship_columns_are_rejected(): + rel = (" relationships:\n" + " - name: r\n from: orders\n to: orders\n" + " from_columns: [a, b]\n to_columns: [c]\n") + with pytest.raises(ConversionError, match="same length"): + convert_ossie_to_cube(_ossie(_ORDERS, rel)) + + +# --- datasets and fields -------------------------------------------------------- + +def test_source_becomes_sql_table_or_sql(): + files, _ = convert_ossie_to_cube(_ossie(_ORDERS)) + assert _cubes(files)["orders"]["sql_table"] == "sales.public.orders" + + query = _ORDERS.replace("source: sales.public.orders", + "source: SELECT * FROM raw.orders") + files, _ = convert_ossie_to_cube(_ossie(query)) + cube = _cubes(files)["orders"] + assert cube["sql"] == "SELECT * FROM raw.orders" + assert "sql_table" not in cube + + +def test_every_dimension_declares_a_type(): + """Cube's schema requires `type` on every dimension, so the converter always + emits one -- falling back to `string` with an issue when Ossie carries none.""" + no_type = ( + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: note\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: note\n" + ) + files, issues = convert_ossie_to_cube(_ossie(no_type)) + assert _cubes(files)["orders"]["dimensions"][0]["type"] == "string" + assert issues.of_type(IssueType.PARKED_IN_META) + + +@pytest.mark.parametrize("datatype,expected", [ + ("String", "string"), + ("Integer", "number"), + ("Decimal", "number"), + ("Float", "number"), + ("Boolean", "boolean"), + ("Date", "time"), + ("DateTime", "time"), + ("DateTimeTz", "time"), + ("Opaque", "string"), +]) +def test_datatype_maps_to_cube_type(datatype, expected): + ds = ( + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: f\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: f\n" + f" datatype: {datatype}\n" + ) + files, _ = convert_ossie_to_cube(_ossie(ds)) + assert _cubes(files)["orders"]["dimensions"][0]["type"] == expected + + +def test_is_time_on_a_non_temporal_datatype_is_reported(): + """Cube marks time dimensions by `type`, so an Integer year grain cannot carry + the temporal role -- that is a real loss and it is reported, not hidden.""" + ds = ( + " - name: date_dim\n" + " source: t\n" + " fields:\n" + " - name: d_year\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: d_year\n" + " datatype: Integer\n" + " dimension:\n" + " is_time: true\n" + ) + files, issues = convert_ossie_to_cube(_ossie(ds)) + dim = parse(files["model/cubes/date_dim.yml"])["cubes"][0]["dimensions"][0] + assert dim["type"] == "number" + detail = issues.of_type(IssueType.PARKED_IN_META)[0].detail + assert "temporal role is not carried" in detail + + +def test_primary_key_column_without_a_field_is_synthesized(): + ds = ( + " - name: orders\n" + " source: t\n" + " primary_key:\n" + " - ticket_no\n" + " fields:\n" + " - name: amount\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: amount\n" + " datatype: Decimal\n" + ) + files, issues = convert_ossie_to_cube(_ossie(ds)) + dims = by_name(_cubes(files)["orders"]["dimensions"]) + assert dims["ticket_no"] == { + "name": "ticket_no", "sql": "ticket_no", "type": "string", + "primary_key": True, "public": False} + assert issues.of_type(IssueType.PARKED_IN_META) + + +def test_field_name_is_sanitized_and_collisions_are_rejected(): + ds = ( + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: Order Status\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: status\n" + " - name: order status\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: status2\n" + ) + with pytest.raises(ConversionError, match="collides"): + convert_ossie_to_cube(_ossie(ds)) + + +def test_missing_dialect_drops_the_field_with_an_issue(): + ds = ( + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: f\n" + " expression:\n" + " dialects:\n" + " - dialect: MDX\n" + " expression: '[f]'\n" + ) + files, issues = convert_ossie_to_cube(_ossie(ds)) + assert "dimensions" not in _cubes(files)["orders"] + assert issues.of_type(IssueType.NO_USABLE_DIALECT) + + +def test_preferred_dialect_wins_over_ansi(): + ds = ( + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: email\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: LOWER(email)\n" + " - dialect: SNOWFLAKE\n" + " expression: LOWER(email)::VARCHAR\n" + " datatype: String\n" + ) + files, _ = convert_ossie_to_cube(_ossie(ds), dialect="SNOWFLAKE") + assert _cubes(files)["orders"]["dimensions"][0]["sql"] == "LOWER(email)::VARCHAR" + + +# --- joins ---------------------------------------------------------------------- + +_TWO_DATASETS = _ORDERS + ( + " - name: users\n" + " source: sales.public.users\n" + " primary_key:\n" + " - id\n" + " fields:\n" + " - name: id\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: id\n" + " datatype: Integer\n" +) +_REL = (" relationships:\n" + " - name: orders_to_users\n" + " from: orders\n" + " to: users\n" + " from_columns: [user_id]\n" + " to_columns: [id]\n") + + +def test_relationship_lands_on_the_many_side_as_many_to_one(): + files, _ = convert_ossie_to_cube(_ossie(_TWO_DATASETS, _REL)) + join = _cubes(files)["orders"]["joins"][0] + assert join == {"name": "users", "sql": "{CUBE}.user_id = {users.id}", + "relationship": "many_to_one"} + # The one side declares nothing; Cube needs the join on one side only. + assert "joins" not in _cubes(files, "model/cubes/users.yml")["users"] + + +def test_composite_relationship_becomes_an_and_chain(): + rel = (" relationships:\n" + " - name: r\n from: orders\n to: users\n" + " from_columns: [user_id, region]\n" + " to_columns: [id, region]\n") + files, _ = convert_ossie_to_cube(_ossie(_TWO_DATASETS, rel)) + assert _cubes(files)["orders"]["joins"][0]["sql"] == ( + "{CUBE}.user_id = {users.id} AND {CUBE}.region = {users.region}") + + +def test_relationship_ai_context_has_no_cube_home(): + rel = _REL + " ai_context:\n instructions: Join carefully.\n" + _, issues = convert_ossie_to_cube(_ossie(_TWO_DATASETS, rel)) + assert any("ai_context" in i.detail for i in issues.of_type( + IssueType.PARKED_IN_META)) + + +# --- metrics -------------------------------------------------------------------- + +def _metric(name, expr): + return (" metrics:\n" + f" - name: {name}\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + f" expression: {expr}\n") + + +@pytest.mark.parametrize("expr,expected", [ + ("SUM(orders.amount)", {"type": "sum", "sql": "{CUBE.amount}"}), + ("AVG(orders.amount)", {"type": "avg", "sql": "{CUBE.amount}"}), + ("MIN(orders.amount)", {"type": "min", "sql": "{CUBE.amount}"}), + ("MAX(orders.amount)", {"type": "max", "sql": "{CUBE.amount}"}), + ("COUNT(DISTINCT orders.amount)", + {"type": "count_distinct", "sql": "{CUBE.amount}"}), + ("APPROX_COUNT_DISTINCT(orders.amount)", + {"type": "count_distinct_approx", "sql": "{CUBE.amount}"}), + ("COUNT(*)", {"type": "count"}), +]) +def test_aggregate_expressions_become_structured_measures(expr, expected): + files, _ = convert_ossie_to_cube(_ossie(_ORDERS, metrics=_metric("m", expr))) + measure = _cubes(files)["orders"]["measures"][0] + assert {k: v for k, v in measure.items() if k != "name"} == expected + + +def test_count_distinct_over_the_primary_key_becomes_a_bare_count(): + """The inverse of the import rule: COUNT(DISTINCT ) is exactly Cube's + fan-out-safe `type: count`, so it round-trips back to the idiomatic form.""" + files, _ = convert_ossie_to_cube( + _ossie(_ORDERS, metrics=_metric("m", "COUNT(DISTINCT orders.id)"))) + measure = _cubes(files)["orders"]["measures"][0] + assert measure == {"name": "m", "type": "count"} + + +def test_declared_member_gets_a_member_reference_and_a_raw_column_does_not(): + """`{CUBE.member}` reuses a declared member's SQL and is compile-time checked; + `{CUBE}.column` passes a raw column through. The choice follows from whether + the dataset declares a field of that name.""" + files, _ = convert_ossie_to_cube( + _ossie(_ORDERS, metrics=_metric("m", "SUM(orders.shipping_fee)"))) + # `shipping_fee` is not a declared field, so it stays a raw column. + assert _cubes(files)["orders"]["measures"][0]["sql"] == "{CUBE}.shipping_fee" + + +def test_ratio_becomes_a_calculated_measure(): + files, issues = convert_ossie_to_cube(_ossie( + _TWO_DATASETS, _REL, + _metric("aov", "SUM(orders.amount) / COUNT(DISTINCT users.id)"))) + measure = _cubes(files)["orders"]["measures"][0] + assert measure["type"] == "number" + assert measure["sql"] == "SUM({CUBE.amount}) / COUNT(DISTINCT {users.id})" + assert any("spans several datasets" in i.detail + for i in issues.of_type(IssueType.PARKED_IN_META)) + + +def test_metric_lands_on_the_dataset_its_expression_references(): + files, _ = convert_ossie_to_cube(_ossie( + _TWO_DATASETS, _REL, _metric("users_seen", "COUNT(DISTINCT users.id)"))) + assert "measures" not in _cubes(files)["orders"] + assert _cubes(files, "model/cubes/users.yml")["users"]["measures"][0]["name"] == ( + "users_seen") + + +def test_two_metrics_colliding_on_one_cube_are_rejected(): + metrics = (" metrics:\n" + " - name: Total Amount\n" + " expression:\n dialects:\n - dialect: ANSI_SQL\n" + " expression: SUM(orders.amount)\n" + " - name: total amount\n" + " expression:\n dialects:\n - dialect: ANSI_SQL\n" + " expression: SUM(orders.id)\n") + with pytest.raises(ConversionError, match="two metrics map to measure"): + convert_ossie_to_cube(_ossie(_ORDERS, metrics=metrics)) + + +# --- views ---------------------------------------------------------------------- + +def test_generated_view_is_rooted_at_the_fk_sink(): + files, _ = convert_ossie_to_cube(_ossie(_TWO_DATASETS, _REL)) + view = parse(files["model/views/shop.yml"])["views"][0] + assert view["cubes"] == [ + {"join_path": "orders", "includes": "*"}, + {"join_path": "orders.users", "includes": "*"}, + ] + + +def test_ambiguous_base_cube_is_rejected_and_the_hint_resolves_it(): + two_facts = _TWO_DATASETS # no relationships at all + with pytest.raises(ConversionError, match="no relationships"): + convert_ossie_to_cube(_ossie(two_facts)) + files, _ = convert_ossie_to_cube(_ossie(two_facts), base_cube="orders") + assert parse(files["model/views/shop.yml"])["views"][0]["cubes"][0][ + "join_path"] == "orders" + + +def test_unknown_base_cube_is_rejected(): + with pytest.raises(ConversionError, match="not a dataset"): + convert_ossie_to_cube(_ossie(_TWO_DATASETS, _REL), base_cube="nope") + + +def test_synonyms_reach_cube_as_prose_and_are_parked_structurally(): + """Cube has no synonyms field; its docs express them as ai_context prose. The + structured list is parked so the Ossie round trip stays exact.""" + ds = ( + " - name: orders\n" + " source: t\n" + " ai_context:\n" + " instructions: Order facts.\n" + " synonyms:\n" + " - purchases\n" + " - sales\n" + " fields:\n" + " - name: id\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: id\n" + " datatype: Integer\n" + ) + files, _ = convert_ossie_to_cube(_ossie(ds)) + meta = _cubes(files)["orders"]["meta"] + assert meta["ai_context"] == "Order facts.\nAlso known as: purchases, sales." + assert meta["ossie"]["ai_context"]["synonyms"] == ["purchases", "sales"] diff --git a/converters/cube/tests/test_roundtrip.py b/converters/cube/tests/test_roundtrip.py new file mode 100644 index 00000000..760fc22a --- /dev/null +++ b/converters/cube/tests/test_roundtrip.py @@ -0,0 +1,194 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Fixture-based round-trip tests. + +- Cube -> Ossie -> Cube must be lossless (the stash carries everything). +- Ossie -> Cube -> Ossie must be identical up to the documented normalizations. +- Every Ossie document the importer emits must validate against the core-spec + JSON schema (skipped when jsonschema is not installed). +""" + +import json + +import pytest +from _util import REPO_ROOT, load_fixture_dir, parse, parse_files + +from ossie_cube import convert_cube_to_ossie, convert_ossie_to_cube + +FIXTURES = ["fixtureA_cube", "tpcds_cube"] + + +@pytest.mark.parametrize("fixture", FIXTURES) +def test_cube_roundtrip_is_lossless(fixture): + """Cube -> Ossie -> Cube reproduces the original model, structurally. + + Compared parsed rather than byte-for-byte: YAML comments (including the + licence headers on the fixtures) are not part of the data model, and key order + within a mapping is not semantic. + """ + files = load_fixture_dir(fixture) + ossie, _ = convert_cube_to_ossie(files) + files2, _ = convert_ossie_to_cube(ossie) + assert parse_files(files2) == parse_files(files) + + +@pytest.mark.parametrize("fixture", FIXTURES) +def test_imported_ossie_validates_against_core_spec_schema(fixture): + jsonschema = pytest.importorskip("jsonschema") + with open(REPO_ROOT / "core-spec" / "osi-schema.json") as fh: + schema = json.load(fh) + ossie, _ = convert_cube_to_ossie(load_fixture_dir(fixture)) + jsonschema.validate(parse(ossie), schema) + + +@pytest.mark.parametrize("fixture", FIXTURES) +def test_ossie_roundtrip_is_lossless(fixture): + """Ossie -> Cube -> Ossie reproduces the model too. + + Cube has a `meta` field at every level, so the export direction parks what + Cube has no slot for under `meta.ossie` instead of dropping it -- which makes + this direction lossless as well, unlike converters whose target format has + nowhere to put the leftovers. + """ + ossie, _ = convert_cube_to_ossie(load_fixture_dir(fixture)) + files, _ = convert_ossie_to_cube(ossie) + ossie2, _ = convert_cube_to_ossie(files) + assert parse(ossie2) == parse(ossie) + + +def test_hand_authored_ossie_gets_a_generated_view(): + """A model with no stashed views is not from Cube, so export has to invent the + view -- the model boundary Cube users work with.""" + ossie = _HAND_AUTHORED + files, _ = convert_ossie_to_cube(ossie) + assert set(files) == { + "model/cubes/orders.yml", "model/cubes/customers.yml", + "model/views/ecommerce.yml", + } + view = parse(files["model/views/ecommerce.yml"])["views"][0] + assert view["name"] == "ecommerce" + assert view["description"] == "Orders and customers" + # Rooted at the FK sink, with the joined cube addressed by its join path. + assert view["cubes"] == [ + {"join_path": "orders", "includes": "*"}, + {"join_path": "orders.customers", "includes": "*"}, + ] + + +def test_hand_authored_ossie_survives_the_round_trip(): + files, _ = convert_ossie_to_cube(_HAND_AUTHORED) + ossie2, _ = convert_cube_to_ossie(files) + model = parse(ossie2)["semantic_model"][0] + assert model["name"] == "ecommerce" + assert model["description"] == "Orders and customers" + assert [d["name"] for d in model["datasets"]] == ["orders", "customers"] + assert model["relationships"][0]["from_columns"] == ["customer_id"] + metrics = {m["name"]: m for m in model["metrics"]} + assert metrics["total_revenue"]["expression"]["dialects"][0]["expression"] == ( + "SUM(orders.amount)") + + +def test_ossie_only_constructs_are_parked_not_dropped(): + """`unique_keys` and a foreign vendor's extensions have no Cube field, so they + ride under `meta.ossie` and come back intact.""" + files, _ = convert_ossie_to_cube(_HAND_AUTHORED) + orders = parse(files["model/cubes/orders.yml"])["cubes"][0] + parked = orders["meta"]["ossie"] + assert parked["unique_keys"] == [["order_number"]] + assert parked["custom_extensions"][0]["vendor_name"] == "SNOWFLAKE" + + ossie2, _ = convert_cube_to_ossie(files) + ds = {d["name"]: d for d in parse(ossie2)["semantic_model"][0]["datasets"]} + assert ds["orders"]["unique_keys"] == [["order_number"]] + vendors = {e["vendor_name"] for e in ds["orders"]["custom_extensions"]} + assert "SNOWFLAKE" in vendors + + +_HAND_AUTHORED = """ +version: 0.2.0.dev0 +semantic_model: +- name: ecommerce + description: Orders and customers + ai_context: + instructions: Use for sales analysis. + synonyms: + - sales + - purchases + datasets: + - name: orders + source: sales.public.orders + primary_key: + - id + unique_keys: + - - order_number + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + datatype: Integer + - name: customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: customer_id + datatype: Integer + - name: ordered_at + expression: + dialects: + - dialect: ANSI_SQL + expression: ordered_at + datatype: Date + custom_extensions: + - vendor_name: SNOWFLAKE + data: '{"warehouse": "ANALYTICS_WH"}' + - name: customers + source: sales.public.customers + primary_key: + - id + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + datatype: Integer + - name: email + expression: + dialects: + - dialect: ANSI_SQL + expression: LOWER(email) + datatype: String + relationships: + - name: orders_to_customers + from: orders + to: customers + from_columns: + - customer_id + to_columns: + - id + metrics: + - name: total_revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.amount) + description: Total revenue + datatype: Decimal +""" diff --git a/converters/cube/tests/test_roundtrip_properties.py b/converters/cube/tests/test_roundtrip_properties.py new file mode 100644 index 00000000..0a7e4106 --- /dev/null +++ b/converters/cube/tests/test_roundtrip_properties.py @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Property-based round-trip tests over generated Cube models. + +The generators live in `_roundtrip_helpers` and depend only on a tiny +chance/count/pick/text interface, so the same model space is explored whether +Hypothesis is installed or not. Without it, a seeded sweep runs instead -- the +properties are still checked in CI on a Python where hypothesis fails to build. +""" + +import pytest +from _roundtrip_helpers import RandomRnd, build_cube_model, check_model + +try: + from hypothesis import HealthCheck, given, settings + from hypothesis import strategies as st + + HAVE_HYPOTHESIS = True +except ImportError: # pragma: no cover - exercised only without hypothesis + HAVE_HYPOTHESIS = False + + +SEEDS = list(range(60)) + + +@pytest.mark.parametrize("seed", SEEDS) +def test_seeded_models_roundtrip(seed): + """A deterministic sweep, so a failure names a reproducible seed.""" + check_model(build_cube_model(RandomRnd(seed))) + + +if HAVE_HYPOTHESIS: + class _HypothesisRnd: + """The `Rnd` interface backed by a Hypothesis data strategy.""" + + def __init__(self, data): + self.data = data + + def chance(self, p=0.5): + return self.data.draw(st.booleans()) + + def count(self, lo, hi): + return self.data.draw(st.integers(min_value=lo, max_value=hi)) + + def pick(self, seq): + return self.data.draw(st.sampled_from(list(seq))) + + def text(self): + # Printable, no leading/trailing whitespace and no newlines, so the + # value survives a YAML dump/load cycle verbatim. Round-tripping + # arbitrary Unicode is a PyYAML property, not a converter one. + # + # Jinja delimiters are excluded because they are out of the + # round-trippable subset by design: the converter treats a file + # containing them as templated and preserves it whole, exactly as + # Cube's own CubeSchemaConverter does. That behavior has its own + # targeted test. + return self.data.draw(st.text( + alphabet=st.characters(min_codepoint=32, max_codepoint=126), + min_size=1, max_size=24, + ).map(str.strip).filter( + lambda s: s and not s.startswith("#") + and not any(t in s for t in ("{{", "}}", "{%", "%}")))) + + @settings(max_examples=150, deadline=None, + suppress_health_check=[HealthCheck.too_slow]) + @given(st.data()) + def test_generated_models_roundtrip(data): + check_model(build_cube_model(_HypothesisRnd(data))) diff --git a/converters/cube/uv.lock b/converters/cube/uv.lock index c213eb6b..6b158087 100644 --- a/converters/cube/uv.lock +++ b/converters/cube/uv.lock @@ -13,6 +13,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "hypothesis" }, + { name = "jsonschema" }, { name = "pytest" }, ] @@ -22,9 +23,19 @@ requires-dist = [{ name = "pyyaml", specifier = ">=6.0" }] [package.metadata.requires-dev] dev = [ { name = "hypothesis", specifier = ">=6.0" }, + { name = "jsonschema", specifier = ">=4.0" }, { name = "pytest", specifier = ">=8.0" }, ] +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -108,6 +119,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -206,6 +244,143 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/1f/a2dca5ffdbf1d475ffc4e80e4d5d720ff3a00f691795910116960ee12511/rpds_py-2026.6.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7b689145a1485c335569bd056464f3243a29af7ed3871c7be31ad624ba239bc7", size = 342174, upload-time = "2026-06-30T07:14:54.821Z" }, + { url = "https://files.pythonhosted.org/packages/4d/dc/323d08583c0832911768663d1944f0107fcd4088704858d84b5e06d105a0/rpds_py-2026.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:db08f45aecde626498fb3df07bcf6d2ec040af42e859a4f5040d79c200342911", size = 345513, upload-time = "2026-06-30T07:14:56.515Z" }, + { url = "https://files.pythonhosted.org/packages/0b/2a/e31989834d18d2f26ec1d2774c5b1eb3331df4ea8ada525175294c94b48a/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acc992ab27b15f852c76755eb2ab7dce86585ddadba6fa5946e58556088845b4", size = 373783, upload-time = "2026-06-30T07:14:57.736Z" }, + { url = "https://files.pythonhosted.org/packages/87/fe/e80107ee3639585c9941c17d6a42cd65325022f656c023191fce78c324c8/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f88d653e7b3b779d71ae7454e20dcc9b6bae903f33c269db9f2be41bda3f261", size = 378316, upload-time = "2026-06-30T07:14:59.077Z" }, + { url = "https://files.pythonhosted.org/packages/22/6f/81e3adf81acfb6fa694de2a6e4e7d8863121e3e0799e0a7725e6cf5679c4/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e52655eaf81e32593abedaa4bfe33170c8cfedf3365ed9be6e11e07f148f0278", size = 499423, upload-time = "2026-06-30T07:15:00.488Z" }, + { url = "https://files.pythonhosted.org/packages/2d/9a/41263969df0ce3d9af2a96d5005a288200af1989aed3354bfceb5fc0b21f/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dfcc8b909769d19db55c7cc9541eb64b9b774b1057ffffb4f1048070475bb9f9", size = 386077, upload-time = "2026-06-30T07:15:01.911Z" }, + { url = "https://files.pythonhosted.org/packages/5e/19/7e98f468bd50346faff5b10e5297374b443bfdddacc8e9fbc65984539597/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c1255b302953c86a486b81d330d5ee1d5bd937691ce271b6be0ef0e299eaab7", size = 371315, upload-time = "2026-06-30T07:15:03.317Z" }, + { url = "https://files.pythonhosted.org/packages/99/3c/2b973b4d371906a134b03decfea7f5d9835a2c6d263454392e15b64b5b18/rpds_py-2026.6.3-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:8d2294a31386bfa251d8c8a39472beee17db67d4f1a6eabea665d35c9a4461c3", size = 383502, upload-time = "2026-06-30T07:15:04.627Z" }, + { url = "https://files.pythonhosted.org/packages/98/2a/12e2799500af0a307bca76b63361c51f9fe479223561489c29eea1f2ee41/rpds_py-2026.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f8f23ead891a3b762f35ab3b04623da7056545b48aa60d59957e6789914545da", size = 402673, upload-time = "2026-06-30T07:15:05.856Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e3/21e5872d165fe08be4f229e3d5ee9d90019c0bf0e5538de60dbd54009450/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:421aba32367055614287a4292b6a17f1939c9452299f7a0209c117e990b646d4", size = 549964, upload-time = "2026-06-30T07:15:07.159Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d0/5ee0fe36844297de8123bee27bc12078c1a7416ad9f1b8a8ca18d6b0c0ac/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1e5822dfc2f0d4ab7e745eaa6d85945069329beeccef965af3f3bb26058fcab6", size = 615446, upload-time = "2026-06-30T07:15:08.531Z" }, + { url = "https://files.pythonhosted.org/packages/b1/80/1ea5873cb683f2fbe5f21b23ea1f6d179ead19f3c5b249b7eb5dca568ef2/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:83e35b57523816c8613fd0776b40cd8bb9f596b37ddd2692eb4a6bb5ab2f8c93", size = 576975, upload-time = "2026-06-30T07:15:09.97Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e1/90ef639217a5ddb15b7f4f61b1c33911fd044ad03c311bafdd2bcab85582/rpds_py-2026.6.3-cp311-cp311-win32.whl", hash = "sha256:de3eceba0b683bcbb1ab93da016d0270df1f9ae7be716b40214c5dafac6ea45a", size = 204453, upload-time = "2026-06-30T07:15:11.324Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b7/b7a1695d7af36f521fb11e80d6d3adbd744f73b921859bd3c2a2c0dc706f/rpds_py-2026.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:2c54a076ca4d370980ab57bc0e31df57bbe8d41340436a90ef8b1219a3cbb127", size = 223219, upload-time = "2026-06-30T07:15:12.476Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a2/145afacf796e4506062825941176ad9445c2dcf2b3b6a1f13d3030a15e19/rpds_py-2026.6.3-cp311-cp311-win_arm64.whl", hash = "sha256:168c733a7112e071bb7a66460e667edfcff06c017a3c523f7a8a8e08d0140804", size = 219137, upload-time = "2026-06-30T07:15:13.631Z" }, + { url = "https://files.pythonhosted.org/packages/5c/be/2e8974163072e7bab7df1a5acd54c4498e75e35d6d18b864d3a9d5dadc92/rpds_py-2026.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0", size = 343691, upload-time = "2026-06-30T07:15:14.96Z" }, + { url = "https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf", size = 338542, upload-time = "2026-06-30T07:15:16.267Z" }, + { url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" }, + { url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" }, + { url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" }, + { url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" }, + { url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" }, + { url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" }, + { url = "https://files.pythonhosted.org/packages/c9/50/22f73127a41f1ce4f87fe39aadfb9a126345801c274aa93ae88456249327/rpds_py-2026.6.3-cp312-cp312-win32.whl", hash = "sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed", size = 202185, upload-time = "2026-06-30T07:15:32.027Z" }, + { url = "https://files.pythonhosted.org/packages/04/3a/f0ee4d4dde9d3b69dedf1b5f74e7a40017046d55052d173e418c6a94f960/rpds_py-2026.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f", size = 220394, upload-time = "2026-06-30T07:15:33.359Z" }, + { url = "https://files.pythonhosted.org/packages/f3/83/3382fe37f809b59f02aac04dbc4e765b480b46ee0227ed516e3bdc4d3dfc/rpds_py-2026.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96", size = 215753, upload-time = "2026-06-30T07:15:34.778Z" }, + { url = "https://files.pythonhosted.org/packages/a4/9e/b818ee580026ec578138e961027a68820c40afeb1ec8f6819b54fb99e196/rpds_py-2026.6.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223", size = 343012, upload-time = "2026-06-30T07:15:36.005Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6b/686d9dc4359a8f163cfbbf89ee0b4e586431de22fe8248edb63a8cf50d49/rpds_py-2026.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f", size = 338203, upload-time = "2026-06-30T07:15:37.462Z" }, + { url = "https://files.pythonhosted.org/packages/9e/9b/069aa329940f8207615e091f5eedbbd40e1e15eac68a0790fd05ccdf796c/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f", size = 367984, upload-time = "2026-06-30T07:15:39.008Z" }, + { url = "https://files.pythonhosted.org/packages/14/db/34c203e4becff3703e4d3bc121842c00b8689197f398161203a880052f4e/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7", size = 374815, upload-time = "2026-06-30T07:15:40.253Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7d/8071067d2cc453d916ad836e828c943f575e8a44612537759002a1e07381/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6", size = 490545, upload-time = "2026-06-30T07:15:41.729Z" }, + { url = "https://files.pythonhosted.org/packages/a3/42/da06c5aa8f0484ff07f270787434204d9f4535e2f8c3b51ed402267e63c3/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af", size = 382828, upload-time = "2026-06-30T07:15:43.327Z" }, + { url = "https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf", size = 365678, upload-time = "2026-06-30T07:15:44.992Z" }, + { url = "https://files.pythonhosted.org/packages/69/9d/1d8922e1990b2a6eb532b6ff53d3e73d2b3bbffc84116c75826bee73dfc6/rpds_py-2026.6.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885", size = 377811, upload-time = "2026-06-30T07:15:46.523Z" }, + { url = "https://files.pythonhosted.org/packages/b1/3d/198dceafb4fb034a6a47347e1b0735d34e0bd4a50be4e898d408ee66cb14/rpds_py-2026.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4", size = 395382, upload-time = "2026-06-30T07:15:47.955Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/13968e49655d40b6b19d8b9140296bbc6f1d86b3f0f6c346cf9f1adddf4b/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7", size = 543832, upload-time = "2026-06-30T07:15:49.33Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ab/289bcb1b90bd3e40a2900c561fa0e2087345ecbb094f0b870f2345142b7c/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d", size = 611011, upload-time = "2026-06-30T07:15:50.847Z" }, + { url = "https://files.pythonhosted.org/packages/1e/16/5043105e679436ccfbc8e5e0dd2d663ed18a8b8113515fd06a5e5d77c83e/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97", size = 572431, upload-time = "2026-06-30T07:15:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/adab103321c0a6565d5ae1c2998349bc3ee175b82ccc5ae8fc04cc413075/rpds_py-2026.6.3-cp313-cp313-win32.whl", hash = "sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0", size = 201710, upload-time = "2026-06-30T07:15:53.894Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ed/a03b09668e74e5dabbf2e211f6468e1820c0552f7b0500082da31841bf7b/rpds_py-2026.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80", size = 219454, upload-time = "2026-06-30T07:15:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/27/17/b8642c12930b71bc2b25831f6708ccf0f75abcd11883932ec9ce54ba3a78/rpds_py-2026.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb", size = 215063, upload-time = "2026-06-30T07:15:56.573Z" }, + { url = "https://files.pythonhosted.org/packages/b6/36/7fbe9dcdaf857fb3f63c2a2284b62492d95f5e8334e947e5fb6e7f68c9be/rpds_py-2026.6.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e", size = 344510, upload-time = "2026-06-30T07:15:57.921Z" }, + { url = "https://files.pythonhosted.org/packages/ba/54/f785cc3d3f60839ca57a5af4927a9f347b07b2799c373fc20f7949f87c7e/rpds_py-2026.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd", size = 339495, upload-time = "2026-06-30T07:15:59.238Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, + { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, + { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, + { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, + { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, + { url = "https://files.pythonhosted.org/packages/87/03/2a69ab618a789cf6cf85c86bb844c62d090e700ab1a2aa676b3741b6c516/rpds_py-2026.6.3-cp314-cp314-win32.whl", hash = "sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8", size = 202504, upload-time = "2026-06-30T07:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/85/62/a3892ba945f4e24c78f352e5de3c7620d8479f73f211406a97263d13c7d2/rpds_py-2026.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5", size = 220380, upload-time = "2026-06-30T07:16:18.108Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e7/c2bd44dc831931815ad11ebb5f430b5a0a4d3caa9de837107876c30c3432/rpds_py-2026.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703", size = 215976, upload-time = "2026-06-30T07:16:19.654Z" }, + { url = "https://files.pythonhosted.org/packages/79/9c/fff7b74bce9a091ec9a012a03f9ff5f69364eaf9451060dfc4486da2ffdd/rpds_py-2026.6.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90", size = 346840, upload-time = "2026-06-30T07:16:21.268Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/77bcb1168b33704908295533d27f10eb811e9e3e193e8993dc99572211d3/rpds_py-2026.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4", size = 340282, upload-time = "2026-06-30T07:16:22.875Z" }, + { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, + { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, + { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, + { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f6/21101359743cd136ada781e8210a85769578422ba460672eea0e29739200/rpds_py-2026.6.3-cp314-cp314t-win32.whl", hash = "sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e", size = 201068, upload-time = "2026-06-30T07:16:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/a6/b2/9574d4d44f7760c2aa32d92a0a4f41698e33f5b204a0bf5c9758f52c79d5/rpds_py-2026.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2", size = 220600, upload-time = "2026-06-30T07:16:43.091Z" }, + { url = "https://files.pythonhosted.org/packages/08/ae/f23a2697e6ee6340a578b0f136be6483657bef0c6f9497b752bb5c0964bb/rpds_py-2026.6.3-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13", size = 344726, upload-time = "2026-06-30T07:16:44.5Z" }, + { url = "https://files.pythonhosted.org/packages/c3/63/e7b3a1a5358dd32c930a1062d8e15b67fd6e8922e81df9e91706d66ee5c8/rpds_py-2026.6.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05", size = 339587, upload-time = "2026-06-30T07:16:46.255Z" }, + { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, + { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, + { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, + { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, + { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, + { url = "https://files.pythonhosted.org/packages/29/71/4d8fcf700931815594bce892255bbd973b94efaf0fc1932b0590df18d886/rpds_py-2026.6.3-cp315-cp315-win32.whl", hash = "sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369", size = 202864, upload-time = "2026-06-30T07:17:05.746Z" }, + { url = "https://files.pythonhosted.org/packages/eb/62/b577562de0edbb55b2be85ce5fd09c33e386b9b13eee09833af4240fd5c4/rpds_py-2026.6.3-cp315-cp315-win_amd64.whl", hash = "sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146", size = 220430, upload-time = "2026-06-30T07:17:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/c8/95/d6d0b2509825141eef60669a5739eec88dbc6a48053d6c92993a5704defe/rpds_py-2026.6.3-cp315-cp315-win_arm64.whl", hash = "sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e", size = 215877, upload-time = "2026-06-30T07:17:09.008Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bf/f3ea278f0afd615c1d0f19cb69043a41526e2bb600c2b536eb192218eb27/rpds_py-2026.6.3-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b", size = 346933, upload-time = "2026-06-30T07:17:10.762Z" }, + { url = "https://files.pythonhosted.org/packages/9d/29/9907bdf1c5346763cf10b7f6852aad86652168c259def904cbe0082c5864/rpds_py-2026.6.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690", size = 340274, upload-time = "2026-06-30T07:17:12.266Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, + { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, + { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, + { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/ba/76/65002b08596c389105720a8c0d22298b8dc25a4baf89b2ce431343c8b1de/rpds_py-2026.6.3-cp315-cp315t-win32.whl", hash = "sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577", size = 201204, upload-time = "2026-06-30T07:17:30.193Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, + { url = "https://files.pythonhosted.org/packages/b4/9c/f0d19ac587fd0e4ab6b72cda355e9c5a6166b01ef7e064e437aef8eb9fef/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:4cf2d36a2357e4d07bb5a4f98801265327b48256867816cfd2ceb001e9754a8f", size = 349791, upload-time = "2026-06-30T07:17:33.315Z" }, + { url = "https://files.pythonhosted.org/packages/38/c7/1d49d204c9fd2ee6c537601dc4c1ba921e03363ca576bfab94a00254ac9a/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:30c6dc199b24a5e3e81d50da0f00858c5bbdb2617a750395687f4339c5818171", size = 352842, upload-time = "2026-06-30T07:17:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/ac/e5/c0b5dc93cd0d4c06ce1f438907649514e2ea077bcd911e3154a51e96c38e/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9891e594296ab9dada6551c8e7b387b2721f27a67eecd528412e8906247a7b90", size = 382094, upload-time = "2026-06-30T07:17:36.514Z" }, + { url = "https://files.pythonhosted.org/packages/0d/54/ec0e907b4ca8d541112db352409bd15f871c9b243e0c92c9b5a46ae96f01/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5c2dc92304aa48a4a60443b548bb12f12e119d4b72f314015e67b9e1be97fca", size = 388662, upload-time = "2026-06-30T07:17:38.235Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f4/921c22a4fd0f1c1ac13a3996ffbf0aa67951e2c8ad0d1d9574938a2932e8/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:127e08c0642d880cf32ca47ec2a4a77b901f7e2dd1ad9762adb13955d72ffcc9", size = 504896, upload-time = "2026-06-30T07:17:39.689Z" }, + { url = "https://files.pythonhosted.org/packages/0b/1b/a114b972cefa1ab1cdb3c7bb177cd3844a12826c507c722d3a73516dbbaf/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8bb68f03f395eb793220b45c097bd4d8c32944393da0fad8b999efac0868fc8c", size = 391545, upload-time = "2026-06-30T07:17:41.336Z" }, + { url = "https://files.pythonhosted.org/packages/4e/98/af9b3db77d47fcbe6c8c1f36e2c2147ec70292819e99c325f871584a1c11/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3450b693fde92133e9f51060568a4c31fcca76d5e53bbd611e689ca446517e9", size = 380059, upload-time = "2026-06-30T07:17:42.857Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ba/0efd8668b97c1d26a61566386c636a7a7a09829e474fdf807caa15a2c844/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:5e8d07bddee435a2ff6f1920e18feff28d0bc4533e42f4bf6927fbd073312c41", size = 393235, upload-time = "2026-06-30T07:17:44.637Z" }, + { url = "https://files.pythonhosted.org/packages/62/90/8c139ee9690f73b0829f32647de6f40d826f8f443af6fa72644f96351aac/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3a83ae6c67b7676b9878378547ca8e93ed77a580037bcbcd1d32f739e1e6089c", size = 413008, upload-time = "2026-06-30T07:17:46.225Z" }, + { url = "https://files.pythonhosted.org/packages/9c/97/0043896fdd7828ce09a1d9a8b06433714d0960fc4ff3fc4aa72b666b764e/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2bfd04c19ddbd6640de0b51894d764bd2758854d5b75bd102d2ef10cb9c293a9", size = 558118, upload-time = "2026-06-30T07:17:47.759Z" }, + { url = "https://files.pythonhosted.org/packages/f6/40/02355f0e134f783a8f9814c4680a1bd311d37671577a5964ea838573ff37/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:ca6546b66be9dc4738b1b043d5ebd5488c66c578c5ff0fd0e8065313fe3afb76", size = 623138, upload-time = "2026-06-30T07:17:49.355Z" }, + { url = "https://files.pythonhosted.org/packages/10/85/48f0abdcef5cce4e034c7a5b0ceeceba0b01bf0d942824f4bb720afe2dec/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826", size = 586486, upload-time = "2026-06-30T07:17:51.141Z" }, +] + [[package]] name = "sortedcontainers" version = "2.4.0" @@ -214,3 +389,12 @@ sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f233 wheels = [ { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, ] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] From aa011ed8352b144d040f5aedc6a40305fc25513f Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 00:40:39 +0500 Subject: [PATCH 03/46] Close the test gaps coverage exposed; drop two dead code paths Coverage was 91% with several load-bearing branches never executed. Adds test_edge_cases.py (45 tests) for the paths the fixtures and property tests cannot reach, since those generate inside the round-trippable subset by design: - composite primary keys -- the COUNT(DISTINCT CONCAT(CAST(...))) form is central to the fan-out mapping and was never run in either direction; - `count` with `sql` (COUNT(x)), including that it is fan-out-unsafe where a bare count is not; - the export side of the one_to_many flip -- import flipping it was tested, export flipping it back was not; - off-layout file grouping: several cubes in one oddly-named file have to return to that same file, not be split into the canonical layout; - the JavaScript-style mapping form of dimensions/measures/joins; - legacy belongsTo/hasMany/hasOne spellings; - unconvertible joins restored at their original positions; - geo dimension extras, measure `title`, `ai_context.examples`, a bare string `ai_context`, multiple views, and the malformed-input errors. Two dead paths removed, both found by the same pass: - member-level Jinja handling was unreachable. JINJA_RE is checked per file (as Cube's own CubeSchemaConverter does), so a templated member's file never reaches the per-member branch. Renamed the issue type TEMPLATED_MEMBER_DROPPED -> TEMPLATED_FILE_SKIPPED to match what it actually reports, and dropped the matching `extra_dimensions` restore. - `is_cube_name` was never called. Coverage 91% -> 96%; the remaining 40 lines are defensive ConversionError branches on malformed input. 201 tests. Co-Authored-By: Claude Opus 5 --- converters/cube/README.md | 2 +- converters/cube/src/ossie_cube/_common.py | 5 - .../cube/src/ossie_cube/converter_issues.py | 8 +- converters/cube/src/ossie_cube/cube_to_osi.py | 20 +- converters/cube/src/ossie_cube/osi_to_cube.py | 4 - converters/cube/tests/test_cube_to_osi.py | 4 +- converters/cube/tests/test_edge_cases.py | 712 ++++++++++++++++++ 7 files changed, 722 insertions(+), 33 deletions(-) create mode 100644 converters/cube/tests/test_edge_cases.py diff --git a/converters/cube/README.md b/converters/cube/README.md index 7a5c4a4a..83609f2c 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -185,7 +185,7 @@ element it concerns, and a detail string. | `MULTI_STAGE_MEASURE_DROPPED` | A `multi_stage` measure (`group_by`/`reduce_by`/`time_shift`/`rank`) renders as a window function over another grain | | `CUBE_LEVEL_AI_CONTEXT_INERT` | Cube's agent ignores cube-level `meta.ai_context` | | `GEO_DIMENSION_SPLIT` | A `type: geo` dimension became two Ossie fields | -| `TEMPLATED_MEMBER_DROPPED` | Jinja templating, or a `.js`/`.ts` model file | +| `TEMPLATED_FILE_SKIPPED` | Jinja templating anywhere in a file, or a `.js`/`.ts` model file. Detected per file, as Cube's own tooling does, so the file is preserved whole rather than half-converted | | `NO_USABLE_DIALECT` | Export: no `ANSI_SQL` or preferred-dialect expression | | `PARKED_IN_META` | An element preserved in the stash with no native mapping | diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index 1468ab4c..806d0a4d 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -173,11 +173,6 @@ def is_simple_identifier(expr): return isinstance(expr, str) and bool(_IDENTIFIER_RE.match(expr.strip())) -def is_cube_name(name): - """True if `name` is already a valid Cube identifier.""" - return isinstance(name, str) and bool(_CUBE_NAME_RE.match(name)) - - def sanitize_name(name, what, taken): """Coerce an Ossie name into a valid Cube identifier. diff --git a/converters/cube/src/ossie_cube/converter_issues.py b/converters/cube/src/ossie_cube/converter_issues.py index 64dfe4b1..de29c586 100644 --- a/converters/cube/src/ossie_cube/converter_issues.py +++ b/converters/cube/src/ossie_cube/converter_issues.py @@ -53,9 +53,11 @@ class IssueType(Enum): # because an Ossie field holds a single expression. GEO_DIMENSION_SPLIT = "GEO_DIMENSION_SPLIT" - # A dimension or measure whose `sql` uses Jinja templating, or a cube using - # `extends`: no static form, so it is preserved in the stash only. - TEMPLATED_MEMBER_DROPPED = "TEMPLATED_MEMBER_DROPPED" + # A file with no static form -- Jinja templating anywhere in it, or a `.js` / + # `.ts` data model needing Cube's transpiler. Detected per file (as Cube's own + # CubeSchemaConverter does), so the whole file is preserved verbatim in the + # stash rather than half-converted. + TEMPLATED_FILE_SKIPPED = "TEMPLATED_FILE_SKIPPED" # An Ossie field or metric with no usable expression dialect (export). NO_USABLE_DIALECT = "NO_USABLE_DIALECT" diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 35800a2f..bbcd76c6 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -201,12 +201,12 @@ def _collect(files, issues): # A `.js`/`.ts` data model needs Cube's own transpiler and a `.py` one # is Jinja-driven. Preserved verbatim so the round trip keeps the file, # but no cube inside it is converted. - issues.add(IssueType.TEMPLATED_MEMBER_DROPPED, fname, + issues.add(IssueType.TEMPLATED_FILE_SKIPPED, fname, "not a YAML data model; preserved in custom_extensions only") extra_files[fname] = text continue if JINJA_RE.search(text): - issues.add(IssueType.TEMPLATED_MEMBER_DROPPED, fname, + issues.add(IssueType.TEMPLATED_FILE_SKIPPED, fname, "uses Jinja templating, which has no static form; " "preserved in custom_extensions only") extra_files[fname] = text @@ -371,15 +371,8 @@ def _convert_cube(cname, cube, extra_joins, issues): fields = [] primary_key = [] - templated = {} for dim in _as_named_list(cube.get("dimensions"), f"{scope} dimensions"): dname = require_str(dim, "name", f"{scope}: dimension") - if JINJA_RE.search(str(dim.get("sql", ""))): - issues.add(IssueType.TEMPLATED_MEMBER_DROPPED, f"{cname}.{dname}", - "dimension sql uses Jinja templating; preserved in " - "custom_extensions only") - templated[dname] = dim - continue if dim.get("primary_key"): primary_key.append(dname) fields.extend(_convert_dimension(cname, dname, dim, issues)) @@ -387,8 +380,6 @@ def _convert_cube(cname, cube, extra_joins, issues): ds["fields"] = fields if primary_key: ds["primary_key"] = primary_key - if templated: - stash["extra_dimensions"] = templated if extra_joins: stash["extra_joins"] = extra_joins @@ -699,13 +690,6 @@ def expression(self, cname, mname, stack=()): f"multi_stage measure (type '{mtype}'); preserved in " f"custom_extensions only") return None - if JINJA_RE.search(str(measure.get("sql", ""))): - self._issues.add( - IssueType.TEMPLATED_MEMBER_DROPPED, scope, - "measure sql uses Jinja templating; preserved in " - "custom_extensions only") - return None - sql = measure.get("sql") filter_exprs = [ self._translate(f["sql"], cname, stack + (key,)) diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 0dc4d4a5..5d163d54 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -284,10 +284,6 @@ def _build_cube(ds, cname, members, joins, measures, dialect, issues): if dim["name"] in pk_names: dim["primary_key"] = True - dimensions.extend( - _ordered(dict(d, name=n), _DIM_KEY_ORDER) - for n, d in (stash.get("extra_dimensions") or {}).items() - ) if dimensions: cube["dimensions"] = [_ordered(d, _DIM_KEY_ORDER) for d in dimensions] diff --git a/converters/cube/tests/test_cube_to_osi.py b/converters/cube/tests/test_cube_to_osi.py index 44fdb14a..faedcbf7 100644 --- a/converters/cube/tests/test_cube_to_osi.py +++ b/converters/cube/tests/test_cube_to_osi.py @@ -431,7 +431,7 @@ def test_jinja_templated_file_is_preserved_not_parsed(): model = model_of(out) assert by_name(model["datasets"]).keys() == {"orders"} assert "model/cubes/dyn.yml" in stash_of(model)["extra_files"] - assert issues.of_type(IssueType.TEMPLATED_MEMBER_DROPPED) + assert issues.of_type(IssueType.TEMPLATED_FILE_SKIPPED) def test_join_into_a_skipped_file_explains_itself(): @@ -463,7 +463,7 @@ def test_javascript_model_is_preserved_not_parsed(): } out, issues = convert_cube_to_ossie(files) assert "model/cubes/orders.js" in stash_of(model_of(out))["extra_files"] - assert issues.of_type(IssueType.TEMPLATED_MEMBER_DROPPED) + assert issues.of_type(IssueType.TEMPLATED_FILE_SKIPPED) def test_extends_is_refused_rather_than_half_resolved(): diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py new file mode 100644 index 00000000..84874100 --- /dev/null +++ b/converters/cube/tests/test_edge_cases.py @@ -0,0 +1,712 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Coverage-driven tests for paths the fixtures and property tests do not reach. + +The fixture and Hypothesis suites cover the common shapes well, but they generate +inside the round-trippable subset and so never exercise several load-bearing +branches: composite primary keys (central to the fan-out mapping), `count` over an +expression, the export side of the one_to_many flip, off-layout file grouping, the +JavaScript-style mapping form of a collection, and Jinja detection. Each of those +is pinned here, along with the error paths for malformed input. +""" + +import pytest +from _util import by_name, expr_of, model_of, parse, parse_files, stash_of + +from ossie_cube import ( + ConversionError, + IssueType, + convert_cube_to_ossie, + convert_ossie_to_cube, +) + + +def _files(**named): + return {f"model/cubes/{n}.yml": t for n, t in named.items()} + + +def _roundtrip(files): + ossie, issues = convert_cube_to_ossie(files, strict_fanout=False) + back, _ = convert_ossie_to_cube(ossie) + return ossie, back, issues + + +# --- composite primary keys ----------------------------------------------------- + +_COMPOSITE = _files(order_lines=( + "cubes:\n" + " - name: order_lines\n" + " sql_table: public.order_lines\n" + " dimensions:\n" + " - name: order_id\n" + " sql: order_id\n" + " type: number\n" + " primary_key: true\n" + " - name: line_no\n" + " sql: line_no\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: count\n" + " type: count\n" +)) + + +def test_composite_primary_key_becomes_a_concatenated_distinct_count(): + """Cube concatenates a composite key with CAST + CONCAT in `primaryKeyCount`; + the Ossie expression mirrors that so the count stays correct under fan-out and + stays portable (both functions are REQUIRED in the expression language).""" + ossie, _ = convert_cube_to_ossie(_COMPOSITE) + model = model_of(ossie) + assert by_name(model["datasets"])["order_lines"]["primary_key"] == [ + "order_id", "line_no"] + assert expr_of(model["metrics"][0]) == ( + "COUNT(DISTINCT CONCAT(CAST(order_lines.order_id AS VARCHAR), " + "CAST(order_lines.line_no AS VARCHAR)))") + + +def test_composite_key_count_converts_back_to_a_bare_count(): + _, back, _ = _roundtrip(_COMPOSITE) + cube = parse(back["model/cubes/order_lines.yml"])["cubes"][0] + assert cube["measures"] == [{"name": "count", "type": "count"}] + assert [d["name"] for d in cube["dimensions"] if d.get("primary_key")] == [ + "order_id", "line_no"] + + +def test_composite_key_roundtrips(): + _, back, _ = _roundtrip(_COMPOSITE) + assert parse_files(back) == parse_files(_COMPOSITE) + + +# --- count over an expression --------------------------------------------------- + +_COUNT_SQL = _files(orders=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: statuses\n" + " sql: \"{CUBE}.status\"\n" + " type: count\n" +)) + + +def test_count_over_an_expression_keeps_its_operand(): + """`type: count` with `sql` is COUNT(x), not COUNT(*) -- Cube only routes + through the primary key when no sql is given.""" + ossie, _ = convert_cube_to_ossie(_COUNT_SQL) + assert expr_of(model_of(ossie)["metrics"][0]) == "COUNT(orders.status)" + + +def test_count_over_an_expression_roundtrips(): + _, back, _ = _roundtrip(_COUNT_SQL) + assert parse_files(back) == parse_files(_COUNT_SQL) + + +def test_count_over_an_expression_is_fanout_unsafe(): + """Unlike a bare count, COUNT(x) over a fanned-out dataset over-counts.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {users}.id\"\n" + " relationship: many_to_one\n" + " dimensions:\n" + " - name: user_id\n" + " sql: user_id\n" + " type: number\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: emails\n" + " sql: \"{CUBE}.email\"\n" + " type: count\n" + )) + with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): + convert_cube_to_ossie(files) + _, issues = convert_cube_to_ossie(files, strict_fanout=False) + assert issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) + + +# --- join orientation, both ways ------------------------------------------------ + +_ONE_TO_MANY = _files(m=( + "cubes:\n" + " - name: users\n" + " sql_table: public.users\n" + " joins:\n" + " - name: orders\n" + " sql: \"{CUBE}.id = {orders}.user_id\"\n" + " relationship: one_to_many\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: user_id\n" + " sql: user_id\n" + " type: number\n" +)) + + +def test_one_to_many_is_flipped_back_onto_its_original_cube(): + """Ossie's `from` is always the many side, so import flips a one_to_many. Export + has to flip it back -- onto `users`, not `orders`.""" + _, back, _ = _roundtrip(_ONE_TO_MANY) + cubes = by_name(parse(back["model/cubes/m.yml"])["cubes"]) + assert cubes["users"]["joins"] == [{ + "name": "orders", "sql": "{CUBE}.id = {orders}.user_id", + "relationship": "one_to_many"}] + assert "joins" not in cubes["orders"] + + +def test_one_to_one_keeps_its_declared_orientation(): + files = _files(m=_ONE_TO_MANY["model/cubes/m.yml"].replace( + "one_to_many", "one_to_one")) + ossie, issues = convert_cube_to_ossie(files) + rel = model_of(ossie)["relationships"][0] + assert (rel["from"], rel["to"]) == ("users", "orders") + assert any("one_to_one" in i.detail for i in issues.of_type( + IssueType.PARKED_IN_META)) + _, back, _ = _roundtrip(files) + assert parse_files(back) == parse_files(files) + + +@pytest.mark.parametrize("alias,emitted", [ + ("belongsTo", "belongs_to"), + ("belongs_to", "belongs_to"), + ("hasMany", "has_many"), + ("hasOne", "has_one"), +]) +def test_legacy_relationship_spellings_are_accepted_and_kept_semantically( + alias, emitted): + """Cube still accepts belongsTo/hasMany/hasOne. The *kind* of relationship is + preserved rather than modernized to many_to_one, but the spelling is normalized + to snake_case along with every other key -- the documented normalization.""" + files = _files(m=_ONE_TO_MANY["model/cubes/m.yml"].replace( + "one_to_many", alias)) + _, back, _ = _roundtrip(files) + joins = [c.get("joins") for c in parse(back["model/cubes/m.yml"])["cubes"] + if c.get("joins")] + assert joins[0][0]["relationship"] == emitted + + +def test_two_joins_between_one_pair_get_distinct_relationship_names(): + """Ossie relationship names are unique per model, so a second join between the + same two cubes is suffixed rather than colliding.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.buyer_id = {users}.id\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql_table: public.users\n" + " joins:\n" + " - name: orders\n" + " sql: \"{CUBE}.id = {orders}.seller_id\"\n" + " relationship: one_to_many\n" + )) + ossie, _ = convert_cube_to_ossie(files) + names = [r["name"] for r in model_of(ossie)["relationships"]] + assert names == ["orders_to_users", "orders_to_users_2"] + + +def test_unconvertible_join_is_restored_at_its_original_position(): + """A non-equi join has no Ossie form, so it rides in the stash -- and export has + to put it back among the converted joins, in order.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: rates\n" + " sql: \"{CUBE}.day >= {rates}.valid_from\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {users}.id\"\n" + " relationship: many_to_one\n" + " - name: rates\n" + " sql_table: public.rates\n" + " - name: users\n" + " sql_table: public.users\n" + )) + _, back, issues = _roundtrip(files) + assert parse_files(back) == parse_files(files) + orders = by_name(parse(back["model/cubes/m.yml"])["cubes"])["orders"] + assert [j["name"] for j in orders["joins"]] == ["rates", "users"] + assert issues.of_type(IssueType.PARKED_IN_META) + + +def test_join_clause_written_target_side_first_still_decomposes(): + """Either side of the equality may name either cube.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{users.id} = {CUBE}.user_id\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + )) + ossie, back, _ = _roundtrip(files) + rel = model_of(ossie)["relationships"][0] + assert (rel["from_columns"], rel["to_columns"]) == (["user_id"], ["id"]) + assert parse_files(back) == parse_files(files) + + +def test_join_clause_not_spanning_both_cubes_is_preserved(): + """A clause has to relate the two joined cubes. One comparing a cube to itself + (or reaching a third cube) is a valid Cube join with no Ossie relationship form, + so it is preserved verbatim instead of guessed at.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.a = {CUBE}.b\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql_table: public.users\n" + )) + ossie, back, issues = _roundtrip(files) + assert "relationships" not in model_of(ossie) + assert any("references cubes other than" in i.detail for i in issues) + assert parse_files(back) == parse_files(files) + + +def test_join_clause_reaching_an_unrelated_cube_is_preserved(): + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {regions}.id\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql_table: public.users\n" + " - name: regions\n" + " sql_table: public.regions\n" + )) + ossie, back, issues = _roundtrip(files) + assert "relationships" not in model_of(ossie) + assert any("not between two member references" in i.detail for i in issues) + assert parse_files(back) == parse_files(files) + + +def test_join_clause_that_is_not_a_single_equality_is_preserved(): + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.a = {users}.b = 1\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql_table: public.users\n" + )) + ossie, back, issues = _roundtrip(files) + assert "relationships" not in model_of(ossie) + assert any("not a single equality" in i.detail for i in issues) + assert parse_files(back) == parse_files(files) + + +def test_metric_without_a_usable_dialect_is_dropped_with_an_issue(): + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: public.orders\n" + " metrics:\n" + " - name: m\n" + " expression:\n" + " dialects:\n" + " - dialect: MAQL\n" + " expression: SELECT SUM(x)\n" + ) + files, issues = convert_ossie_to_cube(ossie) + assert "measures" not in parse(files["model/cubes/orders.yml"])["cubes"][0] + assert issues.of_type(IssueType.NO_USABLE_DIALECT) + + +# --- file layout ---------------------------------------------------------------- + +def test_off_layout_files_are_restored_with_their_grouping(): + """Import accepts any layout. Several cubes in one oddly-named file have to go + back into that same file, not be split into the canonical per-cube layout.""" + files = { + "schema/warehouse/everything.yaml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " - name: users\n" + " sql_table: public.users\n" + "views:\n" + " - name: main\n" + " description: All of it\n" + ), + } + ossie, back, _ = _roundtrip(files) + assert set(back) == {"schema/warehouse/everything.yaml"} + assert parse_files(back) == parse_files(files) + stash = stash_of(model_of(ossie)) + assert stash["cube_files"]["orders"] == "schema/warehouse/everything.yaml" + assert stash["view_files"]["main"] == "schema/warehouse/everything.yaml" + + +def test_non_model_yaml_is_preserved_verbatim(): + files = { + "model/cubes/orders.yml": ( + "cubes:\n - name: orders\n sql_table: public.orders\n"), + "model/notes.yaml": "just: some data\n", + } + ossie, back, issues = _roundtrip(files) + assert back["model/notes.yaml"] == "just: some data\n" + assert issues.of_type(IssueType.PARKED_IN_META) + + +# --- the JavaScript-style mapping form ------------------------------------------ + +def test_collections_may_be_mappings_keyed_by_name(): + """Cube's post-transpile schema keys dimensions/measures/joins by name, and a + model converted from JavaScript can carry that shape. Both forms are accepted; + export always emits the list form YAML models use.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " id:\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " status:\n" + " sql: status\n" + " type: string\n" + " measures:\n" + " count:\n" + " type: count\n" + )) + ossie, _ = convert_cube_to_ossie(files) + model = model_of(ossie) + fields = by_name(by_name(model["datasets"])["orders"]["fields"]) + assert set(fields) == {"id", "status"} + assert expr_of(model["metrics"][0]) == "COUNT(DISTINCT orders.id)" + + back, _ = convert_ossie_to_cube(ossie) + cube = parse(back["model/cubes/m.yml"])["cubes"][0] + assert isinstance(cube["dimensions"], list) + assert isinstance(cube["measures"], list) + + +def test_a_collection_of_the_wrong_shape_is_rejected(): + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions: not-a-collection\n" + )) + with pytest.raises(ConversionError, match="expected a list or mapping"): + convert_cube_to_ossie(files) + + +# --- Jinja ---------------------------------------------------------------------- + +def test_jinja_anywhere_disqualifies_the_whole_file(): + """Jinja is detected per *file*, not per member -- Cube's own CubeSchemaConverter + uses the same file-level rule. So templating inside a single dimension's `sql` + still costs the whole file, which is preserved verbatim rather than + half-converted. There is deliberately no member-level Jinja path.""" + templated = ( + "cubes:\n" + " - name: templated\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: dyn\n" + " sql: \"{{ 'x' }}\"\n" + " type: string\n" + ) + files = { + "model/cubes/templated.yml": templated, + "model/cubes/plain.yml": ( + "cubes:\n - name: plain\n sql_table: public.plain\n"), + } + ossie, issues = convert_cube_to_ossie(files) + model = model_of(ossie) + assert [d["name"] for d in model["datasets"]] == ["plain"] + assert stash_of(model)["extra_files"]["model/cubes/templated.yml"] == templated + assert issues.of_type(IssueType.TEMPLATED_FILE_SKIPPED) + + # And it comes back byte-for-byte, since it was never parsed. + back, _ = convert_ossie_to_cube(ossie) + assert back["model/cubes/templated.yml"] == templated + + +# --- metadata corners ----------------------------------------------------------- + +def test_measure_title_survives_the_round_trip(): + files = _files(orders=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " measures:\n" + " - name: revenue\n" + " sql: \"{CUBE}.amount\"\n" + " type: sum\n" + " title: Total Revenue\n" + )) + ossie, back, _ = _roundtrip(files) + assert stash_of(model_of(ossie)["metrics"][0])["title"] == "Total Revenue" + assert parse_files(back) == parse_files(files) + + +def test_geo_dimension_extras_survive_the_split_and_merge(): + files = _files(users=( + "cubes:\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: home\n" + " type: geo\n" + " title: Home Location\n" + " description: Where they live\n" + " latitude:\n" + " sql: \"{CUBE}.lat\"\n" + " longitude:\n" + " sql: \"{CUBE}.lon\"\n" + )) + _, back, issues = _roundtrip(files) + assert parse_files(back) == parse_files(files) + assert issues.of_type(IssueType.GEO_DIMENSION_SPLIT) + + +def test_geo_dimension_missing_a_half_is_rejected(): + files = _files(users=( + "cubes:\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: home\n" + " type: geo\n" + " latitude:\n" + " sql: lat\n" + )) + with pytest.raises(ConversionError, match="missing 'longitude.sql'"): + convert_cube_to_ossie(files) + + +def test_ai_context_examples_reach_cube_as_prose_and_park_structurally(): + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " ai_context:\n" + " instructions: Sales model.\n" + " examples:\n" + " - What were sales last month?\n" + " datasets:\n" + " - name: orders\n" + " source: public.orders\n" + ) + files, _ = convert_ossie_to_cube(ossie) + meta = parse(files["model/views/shop.yml"])["views"][0]["meta"] + assert meta["ai_context"] == ( + "Sales model.\nExample questions: What were sales last month?") + assert meta["ossie"]["ai_context"]["examples"] == [ + "What were sales last month?"] + # And the structured form is what comes back, not the flattened prose. + ossie2, _ = convert_cube_to_ossie(files) + assert model_of(ossie2)["ai_context"]["examples"] == [ + "What were sales last month?"] + + +def test_a_plain_string_ai_context_survives_as_a_string(): + """Ossie allows `ai_context` to be a bare string. Import reads Cube's prose back + as {'instructions': ...}, so the original scalar has to be parked to survive.""" + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: public.orders\n" + " ai_context: orders, purchases, sales\n" + ) + files, _ = convert_ossie_to_cube(ossie) + ossie2, _ = convert_cube_to_ossie(files) + ds = by_name(model_of(ossie2)["datasets"])["orders"] + assert ds["ai_context"] == "orders, purchases, sales" + + +# --- multiple views ------------------------------------------------------------- + +_TWO_VIEWS = { + "model/cubes/orders.yml": ( + "cubes:\n - name: orders\n sql_table: public.orders\n"), + "model/views/a.yml": "views:\n - name: a\n description: View A\n", + "model/views/b.yml": "views:\n - name: b\n description: View B\n", +} + + +def test_several_views_need_an_explicit_choice(): + _, issues = convert_cube_to_ossie(_TWO_VIEWS) + assert any("none chosen with --view" in i.detail + for i in issues.of_type(IssueType.PARKED_IN_META)) + + +def test_choosing_a_view_maps_its_metadata_onto_the_model(): + ossie, _ = convert_cube_to_ossie(_TWO_VIEWS, view="b") + model = model_of(ossie) + assert model["name"] == "b" + assert model["description"] == "View B" + # The unchosen view is still preserved whole. + assert set(stash_of(model)["views"]) == {"a", "b"} + + +def test_both_views_are_restored_on_export(): + ossie, _ = convert_cube_to_ossie(_TWO_VIEWS, view="b") + back, _ = convert_ossie_to_cube(ossie) + assert parse_files(back) == parse_files(_TWO_VIEWS) + + +# --- malformed input ------------------------------------------------------------ + +def test_malformed_yaml_is_reported_cleanly(): + with pytest.raises(ConversionError, match="Invalid YAML"): + convert_cube_to_ossie({"model/cubes/m.yml": "cubes: [oops\n"}) + + +def test_empty_input_is_rejected(): + with pytest.raises(ConversionError, match="non-empty mapping"): + convert_cube_to_ossie({}) + + +def test_a_non_string_name_is_rejected_cleanly(): + files = _files(m="cubes:\n - name: 42\n sql_table: t\n") + with pytest.raises(ConversionError, match="must be a string"): + convert_cube_to_ossie(files) + + +def test_ossie_root_must_be_a_mapping(): + with pytest.raises(ConversionError, match="expected a mapping at the root"): + convert_ossie_to_cube("- just\n- a\n- list\n") + + +def test_measure_without_a_type_is_rejected(): + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: t\n" + " measures:\n" + " - name: m\n" + " sql: amount\n" + )) + with pytest.raises(ConversionError, match="missing required 'type'"): + convert_cube_to_ossie(files) + + +def test_unknown_dimension_type_is_rejected(): + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: t\n" + " dimensions:\n" + " - name: d\n" + " sql: d\n" + " type: quaternion\n" + )) + with pytest.raises(ConversionError, match="unknown type 'quaternion'"): + convert_cube_to_ossie(files) + + +def test_unknown_ossie_datatype_is_rejected(): + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: f\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: f\n" + " datatype: Quaternion\n" + ) + with pytest.raises(ConversionError, match="unknown datatype"): + convert_ossie_to_cube(ossie) + + +def test_dataset_without_a_source_is_rejected_on_export(): + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + ) + with pytest.raises(ConversionError, match="missing/empty 'source'"): + convert_ossie_to_cube(ossie) + + +def test_several_semantic_models_convert_the_first_with_an_issue(): + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: first\n" + " datasets:\n" + " - name: orders\n" + " source: t\n" + "- name: second\n" + " datasets:\n" + " - name: users\n" + " source: t\n" + ) + files, issues = convert_ossie_to_cube(ossie) + assert set(files) == {"model/cubes/orders.yml", "model/views/first.yml"} + assert any("converting only the first" in i.detail for i in issues) From 42c561f26ca49fdc7d6d609ba4f86baaafbc26eb Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 00:48:48 +0500 Subject: [PATCH 04/46] Explain a view-only input, accept a single file, and test the CLI Answering "what happens if I pass only a view file?" turned up three things. The behavior was right -- a Cube view projects members from cubes and defines none, so it cannot become an Ossie model on its own and the conversion is refused. But the message said "no convertible cubes found", which reads as if the file was not recognized at all. It now says a view was found, explains why that is not enough, and names the cubes its join_paths reference so the user knows which files to add. Pointing `-i` at a single `.yml` was refused as "not a directory". There is nothing ambiguous about a single model file, so it is now accepted. And cli.py had no tests at all -- 0% coverage. Adds test_cli.py (15 tests) covering the input shapes people reach for first (directory, single file, view-only), that stdout stays pipeable while issues go to stderr, that a fan-out refusal exits non-zero and --no-strict-fanout downgrades it, that node_modules and dotfiles are skipped, and a CLI round trip that reproduces the TPC-DS fixture. 216 tests; coverage 96% -> 97%, cli.py 0% -> 99%. Co-Authored-By: Claude Opus 5 --- converters/cube/README.md | 21 +- converters/cube/src/ossie_cube/cli.py | 24 +- converters/cube/src/ossie_cube/cube_to_osi.py | 54 +++- converters/cube/tests/test_cli.py | 238 ++++++++++++++++++ 4 files changed, 319 insertions(+), 18 deletions(-) create mode 100644 converters/cube/tests/test_cli.py diff --git a/converters/cube/README.md b/converters/cube/README.md index 83609f2c..edce0ef3 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -67,12 +67,18 @@ ossie-cube import -i model/ [-o model.yaml] [--name my_model] [--view sales] ossie-cube export -i model.yaml -o model/ [--dialect SNOWFLAKE] [--base-cube orders] ``` -`import` with no `-o` writes the Ossie YAML to stdout; `export` always needs `-o` -(a directory). Issues always go to stderr. `--view` picks which view's -name/description/AI context map onto the Ossie model when the directory holds -several; `--name` overrides the model name. `--base-cube` picks the cube a -*generated* view is rooted at, and is only consulted for a hand-authored Ossie -model with no stashed views. +`import` takes a model directory *or* a single model file, and with no `-o` writes +the Ossie YAML to stdout; `export` always needs `-o` (a directory). Issues always go +to stderr, so stdout stays pipeable. `--view` picks which view's +name/description/AI context map onto the Ossie model when the input holds several; +`--name` overrides the model name. `--base-cube` picks the cube a *generated* view +is rooted at, and is only consulted for a hand-authored Ossie model with no stashed +views. + +**A view on its own is not a model.** A Cube view projects members from cubes and +defines none of its own, so passing only `views/sales.yml` is refused -- with an +error naming the cubes it references, so you know which files to add. Include the +cube files (or point `-i` at the model directory). ### Python API @@ -229,7 +235,8 @@ uv sync uv run pytest ``` -Example-based unit tests per direction, fixture round-trip tests (including the +216 tests at 97% line coverage: example-based unit tests per direction, CLI +behavior tests, fixture round-trip tests (including the [TPC-DS model](../../examples/tpcds_semantic_model.yaml) the converter guide asks for as a baseline), core-spec JSON Schema validation of every emitted Ossie document, and Hypothesis property-based round-trip tests over generated Cube diff --git a/converters/cube/src/ossie_cube/cli.py b/converters/cube/src/ossie_cube/cli.py index 8616a2a7..1f2c2e4b 100644 --- a/converters/cube/src/ossie_cube/cli.py +++ b/converters/cube/src/ossie_cube/cli.py @@ -49,7 +49,8 @@ def _build_parser(): imp = sub.add_parser( "import", help="Cube data model directory -> Apache Ossie semantic model YAML") - imp.add_argument("-i", "--input", required=True, help="Cube model directory") + imp.add_argument("-i", "--input", required=True, + help="Cube model directory, or a single model file") imp.add_argument("-o", "--output", help="output Ossie YAML file (default: stdout)") imp.add_argument("--name", @@ -76,15 +77,22 @@ def _build_parser(): return parser -def _read_model_dir(path): - """Collect every file under a Cube model directory as {relative path: text}. +def _read_model_input(path): + """Collect a Cube model as {relative path: text}. - Everything is collected, not just YAML: a `.js` data model has no Ossie form, - but the converter preserves it so a round trip does not lose the file. Hidden - files and directories (including `node_modules`) are skipped. + `path` is normally a model directory, but a single file is accepted too -- + pointing at one `.yml` is a natural thing to try and there is nothing ambiguous + about it. + + Under a directory everything is collected, not just YAML: a `.js` data model has + no Ossie form, but the converter preserves it so a round trip does not lose the + file. Hidden files and directories (including `node_modules`) are skipped. """ + if os.path.isfile(path): + with open(path) as fh: + return {os.path.basename(path): fh.read()} if not os.path.isdir(path): - raise ConversionError(f"'{path}' is not a directory") + raise ConversionError(f"'{path}' is not a file or directory") files = {} for dirpath, dirnames, filenames in os.walk(path): dirnames[:] = [d for d in sorted(dirnames) @@ -126,7 +134,7 @@ def main(argv=None): _report(issues) return 0 - files = _read_model_dir(args.input) + files = _read_model_input(args.input) out, issues = convert_cube_to_ossie( files, model_name=args.name, view=args.view, strict_fanout=args.strict_fanout) diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index bbcd76c6..8ae1bd48 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -112,9 +112,7 @@ def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=True) cubes, cube_paths, views, view_paths, extra_files = _collect(files, issues) if not cubes: - raise ConversionError( - "no convertible cubes found (a `.yml` file with a top-level `cubes:` " - "list); nothing to convert") + raise ConversionError(_no_cubes_message(views)) # The mapped view supplies the Ossie model's identity. Cube users are # view-first, and Cube's own agent reads `meta.ai_context` only from views and @@ -272,6 +270,56 @@ def _as_named_list(value, what): f"{what}: expected a list or mapping, got {type(value).__name__}") +def _cubes_referenced_by(view): + """The cube names a view's `cubes:` entries address, in order. + + Every segment of a `join_path` names a cube (`orders.users.addresses` reaches + three), so all of them count as referenced. + """ + names = [] + for entry in view.get("cubes") or []: + if not isinstance(entry, dict): + continue + path = entry.get("join_path") + if not isinstance(path, str) or not path: + continue + for segment in path.split("."): + if segment and segment not in names: + names.append(segment) + return names + + +def _no_cubes_message(views): + """Explain *why* there is nothing to convert. + + Being handed only view files is an easy mistake -- a Cube view looks like a + complete model, and it is what a view-first user thinks of as "the model". But a + view only projects members from cubes and defines none of its own, so it cannot + become an Ossie semantic model on its own. Naming the cubes it references turns + the error into instructions. + """ + if not views: + return ("no convertible cubes found (a `.yml` file with a top-level " + "`cubes:` list); nothing to convert") + referenced = [] + for view in views.values(): + for name in _cubes_referenced_by(view): + if name not in referenced: + referenced.append(name) + which = ", ".join(f"'{v}'" for v in sorted(views)) + needed = ( + f" It references {', '.join(repr(c) for c in referenced)}, so include the " + f"file(s) defining those cubes." + if referenced else + " Include the files defining the cubes it draws from." + ) + return ( + f"found only view(s) {which} and no cubes. A Cube view projects members " + f"from cubes rather than defining any, so it has no Ossie dataset to " + f"convert on its own.{needed}" + ) + + def _order_by_view(cubes, mapped_view): """Order the datasets the way the mapped view presents them. diff --git a/converters/cube/tests/test_cli.py b/converters/cube/tests/test_cli.py new file mode 100644 index 00000000..d4c5f761 --- /dev/null +++ b/converters/cube/tests/test_cli.py @@ -0,0 +1,238 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Command-line behavior: what the user actually types, and what they get back. + +Covers the input shapes people reach for first -- a whole model directory, a single +file, and (a common mistake) just the view -- plus the exit codes and where output +goes, since those are the converter's contract with a shell script. +""" + +import pytest +from _util import REPO_ROOT, load_fixture_dir, parse + +from ossie_cube.cli import main + +_ORDERS = ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: count\n" + " type: count\n" +) +_VIEW = ( + "views:\n" + " - name: sales\n" + " description: Sales overview\n" + " cubes:\n" + " - join_path: orders\n" + " includes: '*'\n" +) + + +def _write(root, **files): + for rel, text in files.items(): + path = root / rel.replace("|", "/") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + return root + + +# --- input shapes --------------------------------------------------------------- + +def test_a_model_directory_converts(tmp_path, capsys): + model = _write(tmp_path / "model", **{ + "cubes|orders.yml": _ORDERS, "views|sales.yml": _VIEW}) + assert main(["import", "-i", str(model)]) == 0 + doc = parse(capsys.readouterr().out) + assert doc["semantic_model"][0]["name"] == "sales" + + +def test_a_single_file_converts(tmp_path, capsys): + """Pointing at one `.yml` is a natural thing to try and there is nothing + ambiguous about it, so it is accepted rather than refused on a technicality.""" + path = tmp_path / "orders.yml" + path.write_text(_ORDERS) + assert main(["import", "-i", str(path)]) == 0 + doc = parse(capsys.readouterr().out) + assert [d["name"] for d in doc["semantic_model"][0]["datasets"]] == ["orders"] + + +def test_only_a_view_file_is_refused_with_an_actionable_message(tmp_path, capsys): + """The likeliest mistake for a view-first user: a Cube view looks like the whole + model, but it projects members from cubes and defines none, so the error names + the cubes whose files are missing rather than claiming nothing was recognized.""" + path = tmp_path / "sales.yml" + path.write_text(_VIEW) + assert main(["import", "-i", str(path)]) == 1 + err = capsys.readouterr().err + assert "found only view(s) 'sales' and no cubes" in err + assert "projects members from cubes" in err + assert "'orders'" in err # named from the view's join_path + + +def test_a_view_with_no_cube_references_still_explains_itself(tmp_path, capsys): + path = tmp_path / "bare.yml" + path.write_text("views:\n - name: sales\n description: Sales\n") + assert main(["import", "-i", str(path)]) == 1 + assert "Include the files defining the cubes it draws from" in \ + capsys.readouterr().err + + +def test_a_missing_path_is_reported_not_traced(tmp_path, capsys): + assert main(["import", "-i", str(tmp_path / "nope")]) == 1 + assert "is not a file or directory" in capsys.readouterr().err + + +def test_an_empty_directory_is_reported(tmp_path, capsys): + empty = tmp_path / "empty" + empty.mkdir() + assert main(["import", "-i", str(empty)]) == 1 + assert "holds no files" in capsys.readouterr().err + + +def test_node_modules_and_dotfiles_are_skipped(tmp_path, capsys): + model = _write(tmp_path / "model", **{ + "cubes|orders.yml": _ORDERS, + "node_modules|junk.yml": "cubes:\n - name: junk\n sql_table: t\n", + ".hidden.yml": "cubes:\n - name: hidden\n sql_table: t\n", + }) + assert main(["import", "-i", str(model)]) == 0 + doc = parse(capsys.readouterr().out) + assert [d["name"] for d in doc["semantic_model"][0]["datasets"]] == ["orders"] + + +# --- output and exit codes ------------------------------------------------------ + +def test_output_goes_to_a_file_when_asked(tmp_path, capsys): + model = _write(tmp_path / "model", **{"cubes|orders.yml": _ORDERS}) + out = tmp_path / "model.yaml" + assert main(["import", "-i", str(model), "-o", str(out)]) == 0 + assert capsys.readouterr().out == "" + assert parse(out.read_text())["semantic_model"][0]["datasets"] + + +def test_issues_go_to_stderr_so_stdout_stays_pipeable(tmp_path, capsys): + model = _write(tmp_path / "model", **{ + "cubes|users.yml": ( + "cubes:\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: home\n" + " type: geo\n" + " latitude:\n" + " sql: lat\n" + " longitude:\n" + " sql: lon\n" + )}) + assert main(["import", "-i", str(model)]) == 0 + captured = capsys.readouterr() + assert "GEO_DIMENSION_SPLIT" in captured.err + assert "conversion issue" in captured.err + parse(captured.out) # stdout is still clean YAML + + +def test_fanout_refusal_exits_nonzero_and_the_flag_downgrades_it(tmp_path, capsys): + model = _write(tmp_path / "model", **{"cubes|m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {users}.id\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: ltv\n" + " sql: \"{CUBE}.ltv\"\n" + " type: sum\n" + )}) + assert main(["import", "-i", str(model)]) == 1 + assert "FANOUT_UNSAFE_METRIC" in capsys.readouterr().err + + assert main(["import", "-i", str(model), "--no-strict-fanout"]) == 0 + captured = capsys.readouterr() + assert "FANOUT_UNSAFE_METRIC" in captured.err + assert parse(captured.out)["semantic_model"][0]["metrics"] + + +def test_view_and_name_flags_take_effect(tmp_path, capsys): + model = _write(tmp_path / "model", **{ + "cubes|orders.yml": _ORDERS, + "views|a.yml": "views:\n - name: a\n description: A\n", + "views|b.yml": "views:\n - name: b\n description: B\n", + }) + assert main(["import", "-i", str(model), "--view", "b"]) == 0 + assert parse(capsys.readouterr().out)["semantic_model"][0]["description"] == "B" + + assert main(["import", "-i", str(model), "--view", "b", + "--name", "custom"]) == 0 + assert parse(capsys.readouterr().out)["semantic_model"][0]["name"] == "custom" + + assert main(["import", "-i", str(model), "--view", "ghost"]) == 1 + assert "not found" in capsys.readouterr().err + + +# --- export --------------------------------------------------------------------- + +def test_export_writes_the_model_directory(tmp_path, capsys): + out = tmp_path / "out" + assert main(["export", "-i", + str(REPO_ROOT / "examples" / "tpcds_semantic_model.yaml"), + "-o", str(out)]) == 0 + assert (out / "model" / "cubes" / "store_sales.yml").is_file() + assert (out / "model" / "views" / "tpcds_retail_model.yml").is_file() + assert "Wrote 6 file(s)" in capsys.readouterr().err + + +def test_export_of_a_missing_input_is_reported(tmp_path, capsys): + assert main(["export", "-i", str(tmp_path / "nope.yaml"), + "-o", str(tmp_path / "out")]) == 1 + assert "Error:" in capsys.readouterr().err + + +def test_a_cli_round_trip_reproduces_the_fixture(tmp_path, capsys): + fixture = load_fixture_dir("tpcds_cube") + src = _write(tmp_path / "src", **{k.replace("/", "|"): v + for k, v in fixture.items()}) + ossie = tmp_path / "model.yaml" + back = tmp_path / "back" + assert main(["import", "-i", str(src), "-o", str(ossie)]) == 0 + assert main(["export", "-i", str(ossie), "-o", str(back)]) == 0 + capsys.readouterr() + for rel in fixture: + assert (back / rel.replace("/", "/")).is_file(), rel + assert parse((back / rel).read_text()) == parse(fixture[rel]) + + +def test_no_subcommand_is_a_usage_error(): + with pytest.raises(SystemExit) as excinfo: + main([]) + assert excinfo.value.code == 2 From f39803e6c2bc5c4b448a3f9726a40b176c94b874 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 01:02:45 +0500 Subject: [PATCH 05/46] tests: add more edge cases --- converters/cube/tests/test_edge_cases.py | 80 ++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 84874100..22b650dd 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -397,6 +397,86 @@ def test_off_layout_files_are_restored_with_their_grouping(): assert stash["view_files"]["main"] == "schema/warehouse/everything.yaml" +_MIXED_VIEW_FILE = ( + "views:\n" + " - name: sales\n" + " description: Sales overview\n" + " meta:\n" + " ai_context: Use for revenue questions.\n" + " cubes:\n" + " - join_path: orders\n" + " includes: '*'\n" + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: revenue\n" + " sql: \"{CUBE}.amount\"\n" + " type: sum\n" +) + + +def test_a_view_file_may_also_define_cubes(): + """`cubes:` and `views:` are independent top-level keys, so one file can hold + both -- a self-contained model. Note the view's own nested `cubes:` (its + include list) is a different key at a different level and is not confused with + cube definitions.""" + files = {"model/views/sales.yml": _MIXED_VIEW_FILE} + ossie, _ = convert_cube_to_ossie(files) + model = model_of(ossie) + # The view supplied the model identity... + assert model["name"] == "sales" + assert model["description"] == "Sales overview" + assert model["ai_context"]["instructions"] == "Use for revenue questions." + # ...and the cube in the same file became the dataset. + assert [d["name"] for d in model["datasets"]] == ["orders"] + assert expr_of(model["metrics"][0]) == "SUM(orders.amount)" + # The view's include list round-trips as curation, not as a dataset. + assert stash_of(model)["views"]["sales"]["cubes"] == [ + {"join_path": "orders", "includes": "*"}] + + +def test_a_mixed_file_is_rebuilt_as_one_file(): + """Both halves have to go back into the single file they came from, rather than + being split into the canonical per-cube and per-view layout.""" + files = {"model/views/sales.yml": _MIXED_VIEW_FILE} + _, back, _ = _roundtrip(files) + assert set(back) == {"model/views/sales.yml"} + assert parse_files(back) == parse_files(files) + rebuilt = parse(back["model/views/sales.yml"]) + assert [c["name"] for c in rebuilt["cubes"]] == ["orders"] + assert [v["name"] for v in rebuilt["views"]] == ["sales"] + + +def test_a_cube_file_may_also_define_views(): + """The mirror image: the canonical cube path holding the view. The view's path is + the off-layout one here, so it is the one that gets stashed.""" + files = {"model/cubes/orders.yml": _MIXED_VIEW_FILE} + ossie, back, _ = _roundtrip(files) + assert stash_of(model_of(ossie))["view_files"]["sales"] == ( + "model/cubes/orders.yml") + assert "cube_files" not in stash_of(model_of(ossie)) + assert set(back) == {"model/cubes/orders.yml"} + assert parse_files(back) == parse_files(files) + + +def test_a_single_monolithic_file_round_trips(): + """Neither path is canonical, so both are stashed and both return to the one + file -- the shape you get from `-i model.yml`.""" + files = {"model.yml": _MIXED_VIEW_FILE} + ossie, back, _ = _roundtrip(files) + stash = stash_of(model_of(ossie)) + assert stash["cube_files"]["orders"] == "model.yml" + assert stash["view_files"]["sales"] == "model.yml" + assert set(back) == {"model.yml"} + assert parse_files(back) == parse_files(files) + + def test_non_model_yaml_is_preserved_verbatim(): files = { "model/cubes/orders.yml": ( From c1e778ecba2f8259da5994e08800d5d462a9fb82 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 01:24:32 +0500 Subject: [PATCH 06/46] Accept several input paths on import, not just one root Cube itself has a single model root (`CUBEJS_SCHEMA_PATH` is one string, default `model`), so pointing at that root is the idiomatic whole-project case and the recursive walk already handles files spread across subdirectories under it. But *requiring* one path was friction: converting two cubes out of fifty, or files that live in separate trees, meant assembling a directory first just to satisfy the CLI. `-i` now takes any number of files and/or directories, so globs work too. Files are keyed relative to the deepest directory containing every input, because those keys decide where export writes them back. That generalization is deliberately behavior-preserving: one directory anchors to itself and one file to its own directory, so existing round trips key exactly as before. Two files from `cubes/` and `views/` key as `cubes/orders.yml` and `views/sales.yml`, and export reproduces that tree. Overlapping inputs (a directory plus a file inside it) are now an error rather than reading the same file twice. Verified on a real Cube model passed as two explicit file paths: round trip is content-identical with the original filenames preserved, the Ossie output passes validation/validate.py, and the fan-out guard correctly refuses both measures on the joined cube's `one` side. 226 tests, 96% line coverage. Co-Authored-By: Claude Opus 5 --- converters/cube/README.md | 31 ++++++--- converters/cube/src/ossie_cube/cli.py | 96 ++++++++++++++++++--------- converters/cube/tests/test_cli.py | 74 +++++++++++++++++++++ 3 files changed, 162 insertions(+), 39 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index edce0ef3..118f2884 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -67,13 +67,28 @@ ossie-cube import -i model/ [-o model.yaml] [--name my_model] [--view sales] ossie-cube export -i model.yaml -o model/ [--dialect SNOWFLAKE] [--base-cube orders] ``` -`import` takes a model directory *or* a single model file, and with no `-o` writes -the Ossie YAML to stdout; `export` always needs `-o` (a directory). Issues always go -to stderr, so stdout stays pipeable. `--view` picks which view's -name/description/AI context map onto the Ossie model when the input holds several; -`--name` overrides the model name. `--base-cube` picks the cube a *generated* view -is rooted at, and is only consulted for a hand-authored Ossie model with no stashed -views. +`import` accepts a model directory (walked recursively), individual files, or any +mix of several — so converting part of a model does not mean assembling a directory +first: + +```bash +ossie-cube import -i model/ # the whole model +ossie-cube import -i model/cubes/orders.yml # one file +ossie-cube import -i model/cubes/orders.yml model/views/*.yml # a subset +``` + +Cube itself has a single model root (`CUBEJS_SCHEMA_PATH` is one path), so pointing +at that root is the idiomatic whole-project case. With several paths, files are keyed +relative to their common parent directory — which is what decides where `export` +writes them back — and the single-directory and single-file cases are keyed exactly +as they would be alone. + +With no `-o`, `import` writes the Ossie YAML to stdout; `export` always needs `-o` (a +directory). Issues always go to stderr, so stdout stays pipeable. `--view` picks +which view's name/description/AI context map onto the Ossie model when the input +holds several; `--name` overrides the model name. `--base-cube` picks the cube a +*generated* view is rooted at, and is only consulted for a hand-authored Ossie model +with no stashed views. **A view on its own is not a model.** A Cube view projects members from cubes and defines none of its own, so passing only `views/sales.yml` is refused -- with an @@ -235,7 +250,7 @@ uv sync uv run pytest ``` -216 tests at 97% line coverage: example-based unit tests per direction, CLI +226 tests at 96% line coverage: example-based unit tests per direction, CLI behavior tests, fixture round-trip tests (including the [TPC-DS model](../../examples/tpcds_semantic_model.yaml) the converter guide asks for as a baseline), core-spec JSON Schema validation of every emitted Ossie diff --git a/converters/cube/src/ossie_cube/cli.py b/converters/cube/src/ossie_cube/cli.py index 1f2c2e4b..ce4804b1 100644 --- a/converters/cube/src/ossie_cube/cli.py +++ b/converters/cube/src/ossie_cube/cli.py @@ -18,12 +18,15 @@ """Command-line interface for the Apache Ossie <-> Cube converter. ossie-cube import -i model/ [-o model.yaml] [--name my_model] [--view sales] + ossie-cube import -i cubes/orders.yml cubes/users.yml views/sales.yml ossie-cube export -i model.yaml -o model/ [--dialect SNOWFLAKE] [--base-cube orders] -`import` converts a Cube data model directory (any `.yml` holding `cubes:` / -`views:`) into an Apache Ossie semantic model; with no `-o` the Ossie YAML goes to -stdout. `export` does the reverse and always needs `-o` (a directory). -Conversions that could not carry something across print an issue list to stderr. +`import` converts a Cube data model (any `.yml` holding `cubes:` / `views:`) into an +Apache Ossie semantic model; with no `-o` the Ossie YAML goes to stdout. It accepts +a model directory, individual files, or a mix of several -- so converting part of a +model does not mean assembling a directory first. `export` does the reverse and +always needs `-o` (a directory). Conversions that could not carry something across +print an issue list to stderr. By default a metric whose value a static Ossie expression cannot keep correct under row multiplication is refused on import, mirroring Cube's own refusal to @@ -49,8 +52,11 @@ def _build_parser(): imp = sub.add_parser( "import", help="Cube data model directory -> Apache Ossie semantic model YAML") - imp.add_argument("-i", "--input", required=True, - help="Cube model directory, or a single model file") + imp.add_argument("-i", "--input", required=True, nargs="+", + metavar="PATH", + help="Cube model directories and/or files. A directory is " + "walked recursively; several paths are merged, keyed " + "relative to their common parent (globs work)") imp.add_argument("-o", "--output", help="output Ossie YAML file (default: stdout)") imp.add_argument("--name", @@ -77,36 +83,64 @@ def _build_parser(): return parser -def _read_model_input(path): - """Collect a Cube model as {relative path: text}. +def _read_model_input(paths): + """Collect a Cube model as {relative path: text} from one or more paths. - `path` is normally a model directory, but a single file is accepted too -- - pointing at one `.yml` is a natural thing to try and there is nothing ambiguous - about it. + Cube itself has a single model root (`CUBEJS_SCHEMA_PATH`, one string), so + pointing at a model directory is the idiomatic whole-project case. But + converting part of a model -- two cubes out of fifty, or files that live in + different trees -- is a real workflow, so several paths merge into one model + rather than forcing the caller to assemble a directory first. - Under a directory everything is collected, not just YAML: a `.js` data model has - no Ossie form, but the converter preserves it so a round trip does not lose the - file. Hidden files and directories (including `node_modules`) are skipped. + Keys are relative to the deepest directory containing every input, which + leaves the single-directory and single-file cases keyed exactly as before. + Directories are walked recursively, collecting everything rather than only + YAML: a `.js` data model has no Ossie form, but the converter preserves it so a + round trip does not lose the file. Hidden files and directories (including + `node_modules`) are skipped. """ - if os.path.isfile(path): - with open(path) as fh: - return {os.path.basename(path): fh.read()} - if not os.path.isdir(path): - raise ConversionError(f"'{path}' is not a file or directory") + resolved = [os.path.abspath(p) for p in paths] + for path, original in zip(resolved, paths): + if not os.path.exists(path): + raise ConversionError(f"'{original}' is not a file or directory") + + # The anchor keys every file. Using the inputs' common parent means one + # directory anchors to itself and one file to its own directory, so those + # cases are unchanged; several inputs stay distinguishable from each other. + containers = [p if os.path.isdir(p) else os.path.dirname(p) for p in resolved] + try: + anchor = os.path.commonpath(containers) + except ValueError: + # No shared prefix at all (different drives on Windows); fall back to bare + # file names, which are still unique or else reported as a collision below. + anchor = None + files = {} - for dirpath, dirnames, filenames in os.walk(path): - dirnames[:] = [d for d in sorted(dirnames) - if not d.startswith(".") and d != "node_modules"] - for fname in sorted(filenames): - if fname.startswith("."): - continue - rel = os.path.relpath(os.path.join(dirpath, fname), path) - rel = rel.replace(os.sep, "/") - with open(os.path.join(dirpath, fname)) as fh: - files[rel] = fh.read() + for path in resolved: + if os.path.isfile(path): + _collect_file(files, path, anchor) + continue + for dirpath, dirnames, filenames in os.walk(path): + dirnames[:] = [d for d in sorted(dirnames) + if not d.startswith(".") and d != "node_modules"] + for fname in sorted(filenames): + if not fname.startswith("."): + _collect_file(files, os.path.join(dirpath, fname), anchor) if not files: - raise ConversionError(f"'{path}' holds no files") - return files + raise ConversionError( + f"{', '.join(repr(p) for p in paths)} holds no files") + return dict(sorted(files.items())) + + +def _collect_file(files, path, anchor): + rel = (os.path.basename(path) if anchor is None + else os.path.relpath(path, anchor)).replace(os.sep, "/") + if rel in files: + raise ConversionError( + f"two inputs both resolve to '{rel}'; pass their common parent " + f"directory instead, or rename one") + with open(path) as fh: + files[rel] = fh.read() def _report(issues): diff --git a/converters/cube/tests/test_cli.py b/converters/cube/tests/test_cli.py index d4c5f761..e608bd4f 100644 --- a/converters/cube/tests/test_cli.py +++ b/converters/cube/tests/test_cli.py @@ -78,6 +78,80 @@ def test_a_single_file_converts(tmp_path, capsys): assert [d["name"] for d in doc["semantic_model"][0]["datasets"]] == ["orders"] +def test_several_paths_merge_into_one_model(tmp_path, capsys): + """Cube has a single model root, but converting part of a model -- or files from + different trees -- should not require assembling a directory first.""" + a = tmp_path / "cubes" / "orders.yml" + b = tmp_path / "views" / "sales.yml" + for path, text in ((a, _ORDERS), (b, _VIEW)): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + assert main(["import", "-i", str(a), str(b)]) == 0 + model = parse(capsys.readouterr().out)["semantic_model"][0] + assert model["name"] == "sales" # the view was picked up + assert [d["name"] for d in model["datasets"]] == ["orders"] + + +def test_several_paths_are_keyed_relative_to_their_common_parent(tmp_path, capsys): + """The keys decide where export writes the files back, so two inputs from + different subtrees have to stay distinguishable.""" + a = tmp_path / "cubes" / "orders.yml" + b = tmp_path / "views" / "sales.yml" + for path, text in ((a, _ORDERS), (b, _VIEW)): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + out = tmp_path / "model.yaml" + assert main(["import", "-i", str(a), str(b), "-o", str(out)]) == 0 + back = tmp_path / "back" + assert main(["export", "-i", str(out), "-o", str(back)]) == 0 + capsys.readouterr() + assert (back / "cubes" / "orders.yml").is_file() + assert (back / "views" / "sales.yml").is_file() + + +def test_mixing_a_directory_and_a_file_works(tmp_path, capsys): + model = _write(tmp_path / "model", **{"cubes|orders.yml": _ORDERS}) + extra = tmp_path / "extra.yml" + extra.write_text(_VIEW) + assert main(["import", "-i", str(model), str(extra)]) == 0 + assert parse(capsys.readouterr().out)["semantic_model"][0]["name"] == "sales" + + +def test_overlapping_inputs_are_reported(tmp_path, capsys): + """Passing a directory and a file inside it is an easy mistake (an overlapping + glob), and it would otherwise read the same file twice.""" + model = _write(tmp_path / "model", **{"cubes|orders.yml": _ORDERS}) + assert main(["import", "-i", str(model), + str(model / "cubes" / "orders.yml")]) == 1 + err = capsys.readouterr().err + assert "both resolve to 'cubes/orders.yml'" in err + + +def test_the_same_cube_in_two_inputs_is_reported(tmp_path, capsys): + a = tmp_path / "one" / "orders.yml" + b = tmp_path / "two" / "orders.yml" + for path in (a, b): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(_ORDERS) + # Distinct keys ('one/orders.yml', 'two/orders.yml'), but the same cube name. + assert main(["import", "-i", str(a), str(b)]) == 1 + assert "defined twice" in capsys.readouterr().err + + +def test_a_single_path_is_keyed_exactly_as_before(tmp_path, capsys): + """The multi-path anchor must not change the one-directory case, since the keys + are what export writes back.""" + model = _write(tmp_path / "model", **{ + "cubes|orders.yml": _ORDERS, "views|sales.yml": _VIEW}) + out = tmp_path / "model.yaml" + assert main(["import", "-i", str(model), "-o", str(out)]) == 0 + back = tmp_path / "back" + assert main(["export", "-i", str(out), "-o", str(back)]) == 0 + capsys.readouterr() + assert (back / "cubes" / "orders.yml").is_file() + assert (back / "views" / "sales.yml").is_file() + + def test_only_a_view_file_is_refused_with_an_actionable_message(tmp_path, capsys): """The likeliest mistake for a view-first user: a Cube view looks like the whole model, but it projects members from cubes and defines none, so the error names From c24f7bf9c0f3657f6cc8a7a05c2996011766d7c7 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 01:34:42 +0500 Subject: [PATCH 07/46] Register CUBE in the converters supported-vendors table The converter emits vendor_name: CUBE in custom_extensions, so it belongs in the table converters/README.md keeps of vendors with defined extensions. Deliberately not touching the parallel list in core-spec/spec.md: vendor_name is a free-form string, so no spec change is needed, and edits under core-spec/ carry the heavier review process. Co-Authored-By: Claude Opus 5 --- converters/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/converters/README.md b/converters/README.md index 5c9a4d54..ee0b6099 100644 --- a/converters/README.md +++ b/converters/README.md @@ -76,6 +76,7 @@ The Ossie specification currently defines extensions for the following vendors: | `OMNI` | Omni semantic model | | `WISDOM` | WisdomAI domain | | `NVIDIA_GSF` | NVIDIA Generative Semantic Fabric standalone YAML | +| `CUBE` | Cube data model | Each vendor may define custom extensions (via the `custom_extensions` field in the Ossie spec) to carry vendor-specific metadata that does not have an equivalent in the core specification. From dd00190968578a4b3255d30c1431ea8fe75d8cca Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 12:12:08 +0500 Subject: [PATCH 08/46] Add Ossie-side test fixtures and snapshot both directions Every other bidirectional converter in the repo pairs a vendor fixture with an Ossie one (databricks, omni, gooddata, orionbelt); this converter was the only one keeping its Ossie inputs as inline Python strings. Adds fixtureA_ossie.yaml and tpcds_ossie.yaml, and moves the hand-authored Ossie model out of test_roundtrip.py into hand_authored_ossie.yaml. The point is not just tidiness. Following the databricks pattern, the fixtures are asserted as whole-document snapshots in both directions: import must reproduce the Ossie fixture, and exporting that fixture must reproduce the Cube fixture. Field-level assertions cannot see an unintended change elsewhere in the document; a snapshot shows it as a readable diff. Each fixture carries the command to regenerate it. Co-Authored-By: Claude Opus 5 --- .../cube/tests/fixtures/fixtureA_ossie.yaml | 203 ++++++ .../tests/fixtures/hand_authored_ossie.yaml | 95 +++ .../cube/tests/fixtures/tpcds_ossie.yaml | 645 ++++++++++++++++++ converters/cube/tests/test_roundtrip.py | 110 +-- 4 files changed, 974 insertions(+), 79 deletions(-) create mode 100644 converters/cube/tests/fixtures/fixtureA_ossie.yaml create mode 100644 converters/cube/tests/fixtures/hand_authored_ossie.yaml create mode 100644 converters/cube/tests/fixtures/tpcds_ossie.yaml diff --git a/converters/cube/tests/fixtures/fixtureA_ossie.yaml b/converters/cube/tests/fixtures/fixtureA_ossie.yaml new file mode 100644 index 00000000..d78abf3d --- /dev/null +++ b/converters/cube/tests/fixtures/fixtureA_ossie.yaml @@ -0,0 +1,203 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# The Ossie form of fixtureA_cube/, asserted as a whole-document snapshot so an +# unintended change anywhere in the output shows up as a readable diff. +# Regenerate with: uv run ossie-cube import -i tests/fixtures/fixtureA_cube + +version: 0.2.0.dev0 +semantic_model: +- name: sales + description: Sales overview + ai_context: + instructions: | + Primary view for revenue analysis. Use it for any question about sales, orders, or customer spend. + datasets: + - name: orders + source: public.orders + description: Customer orders + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "id", "type": "number"}' + - name: user_id + expression: + dialects: + - dialect: ANSI_SQL + expression: user_id + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "user_id", "type": "number"}' + - name: status + expression: + dialects: + - dialect: ANSI_SQL + expression: status + datatype: String + label: Order Status + description: Current order status + ai_context: + instructions: Values are pending, shipped, and completed. + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "status"}' + - name: created_at + expression: + dialects: + - dialect: ANSI_SQL + expression: created_at + datatype: DateTime + dimension: + is_time: true + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "created_at"}' + - name: is_large + expression: + dialects: + - dialect: ANSI_SQL + expression: amount > 500 + datatype: Boolean + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "{CUBE}.amount > 500"}' + primary_key: + - id + - name: users + source: SELECT * FROM public.users WHERE deleted_at IS NULL + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "id", "type": "number"}' + - name: city + expression: + dialects: + - dialect: ANSI_SQL + expression: city + datatype: String + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "city"}' + - name: location_latitude + expression: + dialects: + - dialect: ANSI_SQL + expression: lat + datatype: Float + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "geo": {"of": "location", "part": "latitude", "sql": "{CUBE}.lat"}}' + - name: location_longitude + expression: + dialects: + - dialect: ANSI_SQL + expression: lon + datatype: Float + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "geo": {"of": "location", "part": "longitude", "sql": "{CUBE}.lon"}}' + primary_key: + - id + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube_extras": {"segments": [{"name": "active", "sql": "{CUBE}.status + = ''active''"}]}}' + relationships: + - name: orders_to_users + from: orders + to: users + from_columns: + - user_id + to_columns: + - id + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "declared_on": "orders", "relationship": "many_to_one", "sql": + "{CUBE}.user_id = {users}.id"}' + metrics: + - name: orders__count + expression: + dialects: + - dialect: ANSI_SQL + expression: COUNT(DISTINCT orders.id) + datatype: Integer + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "orders", "name": "count"}' + - name: total_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.amount) + description: Total order amount + ai_context: + instructions: Use this for revenue questions. + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "orders", "measure": {"name": "total_amount", "sql": + "{CUBE}.amount", "type": "sum", "format": "currency"}}' + - name: completed_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(CASE WHEN (orders.status = 'completed') THEN orders.amount + END) + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "orders", "measure": {"name": "completed_amount", "sql": + "{CUBE}.amount", "type": "sum", "filters": [{"sql": "{CUBE}.status = ''completed''"}]}}' + - name: avg_order_value + expression: + dialects: + - dialect: ANSI_SQL + expression: (SUM(orders.amount)) / (COUNT(DISTINCT orders.id)) + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "orders", "measure": {"name": "avg_order_value", "sql": + "{total_amount} / {count}", "type": "number"}}' + - name: users__count + expression: + dialects: + - dialect: ANSI_SQL + expression: COUNT(DISTINCT users.id) + datatype: Integer + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "users", "name": "count"}' + - name: cities + expression: + dialects: + - dialect: ANSI_SQL + expression: COUNT(DISTINCT users.city) + datatype: Integer + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "users", "sql": "{CUBE}.city"}' + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "views": {"sales": {"name": "sales", "cubes": [{"join_path": + "orders", "includes": "*"}, {"join_path": "orders.users", "includes": ["city"]}]}}, + "mapped_view": "sales"}' diff --git a/converters/cube/tests/fixtures/hand_authored_ossie.yaml b/converters/cube/tests/fixtures/hand_authored_ossie.yaml new file mode 100644 index 00000000..e62f18f4 --- /dev/null +++ b/converters/cube/tests/fixtures/hand_authored_ossie.yaml @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# An Ossie model authored by hand rather than imported from Cube: no +# custom_extensions[CUBE] stash anywhere. Export therefore has to derive +# everything -- the cube layout, the join orientation, and a generated view -- +# instead of restoring it. Also carries the constructs Cube has no field for +# (unique_keys, a foreign vendor's extensions, structured ai_context), which +# export parks under meta.ossie. + +version: 0.2.0.dev0 +semantic_model: +- name: ecommerce + description: Orders and customers + ai_context: + instructions: Use for sales analysis. + synonyms: + - sales + - purchases + datasets: + - name: orders + source: sales.public.orders + primary_key: + - id + unique_keys: + - - order_number + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + datatype: Integer + - name: customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: customer_id + datatype: Integer + - name: ordered_at + expression: + dialects: + - dialect: ANSI_SQL + expression: ordered_at + datatype: Date + custom_extensions: + - vendor_name: SNOWFLAKE + data: '{"warehouse": "ANALYTICS_WH"}' + - name: customers + source: sales.public.customers + primary_key: + - id + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + datatype: Integer + - name: email + expression: + dialects: + - dialect: ANSI_SQL + expression: LOWER(email) + datatype: String + relationships: + - name: orders_to_customers + from: orders + to: customers + from_columns: + - customer_id + to_columns: + - id + metrics: + - name: total_revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.amount) + description: Total revenue + datatype: Decimal diff --git a/converters/cube/tests/fixtures/tpcds_ossie.yaml b/converters/cube/tests/fixtures/tpcds_ossie.yaml new file mode 100644 index 00000000..5ee7e1ae --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_ossie.yaml @@ -0,0 +1,645 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# The Ossie form of tpcds_cube/, asserted as a whole-document snapshot. +# Regenerate with: uv run ossie-cube import -i tests/fixtures/tpcds_cube + +version: 0.2.0.dev0 +semantic_model: +- name: tpcds_retail_model + description: TPC-DS retail semantic model for sales and customer analytics + ai_context: + instructions: Use this semantic model for retail analytics. It provides comprehensive + sales, customer, product, and store data from the TPC-DS benchmark. The model + supports time-based analysis, customer segmentation, product performance, and + store operations metrics. + datasets: + - name: store_sales + source: tpcds.public.store_sales + description: Fact table containing all store sales transactions + ai_context: + synonyms: + - sales transactions + - store purchases + - retail sales + - POS data + unique_keys: + - - ss_item_sk + - ss_ticket_number + fields: + - name: ss_sold_date_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_sold_date_sk + description: Foreign key to date dimension + ai_context: + synonyms: + - sale date + - transaction date + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "ss_sold_date_sk", "type": "number"}' + - name: ss_item_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_item_sk + description: Foreign key to item dimension + ai_context: + synonyms: + - product + - item + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "ss_item_sk", "type": "number"}' + - name: ss_customer_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_customer_sk + description: Foreign key to customer dimension + ai_context: + synonyms: + - customer + - buyer + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "ss_customer_sk", "type": "number"}' + - name: ss_store_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_store_sk + description: Foreign key to store dimension + ai_context: + synonyms: + - store + - location + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "ss_store_sk", "type": "number"}' + - name: ss_quantity + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_quantity + description: Quantity of items sold + ai_context: + synonyms: + - units sold + - quantity + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "ss_quantity", "type": "number"}' + - name: ss_sales_price + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_sales_price + description: Sales price per unit + ai_context: + synonyms: + - unit price + - price + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "ss_sales_price", "type": "number"}' + - name: ss_ext_sales_price + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_ext_sales_price + description: Extended sales price (quantity * price) + ai_context: + synonyms: + - total price + - line total + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "ss_ext_sales_price", "type": "number"}' + - name: ss_net_profit + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_net_profit + description: Net profit from the sale + ai_context: + synonyms: + - profit + - margin + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "ss_net_profit", "type": "number"}' + - name: ss_ticket_number + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_ticket_number + datatype: String + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "ss_ticket_number", "public": false}' + primary_key: + - ss_item_sk + - ss_ticket_number + - name: date_dim + source: tpcds.public.date_dim + description: Date dimension with calendar attributes + ai_context: + synonyms: + - calendar + - dates + - time periods + unique_keys: + - - d_date_sk + fields: + - name: d_date_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: d_date_sk + description: Surrogate key for date + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "d_date_sk", "type": "number"}' + - name: d_date + expression: + dialects: + - dialect: ANSI_SQL + expression: d_date + datatype: DateTime + dimension: + is_time: true + description: Actual date value + ai_context: + synonyms: + - date + - calendar date + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "d_date"}' + - name: d_year + expression: + dialects: + - dialect: ANSI_SQL + expression: d_year + description: Year + ai_context: + synonyms: + - year + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "d_year", "type": "number"}' + - name: d_quarter_name + expression: + dialects: + - dialect: ANSI_SQL + expression: d_quarter_name + datatype: DateTime + dimension: + is_time: true + description: Quarter name (e.g., 2024Q1) + ai_context: + synonyms: + - quarter + - fiscal quarter + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "d_quarter_name"}' + - name: d_month_name + expression: + dialects: + - dialect: ANSI_SQL + expression: d_month_name + datatype: DateTime + dimension: + is_time: true + description: Month name + ai_context: + synonyms: + - month + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "d_month_name"}' + primary_key: + - d_date_sk + - name: customer + source: tpcds.public.customer + description: Customer dimension with demographic information + ai_context: + synonyms: + - customers + - shoppers + - buyers + unique_keys: + - - c_customer_sk + fields: + - name: c_customer_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: c_customer_sk + description: Surrogate key for customer + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "c_customer_sk", "type": "number"}' + - name: c_customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: c_customer_id + datatype: String + description: Business key for customer + ai_context: + synonyms: + - customer ID + - customer number + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "c_customer_id"}' + - name: c_first_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_first_name + datatype: String + description: Customer first name + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "c_first_name"}' + - name: c_last_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_last_name + datatype: String + description: Customer last name + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "c_last_name"}' + - name: customer_full_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_first_name || ' ' || c_last_name + datatype: String + description: Customer full name (computed field) + ai_context: + synonyms: + - full name + - customer name + - name: c_email_address + expression: + dialects: + - dialect: ANSI_SQL + expression: c_email_address + datatype: String + description: Customer email address + ai_context: + synonyms: + - email + - contact + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "c_email_address"}' + primary_key: + - c_customer_sk + - name: item + source: tpcds.public.item + description: Item/Product dimension with product attributes + ai_context: + synonyms: + - products + - items + - merchandise + unique_keys: + - - i_item_sk + fields: + - name: i_item_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_sk + description: Surrogate key for item + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "i_item_sk", "type": "number"}' + - name: i_item_id + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_id + datatype: String + description: Business key for item + ai_context: + synonyms: + - item ID + - product ID + - SKU + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "i_item_id"}' + - name: i_item_desc + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_desc + datatype: String + description: Item description + ai_context: + synonyms: + - product description + - item name + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "i_item_desc"}' + - name: i_brand + expression: + dialects: + - dialect: ANSI_SQL + expression: i_brand + datatype: String + description: Brand name + ai_context: + synonyms: + - brand + - manufacturer + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "i_brand"}' + - name: i_category + expression: + dialects: + - dialect: ANSI_SQL + expression: i_category + datatype: String + description: Item category + ai_context: + synonyms: + - product category + - department + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "i_category"}' + - name: i_current_price + expression: + dialects: + - dialect: ANSI_SQL + expression: i_current_price + description: Current price of the item + ai_context: + synonyms: + - price + - list price + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "i_current_price", "type": "number"}' + primary_key: + - i_item_sk + - name: store + source: tpcds.public.store + description: Store dimension with location and store attributes + ai_context: + synonyms: + - stores + - retail locations + - branches + unique_keys: + - - s_store_id + fields: + - name: s_store_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: s_store_sk + description: Surrogate key for store + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "s_store_sk", "type": "number"}' + - name: s_store_id + expression: + dialects: + - dialect: ANSI_SQL + expression: s_store_id + datatype: String + description: Business key for store + ai_context: + synonyms: + - store ID + - store number + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "s_store_id"}' + - name: s_store_name + expression: + dialects: + - dialect: ANSI_SQL + expression: s_store_name + datatype: String + description: Store name + ai_context: + synonyms: + - store name + - location name + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "s_store_name"}' + - name: s_city + expression: + dialects: + - dialect: ANSI_SQL + expression: s_city + datatype: String + description: City where store is located + ai_context: + synonyms: + - city + - location + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "s_city"}' + - name: s_state + expression: + dialects: + - dialect: ANSI_SQL + expression: s_state + datatype: String + description: State where store is located + ai_context: + synonyms: + - state + - region + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "s_state"}' + - name: s_number_employees + expression: + dialects: + - dialect: ANSI_SQL + expression: s_number_employees + description: Number of employees at the store + ai_context: + synonyms: + - employee count + - staff size + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "s_number_employees", "type": "number"}' + primary_key: + - s_store_sk + relationships: + - name: store_sales_to_date_dim + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - d_date_sk + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "declared_on": "store_sales", "relationship": "many_to_one"}' + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: + - ss_customer_sk + to_columns: + - c_customer_sk + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "declared_on": "store_sales", "relationship": "many_to_one"}' + - name: store_sales_to_item + from: store_sales + to: item + from_columns: + - ss_item_sk + to_columns: + - i_item_sk + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "declared_on": "store_sales", "relationship": "many_to_one"}' + - name: store_sales_to_store + from: store_sales + to: store + from_columns: + - ss_store_sk + to_columns: + - s_store_sk + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "declared_on": "store_sales", "relationship": "many_to_one"}' + metrics: + - name: total_sales + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) + description: Total sales revenue across all transactions + ai_context: + synonyms: + - total revenue + - gross sales + - sales amount + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "store_sales", "sql": "{CUBE.ss_ext_sales_price}"}' + - name: total_profit + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_net_profit) + description: Total net profit from store sales + ai_context: + synonyms: + - net profit + - total earnings + - profit + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "store_sales", "sql": "{CUBE.ss_net_profit}"}' + - name: customer_lifetime_value + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk) + description: Average lifetime sales value per customer + ai_context: + synonyms: + - CLV + - LTV + - customer value + - lifetime revenue + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "store_sales", "measure": {"name": "customer_lifetime_value", + "sql": "SUM({CUBE.ss_ext_sales_price}) / COUNT(DISTINCT {customer.c_customer_sk})", + "type": "number"}}' + - name: sales_by_brand + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) + description: Total sales by brand (requires grouping by item.i_brand) + ai_context: + synonyms: + - brand sales + - brand performance + - brand revenue + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "store_sales", "sql": "{CUBE.ss_ext_sales_price}"}' + - name: store_productivity + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), + 0) + description: Sales per employee across stores + ai_context: + synonyms: + - sales per employee + - employee productivity + - revenue per employee + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "store_sales", "measure": {"name": "store_productivity", + "sql": "SUM({CUBE.ss_ext_sales_price}) / NULLIF(SUM({store.s_number_employees}), + 0)", "type": "number"}}' + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "views": {"tpcds_retail_model": {"name": "tpcds_retail_model", + "cubes": [{"join_path": "store_sales", "includes": "*"}, {"join_path": "store_sales.date_dim", + "includes": "*"}, {"join_path": "store_sales.customer", "includes": "*"}, {"join_path": + "store_sales.item", "includes": "*"}, {"join_path": "store_sales.store", "includes": + "*"}]}}, "mapped_view": "tpcds_retail_model"}' + - vendor_name: SALESFORCE + data: | + { + "tableau_workbook_id": "tpcds_retail_dashboard", + "einstein_enabled": true, + "crm_sync": { + "enabled": true, + "sync_frequency": "daily", + "customer_mapping": "customer.c_customer_id -> Account.AccountNumber" + }, + "tableau_semantics": { + "published": true, + "version": "0.1.1" + } + } + - vendor_name: DBT + data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' diff --git a/converters/cube/tests/test_roundtrip.py b/converters/cube/tests/test_roundtrip.py index 760fc22a..d5914c8a 100644 --- a/converters/cube/tests/test_roundtrip.py +++ b/converters/cube/tests/test_roundtrip.py @@ -26,7 +26,8 @@ import json import pytest -from _util import REPO_ROOT, load_fixture_dir, parse, parse_files +from _util import (REPO_ROOT, canon, load_fixture, load_fixture_dir, parse, + parse_files) from ossie_cube import convert_cube_to_ossie, convert_ossie_to_cube @@ -47,6 +48,32 @@ def test_cube_roundtrip_is_lossless(fixture): assert parse_files(files2) == parse_files(files) +@pytest.mark.parametrize("cube_dir,ossie_file", [ + ("fixtureA_cube", "fixtureA_ossie.yaml"), + ("tpcds_cube", "tpcds_ossie.yaml"), +]) +def test_import_matches_the_committed_ossie_fixture(cube_dir, ossie_file): + """Whole-document snapshot, so an unintended change anywhere in the output shows + up as a readable diff rather than slipping past field-level assertions. + + Regenerate with `ossie-cube import -i tests/fixtures/` when a change + to the output is intended. + """ + ossie, _ = convert_cube_to_ossie(load_fixture_dir(cube_dir)) + assert canon(parse(ossie)) == canon(parse(load_fixture(ossie_file))) + + +@pytest.mark.parametrize("cube_dir,ossie_file", [ + ("fixtureA_cube", "fixtureA_ossie.yaml"), + ("tpcds_cube", "tpcds_ossie.yaml"), +]) +def test_export_of_the_ossie_fixture_matches_the_cube_fixture(cube_dir, ossie_file): + """The same snapshot in the other direction: the committed Ossie fixture has to + export back to the committed Cube fixture.""" + files, _ = convert_ossie_to_cube(load_fixture(ossie_file)) + assert parse_files(files) == parse_files(load_fixture_dir(cube_dir)) + + @pytest.mark.parametrize("fixture", FIXTURES) def test_imported_ossie_validates_against_core_spec_schema(fixture): jsonschema = pytest.importorskip("jsonschema") @@ -74,7 +101,7 @@ def test_ossie_roundtrip_is_lossless(fixture): def test_hand_authored_ossie_gets_a_generated_view(): """A model with no stashed views is not from Cube, so export has to invent the view -- the model boundary Cube users work with.""" - ossie = _HAND_AUTHORED + ossie = load_fixture("hand_authored_ossie.yaml") files, _ = convert_ossie_to_cube(ossie) assert set(files) == { "model/cubes/orders.yml", "model/cubes/customers.yml", @@ -91,7 +118,7 @@ def test_hand_authored_ossie_gets_a_generated_view(): def test_hand_authored_ossie_survives_the_round_trip(): - files, _ = convert_ossie_to_cube(_HAND_AUTHORED) + files, _ = convert_ossie_to_cube(load_fixture("hand_authored_ossie.yaml")) ossie2, _ = convert_cube_to_ossie(files) model = parse(ossie2)["semantic_model"][0] assert model["name"] == "ecommerce" @@ -106,7 +133,7 @@ def test_hand_authored_ossie_survives_the_round_trip(): def test_ossie_only_constructs_are_parked_not_dropped(): """`unique_keys` and a foreign vendor's extensions have no Cube field, so they ride under `meta.ossie` and come back intact.""" - files, _ = convert_ossie_to_cube(_HAND_AUTHORED) + files, _ = convert_ossie_to_cube(load_fixture("hand_authored_ossie.yaml")) orders = parse(files["model/cubes/orders.yml"])["cubes"][0] parked = orders["meta"]["ossie"] assert parked["unique_keys"] == [["order_number"]] @@ -117,78 +144,3 @@ def test_ossie_only_constructs_are_parked_not_dropped(): assert ds["orders"]["unique_keys"] == [["order_number"]] vendors = {e["vendor_name"] for e in ds["orders"]["custom_extensions"]} assert "SNOWFLAKE" in vendors - - -_HAND_AUTHORED = """ -version: 0.2.0.dev0 -semantic_model: -- name: ecommerce - description: Orders and customers - ai_context: - instructions: Use for sales analysis. - synonyms: - - sales - - purchases - datasets: - - name: orders - source: sales.public.orders - primary_key: - - id - unique_keys: - - - order_number - fields: - - name: id - expression: - dialects: - - dialect: ANSI_SQL - expression: id - datatype: Integer - - name: customer_id - expression: - dialects: - - dialect: ANSI_SQL - expression: customer_id - datatype: Integer - - name: ordered_at - expression: - dialects: - - dialect: ANSI_SQL - expression: ordered_at - datatype: Date - custom_extensions: - - vendor_name: SNOWFLAKE - data: '{"warehouse": "ANALYTICS_WH"}' - - name: customers - source: sales.public.customers - primary_key: - - id - fields: - - name: id - expression: - dialects: - - dialect: ANSI_SQL - expression: id - datatype: Integer - - name: email - expression: - dialects: - - dialect: ANSI_SQL - expression: LOWER(email) - datatype: String - relationships: - - name: orders_to_customers - from: orders - to: customers - from_columns: - - customer_id - to_columns: - - id - metrics: - - name: total_revenue - expression: - dialects: - - dialect: ANSI_SQL - expression: SUM(orders.amount) - description: Total revenue - datatype: Decimal -""" From 5d7dc5f25c82da12e6aec4b52971e4af3c78f6d9 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 12:12:08 +0500 Subject: [PATCH 09/46] Resolve dimension names once; make the Hypothesis driver honour p MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both from the Copilot review on #289. Dimension names were sanitized separately in _convert_model (to decide which members a cube has) and again in _build_dimensions (to name them). The first pass used a fresh `taken` set per field, so a collision was silently swallowed by a set comprehension there and only rejected later in the second pass -- meaning the member set that decides `{CUBE.member}` vs `{CUBE}.column`, and where a measure lands, could be short a name while measures were being placed. Demonstrated: "Order Status" and "order status" collapsed to one name with no error. Now resolved once in _resolve_dimension_names and reused. That also fixes a defect the review did not mention: the old set included the two halves of a split geo dimension (location_latitude, location_longitude), which never exist as Cube dimensions since they merge back into `location`, so a metric referencing one would emit an unresolvable `{CUBE.location_…}`. The halves now resolve to the dimension they merge into. _HypothesisRnd.chance() ignored its `p` argument and always drew an unweighted boolean, so the Hypothesis driver explored a different distribution than the seeded one despite the docstring claiming they share a generator. Now weighted, and drawn so the minimal value means False -- shrinking toward the smallest model rather than the largest. 231 tests. Co-Authored-By: Claude Opus 5 --- converters/cube/src/ossie_cube/osi_to_cube.py | 60 +++++++++++++++---- converters/cube/tests/test_osi_to_cube.py | 25 ++++++++ .../cube/tests/test_roundtrip_properties.py | 8 ++- 3 files changed, 80 insertions(+), 13 deletions(-) diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 5d163d54..8fc1731d 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -131,14 +131,19 @@ def _convert_model(model, dialect, base_cube, issues): model_stash = read_stash(model) # Per-cube facts the join and measure stages need. + # Field -> dimension names are resolved once here and reused by every stage. + # Sanitizing per stage would let a collision go undetected in one place and be + # rejected in another, and would disagree about which members a cube actually + # has -- which decides `{CUBE.member}` vs `{CUBE}.column` and where a measure + # lands. + dim_names_by_cube = {} members_by_cube = {} pk_by_cube = {} for ds_name, ds in datasets.items(): cname = cube_names[ds_name] - members_by_cube[cname] = { - sanitize_name(f["name"], f"dataset '{ds_name}': field", set()) - for f in (ds.get("fields") or []) - } + dim_names_by_cube[cname] = _resolve_dimension_names( + ds, f"Model '{name}': dataset '{ds_name}'") + members_by_cube[cname] = set(dim_names_by_cube[cname].values()) pk_by_cube[cname] = [str(c) for c in (ds.get("primary_key") or [])] joins_by_cube = _build_joins(relationships, cube_names, issues) @@ -152,7 +157,7 @@ def _convert_model(model, dialect, base_cube, issues): files_content = {} for ds_name, ds in datasets.items(): cname = cube_names[ds_name] - cube = _build_cube(ds, cname, members_by_cube[cname], + cube = _build_cube(ds, cname, dim_names_by_cube[cname], joins_by_cube.get(cname), measures_by_cube.get(cname), dialect, issues) path = stashed_paths.get(cname) or cube_file(cname) @@ -233,7 +238,7 @@ def _ordered(obj, order): # --- cubes ---------------------------------------------------------------------- -def _build_cube(ds, cname, members, joins, measures, dialect, issues): +def _build_cube(ds, cname, dim_names, joins, measures, dialect, issues): ds_name = ds["name"] scope = f"dataset '{ds_name}'" stash = read_stash(ds) @@ -262,7 +267,8 @@ def _build_cube(ds, cname, members, joins, measures, dialect, issues): "Cube's agent reads ai_context only on views and members, " "so this cube-level value has no effect in Cube") - dimensions, covered = _build_dimensions(ds, cname, members, dialect, issues) + dimensions, covered = _build_dimensions( + ds, cname, dim_names, dialect, issues) # A primary-key column no field covers still has to exist as a dimension for # Cube to join or roll up the cube. pk_names = [] @@ -301,17 +307,47 @@ def _build_cube(ds, cname, members, joins, measures, dialect, issues): return _ordered(cube, _CUBE_KEY_ORDER) -def _build_dimensions(ds, cname, members, dialect, issues): +def _resolve_dimension_names(ds, scope): + """Map each of a dataset's fields to the Cube dimension name it becomes. + + Sanitization and collision detection happen here and nowhere else, so every + stage agrees on the result. Two subtleties the mapping has to get right: + + - A collision is an error, not a silent merge. Sanitizing with a fresh `taken` + set per field would hide one. + - The two halves of a split `geo` dimension map back to the *single* dimension + they merge into, so `location_latitude` resolves to `location`. Treating the + halves as members of their own would let a metric emit a `{CUBE.…}` reference + to a dimension the exported cube does not have. + """ + names = {} + taken = set() + for field in (ds.get("fields") or []): + fname = require_str(field, "name", f"{scope}: field") + geo = read_stash(field).get("geo") + if geo: + base = geo["of"] + names[fname] = base + taken.add(base.lower()) + continue + dname = sanitize_name(fname, f"{scope}: field", taken) + taken.add(dname.lower()) + names[fname] = dname + return names + + +def _build_dimensions(ds, cname, dim_names, dialect, issues): """Build a cube's dimensions from an Ossie dataset's fields. Returns (dimensions, {column or field name: dimension name}) -- the second value is what primary-key resolution matches against. Fields carrying a `geo` stash are re-merged into the single Cube dimension they were split from. + Dimension names come from `dim_names` (see `_resolve_dimension_names`) rather + than being sanitized again here. """ ds_name = ds["name"] dimensions = [] covered = {} - taken = set() geo_parts = {} for field in (ds.get("fields") or []): fname = require_str(field, "name", f"dataset '{ds_name}': field") @@ -326,8 +362,7 @@ def _build_dimensions(ds, cname, members, dialect, issues): dimensions.append(None) # placeholder, filled in below continue - dname = sanitize_name(fname, f"dataset '{ds_name}': field", taken) - taken.add(dname.lower()) + dname = dim_names[fname] expr = pick_expression(field.get("expression"), dialect) if expr is None: issues.add(IssueType.NO_USABLE_DIALECT, f"{ds_name}.{fname}", @@ -339,7 +374,8 @@ def _build_dimensions(ds, cname, members, dialect, issues): # The exact Cube spelling a prior import saw. dim["sql"] = stash["sql"] else: - dim["sql"] = ossie_expr_to_cube_sql(expr, cname, members, ()) + dim["sql"] = ossie_expr_to_cube_sql( + expr, cname, set(dim_names.values()), ()) dim["type"] = _dimension_type(field, stash, f"{ds_name}.{fname}", issues) if field.get("label"): dim["title"] = field["label"] diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 87099837..331ce2ff 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -218,6 +218,31 @@ def test_field_name_is_sanitized_and_collisions_are_rejected(): convert_ossie_to_cube(_ossie(ds)) +def test_field_collision_is_rejected_before_any_metric_is_placed(): + """Dimension names are resolved once, up front. Resolving them per stage let a + collision go undetected while measures were being placed -- so the member set + that decides `{CUBE.member}` vs `{CUBE}.column` could be silently short a name, + and the error surfaced later and less clearly.""" + ds = ( + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: Order Status\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: status\n" + " - name: order status\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: status2\n" + ) + metrics = _metric("m", "SUM(orders.amount)") + with pytest.raises(ConversionError, match="collides"): + convert_ossie_to_cube(_ossie(ds, metrics=metrics)) + + def test_missing_dialect_drops_the_field_with_an_issue(): ds = ( " - name: orders\n" diff --git a/converters/cube/tests/test_roundtrip_properties.py b/converters/cube/tests/test_roundtrip_properties.py index 0a7e4106..3137245f 100644 --- a/converters/cube/tests/test_roundtrip_properties.py +++ b/converters/cube/tests/test_roundtrip_properties.py @@ -52,7 +52,13 @@ def __init__(self, data): self.data = data def chance(self, p=0.5): - return self.data.draw(st.booleans()) + # `st.booleans()` is unweighted, so it would ignore `p` and explore a + # different distribution than RandomRnd -- defeating the point of the + # two drivers sharing one generator. Drawn so the minimal value (0) + # means False, which shrinks toward the smallest model rather than the + # largest. + return self.data.draw( + st.integers(min_value=0, max_value=99)) >= 100 - round(p * 100) def count(self, lo, hi): return self.data.draw(st.integers(min_value=lo, max_value=hi)) From d676e76658acf022e9e71e33e1aff78f1e2c9319 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 12:26:05 +0500 Subject: [PATCH 10/46] Inline a split geo dimension's SQL where its halves are referenced A Cube `type: geo` dimension holds two SQL expressions where an Ossie field holds one, so import splits it into `_latitude` / `_longitude`. Export merges them back, so the round trip was already exact. But those half names exist only in Ossie. Cube has neither a column nor a member called `home_latitude` -- the halves merge into `home` -- so a metric or field expression referencing one had nothing valid to emit: AVG(users.home_latitude) -> sql: '{CUBE}.home_latitude' which names a column that does not exist (the column is `lat`). The member-reference form would have been just as wrong, failing at Cube compile time instead of in the database. The half's real SQL is already in the stash, so a reference to one is now replaced by that SQL: AVG(users.home_latitude) -> AVG({CUBE}.lat) AVG(users.home_latitude) - MIN(orders.amt) -> AVG({users}.lat) - MIN({CUBE.amt}) `{CUBE}` means "the cube this is declared on", so an inlined snippet is requalified to name its original cube when it crosses into another cube's SQL -- otherwise it would silently rebind to the wrong cube. One normalization follows and is documented: after a round trip such a metric names the column the half actually reads (`users.lat`) rather than the Ossie-only field name. Same reference, and the only form Cube can express. 234 tests. Co-Authored-By: Claude Opus 5 --- converters/cube/README.md | 29 ++++++- converters/cube/src/ossie_cube/_common.py | 31 ++++++- converters/cube/src/ossie_cube/osi_to_cube.py | 54 +++++++----- converters/cube/tests/test_edge_cases.py | 86 +++++++++++++++++++ 4 files changed, 175 insertions(+), 25 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index 118f2884..94572ddb 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -129,7 +129,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | `field.dimension.is_time` | `type: time` | Import sets `is_time: true` for a time dimension. | | `field.label` / `description` | dimension `title` / `description` | | | `field.ai_context.instructions` | dimension `meta.ai_context` | Cube's documented AI-only context field. | -| — | `type: geo` dimension | An Ossie field holds one expression and a geo dimension has two, so it **splits** into `_latitude` / `_longitude` (`Float`). Reconstruction data rides on the latitude half. | +| — | `type: geo` dimension | An Ossie field holds one expression and a geo dimension has two, so it **splits** into `_latitude` / `_longitude` (`Float`). Reconstruction data rides on the latitude half. See [Geo dimensions](#geo-dimensions). | | relationship | `joins[]` on a cube | `many_to_one` on cube A -> `from: A`(many), `to: B`(one). `one_to_many` is flipped so Ossie's `from` is the many side; the declared side and type are stashed so export restores the original. | | `from_columns` / `to_columns` | join `sql` | Only an AND-chain of equalities between two member references maps. Anything else (non-equi, range, literal, third cube) is preserved verbatim in the stash. | | metric | `measures[]` on the cube its expression references | Import hoists cube-scoped measures to the model level, qualifying a colliding name as `__` and stashing the original name and owning cube. | @@ -195,6 +195,31 @@ different number. > `non_additive_dimension` is the nearest precedent, and this repo's dbt converter > already loses the same information. Worth raising on `dev@`. +## Geo dimensions + +A Cube `type: geo` dimension carries two SQL expressions where an Ossie field carries one, so it splits on import: + +```yaml +# Cube # Ossie +- name: home - name: home_latitude (expression: lat) + type: geo - name: home_longitude (expression: lon) + latitude: { sql: "{CUBE}.lat" } + longitude: { sql: "{CUBE}.lon" } +``` + +Export merges the halves back into the single geo dimension, so the round trip is exact. + +The half names exist **only in Ossie** — Cube has neither a column nor a member called `home_latitude`. So when an Ossie metric or field expression references a half, export substitutes the half's own SQL rather than emitting a reference Cube cannot resolve: + +``` +AVG(users.home_latitude) -> sql: AVG({CUBE}.lat) +AVG(users.home_latitude) - MIN(orders.amt) -> sql: AVG({users}.lat) - MIN({CUBE.amt}) +``` + +`{CUBE}` means "the cube this is declared on", so an inlined snippet is requalified to name its original cube when it crosses into another cube's SQL. + +One documented normalization follows: after a round trip such a metric names the column the half actually reads (`users.lat`) rather than the Ossie-only field name (`users.home_latitude`). Same reference, and it is the form Cube can express. + ## Conversion issues `convert_cube_to_ossie` returns `(yaml, IssueLog)`. Each issue carries a type, the @@ -250,7 +275,7 @@ uv sync uv run pytest ``` -226 tests at 96% line coverage: example-based unit tests per direction, CLI +234 tests at 96% line coverage: example-based unit tests per direction, CLI behavior tests, fixture round-trip tests (including the [TPC-DS model](../../examples/tpcds_semantic_model.yaml) the converter guide asks for as a baseline), core-spec JSON Schema validation of every emitted Ossie diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index 806d0a4d..5cb4ab28 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -385,7 +385,22 @@ def repl(m): return out, changed -def ossie_expr_to_cube_sql(expr, own_cube, own_members=(), cube_names=()): +def requalify_self_refs(sql, cube_name): + """Rewrite `{CUBE}` / `{TABLE}` in a Cube SQL snippet to name `cube_name`. + + Needed when a snippet written for one cube is inlined into another cube's SQL: + `{CUBE}` means "the cube this is declared on", so it changes meaning on the + move, while `{orders}.col` is explicit and does not. + """ + return re.sub( + r"\$?\{\s*(?:CUBE|TABLE)\s*(\.\s*[A-Za-z_][A-Za-z0-9_]*\s*)?\}", + lambda m: "{" + cube_name + (m.group(1).strip() if m.group(1) else "") + "}", + str(sql), + ) + + +def ossie_expr_to_cube_sql(expr, own_cube, own_members=(), cube_names=(), + inline_sql=None): """Rewrite an Ossie expression into Cube member-reference form. Only *dotted* `cube.name` references are rewritten -- a bare identifier stays @@ -403,13 +418,27 @@ def ossie_expr_to_cube_sql(expr, own_cube, own_members=(), cube_names=()): The own cube is always referenced as `{CUBE}` rather than by name, so the model keeps working when the cube is extended. Literal braces in the incoming expression are escaped. + + `inline_sql` maps `{cube: {field: cube_sql}}` for Ossie fields that have no + addressable Cube counterpart, and whose SQL therefore has to be substituted + inline. The case that needs it is a split `geo` dimension: `location_latitude` + exists only in Ossie -- Cube has neither a column nor a member by that name -- + so a reference to it becomes the half's own SQL (`{CUBE}.lat`), requalified when + it crosses cubes. """ escaped = str(expr).replace("{", "\\{").replace("}", "\\}") known = set(cube_names) members = set(own_members) + inline = inline_sql or {} def repl(m): head, name = m.group(1), m.group(2) + substitute = (inline.get(head) or {}).get(name) + if substitute is not None: + # Already-Cube SQL, so it bypasses the escaping above; `{CUBE}` inside + # it means `head`, which only stays true while head is the own cube. + return (str(substitute) if head == own_cube + else requalify_self_refs(substitute, head)) if head == own_cube: return "{CUBE." + name + "}" if name in members else "{CUBE}." + name if head in known: diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 8fc1731d..b87995a3 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -138,18 +138,19 @@ def _convert_model(model, dialect, base_cube, issues): # lands. dim_names_by_cube = {} members_by_cube = {} + inline_sql_by_cube = {} pk_by_cube = {} for ds_name, ds in datasets.items(): cname = cube_names[ds_name] - dim_names_by_cube[cname] = _resolve_dimension_names( - ds, f"Model '{name}': dataset '{ds_name}'") + dim_names_by_cube[cname], inline_sql_by_cube[cname] = ( + _resolve_dimension_names(ds, f"Model '{name}': dataset '{ds_name}'")) members_by_cube[cname] = set(dim_names_by_cube[cname].values()) pk_by_cube[cname] = [str(c) for c in (ds.get("primary_key") or [])] joins_by_cube = _build_joins(relationships, cube_names, issues) measures_by_cube = _build_measures( - model, cube_names, members_by_cube, pk_by_cube, datasets, relationships, - base_cube, dialect, issues) + model, cube_names, members_by_cube, inline_sql_by_cube, pk_by_cube, + datasets, relationships, base_cube, dialect, issues) # Cubes, grouped by the file they belong in: several datasets can share one # stashed original path, in which case they go back into the same file. @@ -158,8 +159,8 @@ def _convert_model(model, dialect, base_cube, issues): for ds_name, ds in datasets.items(): cname = cube_names[ds_name] cube = _build_cube(ds, cname, dim_names_by_cube[cname], - joins_by_cube.get(cname), measures_by_cube.get(cname), - dialect, issues) + inline_sql_by_cube[cname], joins_by_cube.get(cname), + measures_by_cube.get(cname), dialect, issues) path = stashed_paths.get(cname) or cube_file(cname) files_content.setdefault(path, {}).setdefault("cubes", []).append(cube) @@ -238,7 +239,8 @@ def _ordered(obj, order): # --- cubes ---------------------------------------------------------------------- -def _build_cube(ds, cname, dim_names, joins, measures, dialect, issues): +def _build_cube(ds, cname, dim_names, inline_sql, joins, measures, dialect, + issues): ds_name = ds["name"] scope = f"dataset '{ds_name}'" stash = read_stash(ds) @@ -268,7 +270,7 @@ def _build_cube(ds, cname, dim_names, joins, measures, dialect, issues): "so this cube-level value has no effect in Cube") dimensions, covered = _build_dimensions( - ds, cname, dim_names, dialect, issues) + ds, cname, dim_names, inline_sql, dialect, issues) # A primary-key column no field covers still has to exist as a dimension for # Cube to join or roll up the cube. pk_names = [] @@ -316,11 +318,14 @@ def _resolve_dimension_names(ds, scope): - A collision is an error, not a silent merge. Sanitizing with a fresh `taken` set per field would hide one. - The two halves of a split `geo` dimension map back to the *single* dimension - they merge into, so `location_latitude` resolves to `location`. Treating the - halves as members of their own would let a metric emit a `{CUBE.…}` reference - to a dimension the exported cube does not have. + they merge into, so `location_latitude` resolves to `location`. + + Returns (names, inline_sql). `inline_sql` holds the fields whose name exists + only in Ossie -- the two halves of a split geo dimension -- mapped to the Cube + SQL a reference to them must be replaced by, since Cube has neither a column nor + a member of that name. """ - names = {} + names, inline_sql = {}, {} taken = set() for field in (ds.get("fields") or []): fname = require_str(field, "name", f"{scope}: field") @@ -328,15 +333,16 @@ def _resolve_dimension_names(ds, scope): if geo: base = geo["of"] names[fname] = base + inline_sql[fname] = geo["sql"] taken.add(base.lower()) continue dname = sanitize_name(fname, f"{scope}: field", taken) taken.add(dname.lower()) names[fname] = dname - return names + return names, inline_sql -def _build_dimensions(ds, cname, dim_names, dialect, issues): +def _build_dimensions(ds, cname, dim_names, inline_sql, dialect, issues): """Build a cube's dimensions from an Ossie dataset's fields. Returns (dimensions, {column or field name: dimension name}) -- the second @@ -375,7 +381,8 @@ def _build_dimensions(ds, cname, dim_names, dialect, issues): dim["sql"] = stash["sql"] else: dim["sql"] = ossie_expr_to_cube_sql( - expr, cname, set(dim_names.values()), ()) + expr, cname, set(dim_names.values()), (), + inline_sql={cname: inline_sql}) dim["type"] = _dimension_type(field, stash, f"{ds_name}.{fname}", issues) if field.get("label"): dim["title"] = field["label"] @@ -501,8 +508,9 @@ def _build_joins(relationships, cube_names, issues): # --- measures ------------------------------------------------------------------- -def _build_measures(model, cube_names, members_by_cube, pk_by_cube, datasets, - relationships, base_cube, dialect, issues): +def _build_measures(model, cube_names, members_by_cube, inline_sql_by_cube, + pk_by_cube, datasets, relationships, base_cube, dialect, + issues): """Group Ossie metrics into per-cube `measures` lists.""" name = model.get("name", "") sanitized = set(cube_names.values()) @@ -548,7 +556,8 @@ def resolve_base(): next(iter(referenced)) if len(referenced) == 1 else resolve_base()) measure = _measure_from_expression( expr, target, mname, stash, members_by_cube.get(target, set()), - pk_by_cube.get(target, []), sanitized, scope, issues) + inline_sql_by_cube, pk_by_cube.get(target, []), sanitized, scope, + issues) _apply_measure_metadata(metric, measure, stash) _place(measures_by_cube, target, measure, name) return measures_by_cube @@ -563,8 +572,8 @@ def _place(measures_by_cube, target, measure, model_name): bucket.append(measure) -def _measure_from_expression(expr, target, mname, stash, members, primary_key, - sanitized, scope, issues): +def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_cube, + primary_key, sanitized, scope, issues): """Turn an Ossie metric expression back into a structured Cube measure. `COUNT(DISTINCT )` is Cube's bare `type: count` -- @@ -590,14 +599,15 @@ def _measure_from_expression(expr, target, mname, stash, members, primary_key, agg = OSSIE_FUNC_TO_AGG.get(func) or ("count" if func == "COUNT" else None) if agg is not None: measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( - inner, target, members, sanitized) + inner, target, members, sanitized, + inline_sql=inline_sql_by_cube) measure["type"] = agg return measure # A ratio, a window expression, or a multi-dataset aggregate: Cube expresses # these as a calculated measure whose sql carries the aggregation. measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( - expr, target, members, sanitized) + expr, target, members, sanitized, inline_sql=inline_sql_by_cube) measure["type"] = "number" if len({ ref for ref in re.findall( diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 22b650dd..d4c9007d 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -603,6 +603,92 @@ def test_geo_dimension_extras_survive_the_split_and_merge(): assert issues.of_type(IssueType.GEO_DIMENSION_SPLIT) +_GEO_MODEL = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: users\n" + " source: public.users\n" + " fields:\n" + " - name: home_latitude\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: lat\n" + " datatype: Float\n" + " custom_extensions:\n" + " - vendor_name: CUBE\n" + " data: '{\"_v\": 1, \"geo\": {\"of\": \"home\", \"part\": \"latitude\"," + " \"sql\": \"{CUBE}.lat\"}}'\n" + " - name: home_longitude\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: lon\n" + " datatype: Float\n" + " custom_extensions:\n" + " - vendor_name: CUBE\n" + " data: '{\"_v\": 1, \"geo\": {\"of\": \"home\", \"part\": \"longitude\"," + " \"sql\": \"{CUBE}.lon\"}}'\n" + " metrics:\n" + " - name: avg_lat\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: AVG(users.home_latitude)\n" +) + + +def test_a_metric_referencing_a_geo_half_inlines_its_sql(): + """A split geo half's name exists only in Ossie: Cube has neither a column nor a + member called `home_latitude`, since the halves merge into the `home` dimension. + So a reference to one is replaced by the half's own SQL, which is valid Cube.""" + files, _ = convert_ossie_to_cube(_GEO_MODEL) + cube = parse(files["model/cubes/users.yml"])["cubes"][0] + assert cube["measures"] == [ + {"name": "avg_lat", "sql": "{CUBE}.lat", "type": "avg"}] + # And the dimension itself still merges back to a single geo member. + assert cube["dimensions"] == [{ + "name": "home", "type": "geo", + "latitude": {"sql": "{CUBE}.lat"}, + "longitude": {"sql": "{CUBE}.lon"}}] + + +def test_a_geo_half_reference_is_requalified_when_it_crosses_cubes(): + """`{CUBE}` means "the cube this is declared on", so inlining a snippet into + another cube's SQL has to name the original cube explicitly.""" + model = _GEO_MODEL.replace( + " - name: users\n", " - name: orders\n source: public.orders\n" + " fields:\n" + " - name: amount\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: amount\n" + " datatype: Decimal\n" + " - name: users\n", 1 + ).replace(" expression: AVG(users.home_latitude)\n", + " expression: AVG(users.home_latitude) - MIN(orders.amount)\n") + files, _ = convert_ossie_to_cube(model, base_cube="orders") + measures = parse(files["model/cubes/orders.yml"])["cubes"][0]["measures"] + # `{users}.lat` names the cube explicitly, since `{CUBE}` here would mean + # `orders`. `{CUBE.amount}` stays a member reference because `amount` is a + # declared field of the cube the measure lands on. + assert measures[0]["sql"] == "AVG({users}.lat) - MIN({CUBE.amount})" + assert measures[0]["type"] == "number" + + +def test_geo_half_references_normalize_to_the_underlying_column(): + """Documented normalization: after a round trip the metric names the column the + geo half actually reads rather than the Ossie-only field name. Semantically the + same reference, and it is what Cube can express.""" + files, _ = convert_ossie_to_cube(_GEO_MODEL) + ossie2, _ = convert_cube_to_ossie(files) + metric = model_of(ossie2)["metrics"][0] + assert expr_of(metric) == "AVG(users.lat)" + + def test_geo_dimension_missing_a_half_is_rejected(): files = _files(users=( "cubes:\n" From 210f6da6da7a7bf26e97b27359a720273954b9db Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 12:46:02 +0500 Subject: [PATCH 11/46] Use a deque for the generated view's BFS; fix a licence/license typo Both from the Copilot review on #289. The BFS popped from the front of a list, which is O(n) per pop; a deque makes it O(1). Semantic models are small enough that this was never going to matter in practice, but the deque is also the more idiomatic form. Co-Authored-By: Claude Opus 5 --- converters/cube/src/ossie_cube/osi_to_cube.py | 5 +++-- converters/cube/tests/test_roundtrip.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index b87995a3..69d700c9 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -32,6 +32,7 @@ """ import re +from collections import deque from ._common import ( DATATYPE_TO_DIM_TYPE, @@ -710,9 +711,9 @@ def _view_cubes(cube_names, relationships, base): entries = [{"join_path": base, "includes": "*"}] paths = {base: base} - queue = [base] + queue = deque([base]) while queue: - current = queue.pop(0) + current = queue.popleft() for neighbor in adjacency.get(current, []): if neighbor in paths: continue diff --git a/converters/cube/tests/test_roundtrip.py b/converters/cube/tests/test_roundtrip.py index d5914c8a..8416088c 100644 --- a/converters/cube/tests/test_roundtrip.py +++ b/converters/cube/tests/test_roundtrip.py @@ -39,7 +39,7 @@ def test_cube_roundtrip_is_lossless(fixture): """Cube -> Ossie -> Cube reproduces the original model, structurally. Compared parsed rather than byte-for-byte: YAML comments (including the - licence headers on the fixtures) are not part of the data model, and key order + license headers on the fixtures) are not part of the data model, and key order within a mapping is not semantic. """ files = load_fixture_dir(fixture) From d9c853d21988cdabc6180b3098fbd38c4a4d208a Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 14:32:50 +0500 Subject: [PATCH 12/46] Refuse rather than drop unparkable extensions; separate drops from parks Both from the Copilot review on #289. Model-level foreign-vendor custom_extensions ride on the view that represents the model. When the source Cube model had several views and none was chosen, there is no such view -- and export silently dropped them. Reachable in practice: import a multi-view Cube model, add a SNOWFLAKE extension to the Ossie model, export, and it is gone. Confirmed by reproducing it. Now refused, with the fix in the message (re-import with `--view`). Parking on an arbitrary view was considered and rejected: only the mapped view's parked extensions are read back on import, so it would look lossless while still losing them. The review also noted the issue type contradicted its own message -- PARKED_IN_META for something reported as "dropped". That was true in two places, not one, and it matters: the README defines PARKED_IN_META as preserved-but-invisible-to-Cube, so a pipeline gating on issue types would have concluded the data survived. Adds DROPPED_NO_CUBE_EQUIVALENT for values that genuinely cannot be preserved, and uses it for relationship ai_context -- a Cube join entry takes only name/sql/relationship, with no `meta` field, making it the one construct with nowhere to go. Also drops the hard-coded test count from the README. It had already drifted out of sync with the PR description, which is the reviewer's point: the number carries no information a reader needs, while the description of what the suite covers does. 236 tests, 97% line coverage. Co-Authored-By: Claude Opus 5 --- converters/cube/README.md | 9 ++++-- .../cube/src/ossie_cube/converter_issues.py | 9 +++++- converters/cube/src/ossie_cube/osi_to_cube.py | 30 ++++++++++++++----- converters/cube/tests/test_edge_cases.py | 30 +++++++++++++++++++ converters/cube/tests/test_osi_to_cube.py | 12 ++++++-- 5 files changed, 76 insertions(+), 14 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index 94572ddb..6641a1c5 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -233,7 +233,8 @@ element it concerns, and a detail string. | `GEO_DIMENSION_SPLIT` | A `type: geo` dimension became two Ossie fields | | `TEMPLATED_FILE_SKIPPED` | Jinja templating anywhere in a file, or a `.js`/`.ts` model file. Detected per file, as Cube's own tooling does, so the file is preserved whole rather than half-converted | | `NO_USABLE_DIALECT` | Export: no `ANSI_SQL` or preferred-dialect expression | -| `PARKED_IN_META` | An element preserved in the stash with no native mapping | +| `PARKED_IN_META` | An element with no native mapping, preserved in the stash or under `meta.ossie` — invisible to Cube but intact through a round trip | +| `DROPPED_NO_CUBE_EQUIVALENT` | A value Cube has nowhere to hold *and* that cannot be parked, so it is genuinely gone. Currently only relationship `ai_context`, since a Cube join entry has no `meta` field. Kept distinct from `PARKED_IN_META` so a caller can tell real loss from "preserved but unreadable by Cube" | ## Requirements @@ -249,6 +250,10 @@ invalid) when an input breaks one of these: - two cubes, two views, or two derived metric names collide; - a dimension has an unknown `type`, or a `geo` dimension is missing `latitude.sql` / `longitude.sql`; +- the model carries foreign-vendor `custom_extensions` but no view is mapped, so + there is nowhere to park them (re-import with `--view `); model-level + metadata rides on the view representing the model, and picking one arbitrarily + would not survive a re-import; - there are no convertible cubes at all; the input YAML is malformed. ## Notes and limitations @@ -275,7 +280,7 @@ uv sync uv run pytest ``` -234 tests at 96% line coverage: example-based unit tests per direction, CLI +Example-based unit tests per direction, CLI behavior tests, fixture round-trip tests (including the [TPC-DS model](../../examples/tpcds_semantic_model.yaml) the converter guide asks for as a baseline), core-spec JSON Schema validation of every emitted Ossie diff --git a/converters/cube/src/ossie_cube/converter_issues.py b/converters/cube/src/ossie_cube/converter_issues.py index de29c586..0c79610d 100644 --- a/converters/cube/src/ossie_cube/converter_issues.py +++ b/converters/cube/src/ossie_cube/converter_issues.py @@ -62,9 +62,16 @@ class IssueType(Enum): # An Ossie field or metric with no usable expression dialect (export). NO_USABLE_DIALECT = "NO_USABLE_DIALECT" - # An Ossie construct Cube has no slot for, parked under `meta.ossie`. + # An Ossie construct Cube has no slot for, parked under `meta.ossie` -- so the + # value survives the round trip even though Cube itself cannot read it. PARKED_IN_META = "PARKED_IN_META" + # A value Cube has nowhere to hold *and* that cannot be parked, so it is gone + # from the output. Distinct from PARKED_IN_META on purpose: a caller gating on + # issue types has to be able to tell "preserved but invisible to Cube" from + # "actually lost". + DROPPED_NO_CUBE_EQUIVALENT = "DROPPED_NO_CUBE_EQUIVALENT" + @dataclass(frozen=True) class ConverterIssue: diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 69d700c9..16bed836 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -166,7 +166,7 @@ def _convert_model(model, dialect, base_cube, issues): files_content.setdefault(path, {}).setdefault("cubes", []).append(cube) for vpath, view in _build_views(model, model_stash, cube_names, relationships, - datasets, base_cube, issues).items(): + datasets, base_cube).items(): files_content.setdefault(vpath, {}).setdefault("views", []).append(view) files = {path: dump_yaml(content) for path, content in files_content.items()} @@ -499,9 +499,12 @@ def _build_joins(relationships, cube_names, issues): if key not in ("declared_on", "relationship", "sql"): join[key] = value if rel.get("ai_context"): - issues.add(IssueType.PARKED_IN_META, f"relationship '{rname}'", - "Cube joins carry no metadata, so relationship ai_context " - "has no home; dropped") + # A Cube join entry takes only name/sql/relationship -- no `meta` -- so + # unlike every other level there is nowhere to park this. + issues.add(IssueType.DROPPED_NO_CUBE_EQUIVALENT, + f"relationship '{rname}'", + "a Cube join carries no metadata field, so relationship " + "ai_context has nowhere to go and is dropped") joins_by_cube.setdefault(own, []).append( _ordered(join, ["name", "sql", "relationship"])) return joins_by_cube @@ -651,7 +654,7 @@ def _balanced(s): # --- views ---------------------------------------------------------------------- def _build_views(model, model_stash, cube_names, relationships, datasets, - base_cube, issues): + base_cube): """Return {file path: view dict}. Stashed views restore verbatim, with the natively mapped description and AI @@ -671,9 +674,20 @@ def _build_views(model, model_stash, cube_names, relationships, datasets, mapped = model_stash.get("mapped_view") paths = model_stash.get("view_files") or {} if foreign and mapped is None: - issues.add(IssueType.PARKED_IN_META, "model", - "no mapped view to park foreign-vendor custom_extensions on; " - "they have no Cube home and are dropped") + # The model's own metadata rides on the view that represents it, and + # there isn't one: the source Cube model had several views and none was + # chosen. Dropping the extensions would be silent data loss, and + # picking a view arbitrarily would not survive a re-import (only the + # mapped view's parked extensions are restored). So this is refused + # with the fix in the message. + vendors = ", ".join( + sorted({str(e.get("vendor_name")) for e in foreign})) + raise ConversionError( + f"Model carries custom_extensions for {vendors}, which have no Cube " + f"field and ride on the view representing the model -- but no view " + f"is mapped, so there is nowhere to put them without losing them. " + f"Re-import naming the view the model maps to (`--view `), or " + f"remove the foreign-vendor extensions.") for vname, view in (model_stash["views"] or {}).items(): view = dict(view) if vname == mapped: diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index d4c9007d..009454f4 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -34,6 +34,7 @@ convert_cube_to_ossie, convert_ossie_to_cube, ) +from ossie_cube._common import dump_yaml def _files(**named): @@ -772,6 +773,35 @@ def test_choosing_a_view_maps_its_metadata_onto_the_model(): assert set(stash_of(model)["views"]) == {"a", "b"} +def test_foreign_extensions_with_no_mapped_view_are_refused_not_dropped(): + """Model-level foreign-vendor extensions ride on the view that represents the + model. With several views and none mapped there is no such view, and picking one + arbitrarily would not survive a re-import -- only the mapped view's parked + extensions are read back. So this is refused rather than silently losing them.""" + ossie, _ = convert_cube_to_ossie(_TWO_VIEWS) + doc = parse(ossie) + doc["semantic_model"][0].setdefault("custom_extensions", []).append( + {"vendor_name": "SNOWFLAKE", "data": '{"warehouse": "ANALYTICS_WH"}'}) + with pytest.raises(ConversionError, match="SNOWFLAKE"): + convert_ossie_to_cube(dump_yaml(doc)) + + +def test_foreign_extensions_survive_once_a_view_is_mapped(): + """The fix the error message points at: choose the view the model maps to, and + the extensions have a home again.""" + ossie, _ = convert_cube_to_ossie(_TWO_VIEWS, view="b") + doc = parse(ossie) + doc["semantic_model"][0].setdefault("custom_extensions", []).append( + {"vendor_name": "SNOWFLAKE", "data": '{"warehouse": "ANALYTICS_WH"}'}) + files, _ = convert_ossie_to_cube(dump_yaml(doc)) + parked = parse(files["model/views/b.yml"])["views"][0]["meta"]["ossie"] + assert parked["custom_extensions"][0]["vendor_name"] == "SNOWFLAKE" + # And they come back as Ossie extensions, not just stashed text. + ossie2, _ = convert_cube_to_ossie(files, view="b") + vendors = {e["vendor_name"] for e in model_of(ossie2)["custom_extensions"]} + assert "SNOWFLAKE" in vendors + + def test_both_views_are_restored_on_export(): ossie, _ = convert_cube_to_ossie(_TWO_VIEWS, view="b") back, _ = convert_ossie_to_cube(ossie) diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 331ce2ff..6372b5bf 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -319,11 +319,17 @@ def test_composite_relationship_becomes_an_and_chain(): "{CUBE}.user_id = {users.id} AND {CUBE}.region = {users.region}") -def test_relationship_ai_context_has_no_cube_home(): +def test_relationship_ai_context_is_reported_as_dropped_not_parked(): + """A Cube join entry takes only name/sql/relationship -- no `meta` -- so this is + one of the few things that genuinely cannot be preserved. It is reported under + DROPPED_NO_CUBE_EQUIVALENT rather than PARKED_IN_META, so a caller gating on + issue types can tell real loss from "preserved but invisible to Cube".""" rel = _REL + " ai_context:\n instructions: Join carefully.\n" _, issues = convert_ossie_to_cube(_ossie(_TWO_DATASETS, rel)) - assert any("ai_context" in i.detail for i in issues.of_type( - IssueType.PARKED_IN_META)) + dropped = issues.of_type(IssueType.DROPPED_NO_CUBE_EQUIVALENT) + assert [i.element_name for i in dropped] == ["relationship 'orders_to_users'"] + assert "ai_context" in dropped[0].detail + assert not issues.of_type(IssueType.PARKED_IN_META) # --- metrics -------------------------------------------------------------------- From 0499913c99cb16ade3ca4b86602777ecdc25ebb0 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 15:31:17 +0500 Subject: [PATCH 13/46] Assemble geo dimensions by name, not by a mid-loop list index From the Copilot review on #289, which found that the placeholder holding a geo dimension's position was only reserved when the half encountered first happened to be `latitude`. With `longitude` first, the recorded index pointed at whatever real dimension had already been appended, and `dimensions[index] = dim` overwrote it. Reproduced: a `city` dimension between the two halves disappeared from the output entirely. Rather than reserve the placeholder earlier, the index arithmetic is gone. Dimensions are now built into a dict keyed by target name, with order taken from each name's first appearance -- which is well defined however the two halves are arranged, adjacent or not, in either order. Probing around the fix turned up two more silent-corruption paths in the same code, both order-dependent: - A geo base colliding with an ordinary field of the same name emitted two dimensions called `home` (invalid Cube) when the ordinary field came first, but was correctly rejected when it came second. Now checked during name resolution, so order does not decide. - Two fields both claiming the same half silently discarded one. Now rejected. Also validates the geo `part` and `of` values, and moves the missing-half check into name resolution so every geo problem is caught in one place before anything is built. 241 tests, 97% line coverage. Co-Authored-By: Claude Opus 5 --- converters/cube/src/ossie_cube/osi_to_cube.py | 69 +++++++++++---- converters/cube/tests/test_edge_cases.py | 84 +++++++++++++++++++ 2 files changed, 135 insertions(+), 18 deletions(-) diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 16bed836..70f0ed1d 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -328,18 +328,46 @@ def _resolve_dimension_names(ds, scope): """ names, inline_sql = {}, {} taken = set() + geo_halves = {} # base -> {part: field name}, for validating the pair for field in (ds.get("fields") or []): fname = require_str(field, "name", f"{scope}: field") geo = read_stash(field).get("geo") if geo: - base = geo["of"] + base, part = geo.get("of"), geo.get("part") + if part not in ("latitude", "longitude"): + raise ConversionError( + f"{scope}: field '{fname}' has a geo part '{part}'; expected " + f"'latitude' or 'longitude'") + if not base: + raise ConversionError( + f"{scope}: field '{fname}' has a geo stash with no 'of'") + seen = geo_halves.setdefault(base, {}) + if part in seen: + raise ConversionError( + f"{scope}: fields '{seen[part]}' and '{fname}' both claim the " + f"{part} of geo dimension '{base}'") + if not seen and base.lower() in taken: + # The base is the name of the merged Cube dimension, so it cannot + # also be an ordinary dimension -- that would emit two members of + # the same name. Order must not decide whether this is caught, so + # it is checked here rather than left to sanitize_name. + raise ConversionError( + f"{scope}: geo dimension '{base}' collides with another field " + f"of that name; rename one in the Ossie model.") + seen[part] = fname + taken.add(base.lower()) names[fname] = base inline_sql[fname] = geo["sql"] - taken.add(base.lower()) continue dname = sanitize_name(fname, f"{scope}: field", taken) taken.add(dname.lower()) names[fname] = dname + for base, seen in geo_halves.items(): + missing = {"latitude", "longitude"} - set(seen) + if missing: + raise ConversionError( + f"{scope}: geo dimension '{base}' is missing its " + f"{' and '.join(sorted(missing))} half") return names, inline_sql @@ -353,23 +381,27 @@ def _build_dimensions(ds, cname, dim_names, inline_sql, dialect, issues): than being sanitized again here. """ ds_name = ds["name"] - dimensions = [] covered = {} - geo_parts = {} + # Built by target dimension name rather than by list position: a geo dimension + # is assembled from two fields that may appear in either order and need not be + # adjacent, so an insertion index computed mid-loop is not a safe way to hold + # its place. `order` records first appearance of each target name, which is + # well defined however the halves are arranged. + order, built, geo_parts = [], {}, {} for field in (ds.get("fields") or []): fname = require_str(field, "name", f"dataset '{ds_name}': field") stash = read_stash(field) + dname = dim_names[fname] + if dname not in order: + order.append(dname) if "geo" in stash: geo = stash["geo"] - slot = geo_parts.setdefault(geo["of"], {"index": len(dimensions)}) + slot = geo_parts.setdefault(dname, {}) slot[geo["part"]] = geo["sql"] if "host" in geo: slot["host"] = geo["host"] - if geo["part"] == "latitude": - dimensions.append(None) # placeholder, filled in below continue - dname = dim_names[fname] expr = pick_expression(field.get("expression"), dialect) if expr is None: issues.add(IssueType.NO_USABLE_DIALECT, f"{ds_name}.{fname}", @@ -400,24 +432,25 @@ def _build_dimensions(ds, cname, dim_names, inline_sql, dialect, issues): for key, value in extras.items(): dim[key] = value - dimensions.append(dim) + built[dname] = dim covered[dname] = dname if is_simple_identifier(expr): covered[expr.strip()] = dname - for of, slot in geo_parts.items(): - if "latitude" not in slot or "longitude" not in slot: - raise ConversionError( - f"dataset '{ds_name}': geo dimension '{of}' is missing its " - f"{'longitude' if 'latitude' in slot else 'latitude'} half") - dim = {"name": of, "type": "geo", + # Both halves are guaranteed present by _resolve_dimension_names, which + # validates the pair before anything is built. + for base, slot in geo_parts.items(): + dim = {"name": base, "type": "geo", "latitude": {"sql": slot["latitude"]}, "longitude": {"sql": slot["longitude"]}} for key, value in (slot.get("host") or {}).items(): dim[key] = value - dimensions[slot["index"]] = dim - covered[of] = of - return [d for d in dimensions if d is not None], covered + built[base] = dim + covered[base] = base + + # A name in `order` with nothing built is a field dropped for want of a usable + # dialect; it simply does not appear. + return [built[n] for n in order if n in built], covered def _dimension_type(field, stash, scope, issues): diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 009454f4..4b82a07b 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -690,6 +690,90 @@ def test_geo_half_references_normalize_to_the_underlying_column(): assert expr_of(metric) == "AVG(users.lat)" +def _geo_stash(part, of="home"): + return ('{"_v": 1, "geo": {"of": "' + of + '", "part": "' + part + + '", "sql": "{CUBE}.' + part[:3] + '"}}') + + +def _ossie_fields(*specs): + """Build an Ossie model from (field name, expression, geo part or None) specs.""" + out = ("version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: users\n" + " source: public.users\n" + " fields:\n") + for fname, expr, part in specs: + out += (f" - name: {fname}\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + f" expression: {expr}\n" + " datatype: String\n") + if part: + out += (" custom_extensions:\n" + " - vendor_name: CUBE\n" + f" data: '{_geo_stash(part)}'\n") + return out + + +def test_geo_halves_may_appear_in_any_order_without_clobbering_a_dimension(): + """The geo dimension is assembled from two fields that need not be adjacent and + may come in either order. Holding its place with a list index computed mid-loop + overwrote whatever real dimension already sat at that index -- here `city` + vanished entirely.""" + model = _ossie_fields( + ("home_longitude", "lon", "longitude"), + ("city", "city", None), + ("home_latitude", "lat", "latitude"), + ) + files, _ = convert_ossie_to_cube(model) + dims = parse(files["model/cubes/users.yml"])["cubes"][0]["dimensions"] + assert [d["name"] for d in dims] == ["home", "city"] + assert by_name(dims)["home"] == { + "name": "home", "type": "geo", + "latitude": {"sql": "{CUBE}.lat"}, + "longitude": {"sql": "{CUBE}.lon"}} + assert by_name(dims)["city"]["sql"] == "city" + + +def test_a_geo_base_colliding_with_a_field_is_rejected_in_either_order(): + """The base is the merged dimension's name, so it cannot also be an ordinary + dimension -- that would emit two members of the same name. Whether the ordinary + field comes first must not decide whether this is caught.""" + for specs in ( + (("home", "home", None), ("home_latitude", "lat", "latitude"), + ("home_longitude", "lon", "longitude")), + (("home_latitude", "lat", "latitude"), + ("home_longitude", "lon", "longitude"), ("home", "home", None)), + ): + with pytest.raises(ConversionError, match="collides"): + convert_ossie_to_cube(_ossie_fields(*specs)) + + +def test_two_fields_claiming_the_same_geo_half_are_rejected(): + model = _ossie_fields( + ("a_lat", "lat", "latitude"), + ("b_lat", "lat2", "latitude"), + ("home_longitude", "lon", "longitude"), + ) + with pytest.raises(ConversionError, match="both claim the latitude"): + convert_ossie_to_cube(model) + + +def test_a_geo_dimension_missing_a_half_is_rejected_on_export(): + model = _ossie_fields(("home_latitude", "lat", "latitude")) + with pytest.raises(ConversionError, match="missing its longitude half"): + convert_ossie_to_cube(model) + + +def test_an_unknown_geo_part_is_rejected(): + model = _ossie_fields(("home_altitude", "alt", "altitude")) + with pytest.raises(ConversionError, match="geo part 'altitude'"): + convert_ossie_to_cube(model) + + def test_geo_dimension_missing_a_half_is_rejected(): files = _files(users=( "cubes:\n" From 07e818647d01ab301508faaaa2c7705c754911df Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 15:54:50 +0500 Subject: [PATCH 14/46] Correct every mislabelled issue type, not just the flagged one From the Copilot review on #289: additional `semantic_model` entries were reported as PARKED_IN_META, but they are neither converted nor preserved anywhere -- a drop. That is the third instance of the same mislabelling, so rather than patch the flagged line I audited all seven export-side uses. Exactly one was a genuine park: unique_keys -> PARKED_IN_META (correct) extra semantic_model entries -> DROPPED (was parked) dimension.is_time role -> DROPPED (was parked) dimension.is_time opt-out -> DROPPED (was parked) synthesized primary-key dimension -> APPROXIMATED (was parked) no datatype -> Cube type 'string' -> APPROXIMATED (was parked) cross-dataset metric placement -> APPROXIMATED (was parked) The import-direction uses were all genuine parks and are unchanged. Adds APPROXIMATED for the middle case, which neither of the existing types described: nothing is lost and nothing is hidden, but Cube requires a value Ossie does not carry, so the converter chose one and the output asserts slightly more than the input did. Calling that "parked" was wrong in the same way as calling a drop "parked" -- nothing was parked. The point of keeping three types apart is that a caller gating on them can distinguish preserved-but-unreadable from actually-lost from emitted-with-a-guess. Two of the three could not be told apart before. 241 tests. Co-Authored-By: Claude Opus 5 --- converters/cube/README.md | 9 +++++++-- .../cube/src/ossie_cube/converter_issues.py | 7 +++++++ converters/cube/src/ossie_cube/osi_to_cube.py | 18 ++++++++++-------- converters/cube/tests/test_edge_cases.py | 4 +++- converters/cube/tests/test_osi_to_cube.py | 11 +++++++---- 5 files changed, 34 insertions(+), 15 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index 6641a1c5..72e3bb64 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -233,8 +233,13 @@ element it concerns, and a detail string. | `GEO_DIMENSION_SPLIT` | A `type: geo` dimension became two Ossie fields | | `TEMPLATED_FILE_SKIPPED` | Jinja templating anywhere in a file, or a `.js`/`.ts` model file. Detected per file, as Cube's own tooling does, so the file is preserved whole rather than half-converted | | `NO_USABLE_DIALECT` | Export: no `ANSI_SQL` or preferred-dialect expression | -| `PARKED_IN_META` | An element with no native mapping, preserved in the stash or under `meta.ossie` — invisible to Cube but intact through a round trip | -| `DROPPED_NO_CUBE_EQUIVALENT` | A value Cube has nowhere to hold *and* that cannot be parked, so it is genuinely gone. Currently only relationship `ai_context`, since a Cube join entry has no `meta` field. Kept distinct from `PARKED_IN_META` so a caller can tell real loss from "preserved but unreadable by Cube" | +| `PARKED_IN_META` | Preserved in the stash or under `meta.ossie` — invisible to Cube, but intact through a round trip | +| `DROPPED_NO_CUBE_EQUIVALENT` | **Gone from the output.** Cube has nowhere to hold it and it cannot be parked: relationship `ai_context` (a Cube join entry has no `meta`), a `dimension.is_time` role or opt-out that Cube expresses only through `type`, and the second and later `semantic_model` entries | +| `APPROXIMATED` | Emitted, but not an exact equivalent: a value Cube requires and Ossie does not carry (so the converter chose one), or a construct rendered in the nearest form Cube has | + +These three are kept distinct on purpose. A caller gating on issue types has to be +able to tell "preserved but unreadable by Cube" from "actually lost" from "emitted, +but asserting slightly more than the input did". ## Requirements diff --git a/converters/cube/src/ossie_cube/converter_issues.py b/converters/cube/src/ossie_cube/converter_issues.py index 0c79610d..0b3dec73 100644 --- a/converters/cube/src/ossie_cube/converter_issues.py +++ b/converters/cube/src/ossie_cube/converter_issues.py @@ -72,6 +72,13 @@ class IssueType(Enum): # "actually lost". DROPPED_NO_CUBE_EQUIVALENT = "DROPPED_NO_CUBE_EQUIVALENT" + # Something *was* emitted, but it is not an exact equivalent: a value Cube + # requires and Ossie does not carry (so the converter had to choose one), or a + # construct rendered in the nearest form Cube has. Nothing is lost and nothing + # is hidden -- but the output asserts a little more than the input did, so it + # is worth a look. + APPROXIMATED = "APPROXIMATED" + @dataclass(frozen=True) class ConverterIssue: diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 70f0ed1d..ab2dd712 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -100,8 +100,9 @@ def convert_ossie_to_cube(ossie_yaml_str, dialect=None, base_cube=None): issues = IssueLog() if len(models) > 1: - issues.add(IssueType.PARKED_IN_META, "model", - f"{len(models)} semantic models found; converting only the first") + issues.add(IssueType.DROPPED_NO_CUBE_EQUIVALENT, "model", + f"{len(models)} semantic models found; only the first is " + f"converted and the rest are not preserved anywhere") return _convert_model(models[0], dialect, base_cube, issues) @@ -280,7 +281,7 @@ def _build_cube(ds, cname, dim_names, inline_sql, joins, measures, dialect, if col in covered: pk_names.append(covered[col]) continue - issues.add(IssueType.PARKED_IN_META, scope, + issues.add(IssueType.APPROXIMATED, scope, f"primary key column '{col}' has no field; emitted as a " f"non-public dimension with type 'string' (Cube requires a type " f"and Ossie carries none here)") @@ -466,18 +467,19 @@ def _dimension_type(field, stash, scope, issues): if ctype is None: raise ConversionError(f"{scope}: unknown datatype '{datatype}'") if explicit_is_time is True and ctype != "time": - issues.add(IssueType.PARKED_IN_META, scope, + issues.add(IssueType.DROPPED_NO_CUBE_EQUIVALENT, scope, f"is_time is true but datatype '{datatype}' maps to Cube " f"type '{ctype}'; Cube marks time dimensions by type, so " f"the temporal role is not carried") elif explicit_is_time is False and ctype == "time": - issues.add(IssueType.PARKED_IN_META, scope, + issues.add(IssueType.DROPPED_NO_CUBE_EQUIVALENT, scope, f"is_time is false but datatype '{datatype}' maps to Cube " - f"type 'time', which Cube always treats as a time dimension") + f"type 'time', which Cube always treats as a time dimension; " + f"the opt-out is not carried") return ctype if explicit_is_time: return "time" - issues.add(IssueType.PARKED_IN_META, scope, + issues.add(IssueType.APPROXIMATED, scope, "no datatype; emitted as Cube type 'string', which Cube requires") return "string" @@ -652,7 +654,7 @@ def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_ expr) if ref in sanitized }) > 1: - issues.add(IssueType.PARKED_IN_META, scope, + issues.add(IssueType.APPROXIMATED, scope, f"expression spans several datasets; emitted as a calculated " f"measure on cube '{target}' -- verify the join path") return measure diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 4b82a07b..1f29359b 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -989,4 +989,6 @@ def test_several_semantic_models_convert_the_first_with_an_issue(): ) files, issues = convert_ossie_to_cube(ossie) assert set(files) == {"model/cubes/orders.yml", "model/views/first.yml"} - assert any("converting only the first" in i.detail for i in issues) + # The other models are not preserved anywhere, so this is a drop. + dropped = issues.of_type(IssueType.DROPPED_NO_CUBE_EQUIVALENT) + assert any("only the first is converted" in i.detail for i in dropped) diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 6372b5bf..1df017ab 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -123,7 +123,8 @@ def test_every_dimension_declares_a_type(): ) files, issues = convert_ossie_to_cube(_ossie(no_type)) assert _cubes(files)["orders"]["dimensions"][0]["type"] == "string" - assert issues.of_type(IssueType.PARKED_IN_META) + # A guess, not a loss and not a park: Cube demands a type Ossie never gave. + assert issues.of_type(IssueType.APPROXIMATED) @pytest.mark.parametrize("datatype,expected", [ @@ -172,7 +173,8 @@ def test_is_time_on_a_non_temporal_datatype_is_reported(): files, issues = convert_ossie_to_cube(_ossie(ds)) dim = parse(files["model/cubes/date_dim.yml"])["cubes"][0]["dimensions"][0] assert dim["type"] == "number" - detail = issues.of_type(IssueType.PARKED_IN_META)[0].detail + # The temporal role is gone from the output, so this is a drop. + detail = issues.of_type(IssueType.DROPPED_NO_CUBE_EQUIVALENT)[0].detail assert "temporal role is not carried" in detail @@ -195,7 +197,8 @@ def test_primary_key_column_without_a_field_is_synthesized(): assert dims["ticket_no"] == { "name": "ticket_no", "sql": "ticket_no", "type": "string", "primary_key": True, "public": False} - assert issues.of_type(IssueType.PARKED_IN_META) + # `type: string` is chosen by the converter, not carried by Ossie. + assert issues.of_type(IssueType.APPROXIMATED) def test_field_name_is_sanitized_and_collisions_are_rejected(): @@ -387,7 +390,7 @@ def test_ratio_becomes_a_calculated_measure(): assert measure["type"] == "number" assert measure["sql"] == "SUM({CUBE.amount}) / COUNT(DISTINCT {users.id})" assert any("spans several datasets" in i.detail - for i in issues.of_type(IssueType.PARKED_IN_META)) + for i in issues.of_type(IssueType.APPROXIMATED)) def test_metric_lands_on_the_dataset_its_expression_references(): From c272df20123a63fc4ee3930de162b32532634f67 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 16:18:03 +0500 Subject: [PATCH 15/46] Fix five round-trip bugs found in review All five reproduce; each was confirmed before being fixed. 1. Multi-stage measures were lost outright. They get no `metrics` entry -- correctly, a window function over another grain has no static Ossie expression -- but nothing stashed them either, and `measures` is a natively-mapped key so `cube_extras` did not carry it. The issue message meanwhile claimed the measure had been preserved, which was simply untrue. They now ride on the owning dataset's stash with their index, the protocol unconvertible joins already use, and export interleaves them back among the measures rebuilt from metrics. Measure conversion moved ahead of dataset construction so the stash is known in time, which meant reading primary keys straight off the dimensions instead of taking them from _convert_cube's return. Renamed MULTI_STAGE_MEASURE_DROPPED -> MULTI_STAGE_MEASURE_PARKED: with the measure genuinely preserved, "dropped" was the same mislabelling corrected in c22e6d2. 2. One-to-one joins were treated as fan-out paths. Neither side of a one-to-one multiplies, so a valid `sum` on the `to` side was refused under strict mode. Now excluded, keyed off the normalized cardinality in the stash so `one_to_one` and legacy `hasOne` both count. A hand-authored relationship carries no Cube cardinality and keeps the conservative assumption. 3. Export could emit Cube its own importer rejects. `COUNT(*)` became a bare `type: count`, but that form is this converter's representation of `COUNT(DISTINCT )`, so importing it demanded a primary key the dataset need not have. `COUNT(*)` now becomes `type: number` with the expression intact -- a pair Cube's own BaseQuery special-cases -- so it round-trips exactly and needs no key. 4./5. Foreign-vendor extensions were parked under `meta.ossie` at field and metric level but only read back at dataset level, so they were dropped on re-import. Restored via one shared helper, after write_stash, so the CUBE entry stays first as it already did for datasets. 248 tests, 97% line coverage. `git diff --check` clean. Co-Authored-By: Claude Opus 5 --- converters/cube/README.md | 2 +- .../cube/src/ossie_cube/converter_issues.py | 8 +- converters/cube/src/ossie_cube/cube_to_osi.py | 120 ++++++++--- converters/cube/src/ossie_cube/osi_to_cube.py | 33 ++- converters/cube/tests/test_cube_to_osi.py | 2 +- converters/cube/tests/test_edge_cases.py | 201 ++++++++++++++++++ converters/cube/tests/test_osi_to_cube.py | 1 - 7 files changed, 323 insertions(+), 44 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index 72e3bb64..ef5771e0 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -228,7 +228,7 @@ element it concerns, and a detail string. | Issue type | Meaning | |---|---| | `FANOUT_UNSAFE_METRIC` | A non-idempotent aggregate on a dataset the graph fans out; see [Fan-out](#fan-out) | -| `MULTI_STAGE_MEASURE_DROPPED` | A `multi_stage` measure (`group_by`/`reduce_by`/`time_shift`/`rank`) renders as a window function over another grain | +| `MULTI_STAGE_MEASURE_PARKED` | A `multi_stage` measure (`group_by`/`reduce_by`/`time_shift`/`rank`) renders as a window function over another grain, so it gets no `metrics` entry — the original is preserved verbatim in the dataset's stash and restored on export | | `CUBE_LEVEL_AI_CONTEXT_INERT` | Cube's agent ignores cube-level `meta.ai_context` | | `GEO_DIMENSION_SPLIT` | A `type: geo` dimension became two Ossie fields | | `TEMPLATED_FILE_SKIPPED` | Jinja templating anywhere in a file, or a `.js`/`.ts` model file. Detected per file, as Cube's own tooling does, so the file is preserved whole rather than half-converted | diff --git a/converters/cube/src/ossie_cube/converter_issues.py b/converters/cube/src/ossie_cube/converter_issues.py index 0b3dec73..93df8842 100644 --- a/converters/cube/src/ossie_cube/converter_issues.py +++ b/converters/cube/src/ossie_cube/converter_issues.py @@ -39,10 +39,10 @@ class IssueType(Enum): FANOUT_UNSAFE_METRIC = "FANOUT_UNSAFE_METRIC" # A `multi_stage` measure (group_by / reduce_by / time_shift / rank). These - # render as window functions over a grain other than the query's, which an - # Ossie expression has no form for; the measure is preserved in the stash - # and omitted from `metrics`. - MULTI_STAGE_MEASURE_DROPPED = "MULTI_STAGE_MEASURE_DROPPED" + # render as window functions over a grain other than the query's, which an Ossie + # expression has no form for -- so the measure gets no `metrics` entry, and the + # original is preserved verbatim in the owning dataset's stash instead. + MULTI_STAGE_MEASURE_PARKED = "MULTI_STAGE_MEASURE_PARKED" # A cube-level `meta.ai_context`. Cube's own agent only consumes ai_context # on views and on individual members, so this value is inert in Cube; it is diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 8ae1bd48..2508942d 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -57,6 +57,7 @@ snake, snake_keys, view_file, + read_stash, write_stash, ) from .converter_issues import IssueLog, IssueType @@ -128,25 +129,25 @@ def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=True) if ai: model["ai_context"] = ai - # Joins are decomposed first: a join with no Ossie form is parked on its - # declaring cube's stash, which has to be known before the dataset is built. + # Anything a cube's stash has to carry is worked out before the dataset is + # built: joins with no Ossie form, and measures with no static Ossie + # expression. Primary keys are read straight off the dimensions so this + # ordering does not depend on the datasets existing yet. relationships, extra_joins = _convert_joins(cubes, sorted(extra_files), issues) - - datasets = [] - pk_by_cube = {} - for cname, cube in cubes.items(): - ds, primary_key = _convert_cube(cname, cube, extra_joins.get(cname), issues) - datasets.append(ds) - pk_by_cube[cname] = primary_key - model["datasets"] = datasets if relationships: model["relationships"] = relationships - - # A dataset on the `to` (one) side of a relationship can be fanned out by rows - # from the `from` (many) side. Derived entirely from the Ossie graph. - fanned_out = {rel["to"]: rel["name"] for rel in relationships} - - metrics = _convert_measures(cubes, pk_by_cube, fanned_out, issues) + fanned_out = _fanned_out_datasets(relationships) + pk_by_cube = {cname: _primary_key_of(cube, cname) + for cname, cube in cubes.items()} + + metrics, extra_measures = _convert_measures( + cubes, pk_by_cube, fanned_out, issues) + + model["datasets"] = [ + _convert_cube(cname, cube, extra_joins.get(cname), + extra_measures.get(cname), issues) + for cname, cube in cubes.items() + ] if metrics: model["metrics"] = metrics @@ -393,10 +394,57 @@ def _meta_without_ai_context(meta): return {k: v for k, v in meta.items() if k not in ("ai_context", "ossie")} +def _fanned_out_datasets(relationships): + """{dataset: relationship name} for datasets a join can multiply rows of. + + A dataset on the `to` (one) side of a many-to-one join is fanned out by rows from + the `from` (many) side. A **one-to-one** join multiplies neither side, so it is + excluded -- otherwise a perfectly safe `sum` on either side would be refused + under strict fan-out mode. The cardinality comes from the stash Cube's join left + behind, in normalized form, so `one_to_one` and the legacy `has_one` both count. + + A hand-authored Ossie relationship carries no Cube cardinality, and Ossie's own + `from`/`to` says only many/one -- so it keeps the conservative assumption. + """ + out = {} + for rel in relationships: + declared = read_stash(rel).get("relationship") + if declared and _RELATIONSHIP_ALIASES.get(snake(declared)) == "one_to_one": + continue + out[rel["to"]] = rel["name"] + return out + + +def _restore_parked_extensions(obj, meta): + """Reattach foreign-vendor extensions a previous export parked under + `meta.ossie.custom_extensions`. + + Called after `write_stash`, so the CUBE entry stays first and the restored + foreign entries follow -- the ordering datasets already used. Without this the + parked entries are stripped by `_meta_without_ai_context` and never come back, + which would make `Ossie -> Cube -> Ossie` lose them. + """ + parked = ((meta or {}).get("ossie") or {}).get("custom_extensions") + if parked: + obj.setdefault("custom_extensions", []).extend(parked) + + # --- cubes ---------------------------------------------------------------------- -def _convert_cube(cname, cube, extra_joins, issues): - """Build one Ossie dataset from a Cube cube. Returns (dataset, primary_key).""" +def _primary_key_of(cube, cname): + """The names of a cube's `primary_key: true` dimensions. + + Read directly off the dimensions so the stages that need it -- measures, and the + fan-out check -- do not have to wait for the dataset to be built. + """ + return [require_str(dim, "name", f"cube '{cname}': dimension") + for dim in _as_named_list(cube.get("dimensions"), + f"cube '{cname}' dimensions") + if dim.get("primary_key")] + + +def _convert_cube(cname, cube, extra_joins, extra_measures, issues): + """Build one Ossie dataset from a Cube cube.""" scope = f"cube '{cname}'" ds = {"name": cname} stash = {} @@ -418,18 +466,22 @@ def _convert_cube(cname, cube, extra_joins, issues): ds["unique_keys"] = [list(k) for k in parked["unique_keys"]] fields = [] - primary_key = [] for dim in _as_named_list(cube.get("dimensions"), f"{scope} dimensions"): dname = require_str(dim, "name", f"{scope}: dimension") - if dim.get("primary_key"): - primary_key.append(dname) fields.extend(_convert_dimension(cname, dname, dim, issues)) if fields: ds["fields"] = fields + primary_key = _primary_key_of(cube, cname) if primary_key: ds["primary_key"] = primary_key if extra_joins: stash["extra_joins"] = extra_joins + if extra_measures: + # Measures with no static Ossie expression (multi-stage ones) ride here with + # their original positions, so export can put them back among the measures it + # rebuilds from metrics. Without this they would be lost outright: `measures` + # is a natively-mapped key, so `cube_extras` does not carry it. + stash["extra_measures"] = extra_measures extras = {snake(k): v for k, v in cube.items() if snake(k) not in _CUBE_NATIVE_KEYS} @@ -444,7 +496,7 @@ def _convert_cube(cname, cube, extra_joins, issues): # stash is written, so the CUBE entry stays first and both survive. if parked.get("custom_extensions"): ds.setdefault("custom_extensions", []).extend(parked["custom_extensions"]) - return ds, primary_key + return ds def _convert_dimension(cname, dname, dim, issues): @@ -504,6 +556,10 @@ def _convert_dimension(cname, dname, dim, issues): if leftover_meta: stash["meta"] = leftover_meta write_stash(field, stash) + # Foreign-vendor extensions a previous export parked under the dimension's + # `meta.ossie` are restored after the stash is written, so the CUBE entry stays + # first -- the same ordering datasets use. + _restore_parked_extensions(field, dim.get("meta")) return [field] @@ -734,7 +790,7 @@ def expression(self, cname, mname, stack=()): # group_by / reduce_by / time_shift / rank render as window functions # over a grain other than the query's; Ossie has no form for that. self._issues.add( - IssueType.MULTI_STAGE_MEASURE_DROPPED, scope, + IssueType.MULTI_STAGE_MEASURE_PARKED, scope, f"multi_stage measure (type '{mtype}'); preserved in " f"custom_extensions only") return None @@ -825,6 +881,12 @@ def _convert_measures(cubes, pk_by_cube, fanned_out, issues): A metric name is the measure name when globally unique, else `__`; the original name and owning cube are stashed so export puts the measure back where it came from. + + Returns (metrics, {cube: [{"index": i, "measure": ...}]}). The second value holds + measures with no static Ossie expression -- a multi-stage measure renders as a + window function over another grain -- which have no `metrics` entry and would + otherwise vanish. They ride on the owning dataset's stash with their positions, + the same protocol unconvertible joins use. """ resolver = _MeasureResolver(cubes, pk_by_cube, issues) @@ -833,10 +895,12 @@ def _convert_measures(cubes, pk_by_cube, fanned_out, issues): counts[mname] = counts.get(mname, 0) + 1 metrics = [] + extra_measures = {} seen = set() for cname, cube in cubes.items(): - for measure in _as_named_list(cube.get("measures"), - f"cube '{cname}' measures"): + for index, measure in enumerate( + _as_named_list(cube.get("measures"), + f"cube '{cname}' measures")): mname = measure["name"] metric_name = mname if counts[mname] == 1 else f"{cname}__{mname}" if metric_name in seen: @@ -848,7 +912,10 @@ def _convert_measures(cubes, pk_by_cube, fanned_out, issues): fanned_out, issues) if metric is not None: metrics.append(metric) - return metrics + else: + extra_measures.setdefault(cname, []).append( + {"index": index, "measure": measure}) + return metrics, extra_measures def _convert_measure(cname, mname, metric_name, measure, resolver, fanned_out, @@ -916,4 +983,5 @@ def _convert_measure(cname, mname, metric_name, measure, resolver, fanned_out, if leftover_meta: stash["meta"] = leftover_meta write_stash(metric, stash) + _restore_parked_extensions(metric, measure.get("meta")) return metric diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index ab2dd712..6cf2f6be 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -303,8 +303,14 @@ def _build_cube(ds, cname, dim_names, inline_sql, joins, measures, dialect, joins.insert(min(item.get("index", 0), len(joins)), item["join"]) if joins: cube["joins"] = joins + measures = [_ordered(m, _MEASURE_KEY_ORDER) for m in (measures or [])] + # Measures a prior import could not express in Ossie (multi-stage ones) go back + # at their original indices, interleaved with the ones rebuilt from metrics. + for item in sorted(stash.get("extra_measures") or [], + key=lambda x: x.get("index", 0)): + measures.insert(min(item.get("index", 0), len(measures)), item["measure"]) if measures: - cube["measures"] = [_ordered(m, _MEASURE_KEY_ORDER) for m in measures] + cube["measures"] = measures for key, value in cube_extras.items(): cube[key] = value @@ -632,16 +638,21 @@ def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_ measure["type"] = "count" return measure func = "COUNT_DISTINCT" - if func == "COUNT" and inner == "*": - measure["type"] = "count" - return measure - agg = OSSIE_FUNC_TO_AGG.get(func) or ("count" if func == "COUNT" else None) - if agg is not None: - measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( - inner, target, members, sanitized, - inline_sql=inline_sql_by_cube) - measure["type"] = agg - return measure + # `COUNT(*)` deliberately falls through to the calculated measure below. + # A bare Cube `type: count` is this converter's representation of + # `COUNT(DISTINCT )` -- handled above -- so emitting one here + # would round-trip back as a different expression, and on a dataset with no + # primary key it would produce a measure the importer refuses. Cube renders + # `type: number` with `count(*)` natively (BaseQuery special-cases exactly + # that pair), so the expression survives intact either way. + if not (func == "COUNT" and inner == "*"): + agg = OSSIE_FUNC_TO_AGG.get(func) or ("count" if func == "COUNT" else None) + if agg is not None: + measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( + inner, target, members, sanitized, + inline_sql=inline_sql_by_cube) + measure["type"] = agg + return measure # A ratio, a window expression, or a multi-dataset aggregate: Cube expresses # these as a calculated measure whose sql carries the aggregation. diff --git a/converters/cube/tests/test_cube_to_osi.py b/converters/cube/tests/test_cube_to_osi.py index faedcbf7..66da6106 100644 --- a/converters/cube/tests/test_cube_to_osi.py +++ b/converters/cube/tests/test_cube_to_osi.py @@ -360,7 +360,7 @@ def test_multi_stage_measure_is_dropped_with_an_issue(): } out, issues = convert_cube_to_ossie(files) assert "metrics" not in model_of(out) - assert issues.of_type(IssueType.MULTI_STAGE_MEASURE_DROPPED) + assert issues.of_type(IssueType.MULTI_STAGE_MEASURE_PARKED) # --- fan-out -------------------------------------------------------------------- diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 1f29359b..5b34565f 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -156,6 +156,140 @@ def test_count_over_an_expression_is_fanout_unsafe(): assert issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) +_MULTI_STAGE = _files(orders=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: revenue\n" + " sql: amount\n" + " type: sum\n" + " - name: rolling\n" + " sql: amount\n" + " type: sum\n" + " multi_stage: true\n" + " rolling_window:\n" + " trailing: 3 month\n" + " - name: cnt\n" + " type: count\n" +)) + + +def test_a_multi_stage_measure_is_not_an_ossie_metric(): + """It renders as a window function over another grain, which an Ossie expression + has no form for -- so it gets no `metrics` entry, and that is reported.""" + ossie, issues = convert_cube_to_ossie(_MULTI_STAGE) + assert [m["name"] for m in model_of(ossie)["metrics"]] == ["revenue", "cnt"] + parked = issues.of_type(IssueType.MULTI_STAGE_MEASURE_PARKED) + assert [i.element_name for i in parked] == ["orders.rolling"] + + +def test_a_multi_stage_measure_survives_the_round_trip_in_place(): + """It used to be lost outright: no metric, and `measures` is a natively-mapped key + so `cube_extras` did not carry it either -- while the issue claimed it had been + preserved. Now it rides on the dataset's stash with its position, like an + unconvertible join, and comes back interleaved with the rebuilt measures.""" + ossie, _ = convert_cube_to_ossie(_MULTI_STAGE) + stashed = stash_of(by_name(model_of(ossie)["datasets"])["orders"]) + assert stashed["extra_measures"] == [ + {"index": 1, "measure": { + "name": "rolling", "sql": "amount", "type": "sum", + "multi_stage": True, "rolling_window": {"trailing": "3 month"}}}] + + back, _ = convert_ossie_to_cube(ossie) + assert parse_files(back) == parse_files(_MULTI_STAGE) + # Order matters: it goes back between the two ordinary measures. + names = [m["name"] for m in parse( + back["model/cubes/orders.yml"])["cubes"][0]["measures"]] + assert names == ["revenue", "rolling", "cnt"] + + +def test_count_star_is_not_emitted_as_a_bare_cube_count(): + """A bare Cube `type: count` is this converter's form for + `COUNT(DISTINCT )`. Emitting one for `COUNT(*)` round-tripped back as a + different expression, and on a dataset with no primary key produced a measure + the importer refuses -- export generating what its own import rejects.""" + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: public.orders\n" + " metrics:\n" + " - name: n\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: COUNT(*)\n" + ) + files, _ = convert_ossie_to_cube(ossie) + measure = parse(files["model/cubes/orders.yml"])["cubes"][0]["measures"][0] + assert measure == {"name": "n", "sql": "COUNT(*)", "type": "number"} + + # And it survives the trip back, without a primary key anywhere in sight. + ossie2, _ = convert_cube_to_ossie(files) + assert expr_of(model_of(ossie2)["metrics"][0]) == "COUNT(*)" + + +def test_field_and_metric_foreign_extensions_survive_the_round_trip(): + """Foreign-vendor extensions are parked under `meta.ossie` at every level, but + only datasets were reading them back -- so field- and metric-level ones were + parked and then silently dropped on re-import.""" + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: public.orders\n" + " fields:\n" + " - name: status\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: status\n" + " datatype: String\n" + " custom_extensions:\n" + " - vendor_name: SNOWFLAKE\n" + " data: '{\"collation\": \"en\"}'\n" + " - name: amount\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: amount\n" + " datatype: Decimal\n" + " metrics:\n" + " - name: total\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: SUM(orders.amount)\n" + " custom_extensions:\n" + " - vendor_name: DBT\n" + " data: '{\"model\": \"fct_orders\"}'\n" + ) + files, _ = convert_ossie_to_cube(ossie) + ossie2, _ = convert_cube_to_ossie(files) + model = model_of(ossie2) + + field = by_name(by_name(model["datasets"])["orders"]["fields"])["status"] + exts = {e["vendor_name"]: e["data"] for e in field["custom_extensions"]} + assert exts["SNOWFLAKE"] == '{"collation": "en"}' + # The CUBE stash is written first, foreign entries appended -- as for datasets. + assert field["custom_extensions"][0]["vendor_name"] == "CUBE" + + metric = by_name(model["metrics"])["total"] + mexts = {e["vendor_name"]: e["data"] for e in metric["custom_extensions"]} + assert mexts["DBT"] == '{"model": "fct_orders"}' + assert metric["custom_extensions"][0]["vendor_name"] == "CUBE" + + # --- join orientation, both ways ------------------------------------------------ _ONE_TO_MANY = _files(m=( @@ -191,6 +325,73 @@ def test_one_to_many_is_flipped_back_onto_its_original_cube(): assert "joins" not in cubes["orders"] +@pytest.mark.parametrize("declared", ["one_to_one", "hasOne", "has_one"]) +def test_a_one_to_one_join_does_not_make_its_target_fanned_out(declared): + """A one-to-one join multiplies neither side, so a `sum` across it is safe. It was + being treated like any other relationship, whose `to` side *is* fanned out, and a + valid measure was refused under strict mode.""" + files = _files(m=( + "cubes:\n" + " - name: users\n" + " sql_table: public.users\n" + " joins:\n" + " - name: profiles\n" + " sql: \"{CUBE}.id = {profiles}.user_id\"\n" + f" relationship: {declared}\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " - name: profiles\n" + " sql_table: public.profiles\n" + " dimensions:\n" + " - name: user_id\n" + " sql: user_id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: score_total\n" + " sql: \"{CUBE}.score\"\n" + " type: sum\n" + )) + # Strict mode is the default; this must simply convert. + ossie, issues = convert_cube_to_ossie(files) + assert not issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) + assert expr_of(by_name(model_of(ossie)["metrics"])["score_total"]) == ( + "SUM(profiles.score)") + + +def test_a_many_to_one_join_still_makes_its_target_fanned_out(): + """The counterpart: excluding one-to-one must not weaken the ordinary case.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {users}.id\"\n" + " relationship: many_to_one\n" + " dimensions:\n" + " - name: user_id\n" + " sql: user_id\n" + " type: number\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: ltv\n" + " sql: \"{CUBE}.ltv\"\n" + " type: sum\n" + )) + with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): + convert_cube_to_ossie(files) + + def test_one_to_one_keeps_its_declared_orientation(): files = _files(m=_ONE_TO_MANY["model/cubes/m.yml"].replace( "one_to_many", "one_to_one")) diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 1df017ab..44af252b 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -355,7 +355,6 @@ def _metric(name, expr): {"type": "count_distinct", "sql": "{CUBE.amount}"}), ("APPROX_COUNT_DISTINCT(orders.amount)", {"type": "count_distinct_approx", "sql": "{CUBE.amount}"}), - ("COUNT(*)", {"type": "count"}), ]) def test_aggregate_expressions_become_structured_measures(expr, expected): files, _ = convert_ossie_to_cube(_ossie(_ORDERS, metrics=_metric("m", expr))) From d42c6f89b1631b5136cf0e32bd7e5b4b83c830d4 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 17:03:10 +0500 Subject: [PATCH 16/46] Keep every view when several share one YAML file _build_views keyed one view per file path, so two views in the same file collapsed to whichever was written last. Reproduced: a file with `alpha` and `beta` came back holding only `beta`. Worse than a plain drop, because the survivor is arbitrary. In the reproduction the lost view was `alpha` -- the *mapped* one, which is where the model's description and AI context live, so the model's own metadata lost its home too. Now grouped path -> [views] and extended into files_content, preserving declaration order. The existing two-view test did not catch this: it put each view in its own file, so the paths never collided. The new tests use one shared file and assert both the round trip and that the mapped view is still the one carrying model metadata. 250 tests, 97% line coverage. `git diff --check` clean. Co-Authored-By: Claude Opus 5 --- converters/cube/src/ossie_cube/osi_to_cube.py | 16 ++++--- converters/cube/tests/test_edge_cases.py | 47 +++++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 6cf2f6be..3033c940 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -166,9 +166,9 @@ def _convert_model(model, dialect, base_cube, issues): path = stashed_paths.get(cname) or cube_file(cname) files_content.setdefault(path, {}).setdefault("cubes", []).append(cube) - for vpath, view in _build_views(model, model_stash, cube_names, relationships, - datasets, base_cube).items(): - files_content.setdefault(vpath, {}).setdefault("views", []).append(view) + for vpath, views in _build_views(model, model_stash, cube_names, relationships, + datasets, base_cube).items(): + files_content.setdefault(vpath, {}).setdefault("views", []).extend(views) files = {path: dump_yaml(content) for path, content in files_content.items()} @@ -701,7 +701,10 @@ def _balanced(s): def _build_views(model, model_stash, cube_names, relationships, datasets, base_cube): - """Return {file path: view dict}. + """Return {file path: [view dict, ...]}. + + A list per path, not a single view: several views can share one YAML file, and + keying one view per path silently kept only the last. Stashed views restore verbatim, with the natively mapped description and AI context re-injected on the mapped one. The `views` stash key being *present* -- @@ -742,7 +745,8 @@ def _build_views(model, model_stash, cube_names, relationships, datasets, meta = _build_meta(model.get("ai_context"), view.get("meta"), parked) if meta: view["meta"] = meta - out[paths.get(vname) or view_file(vname)] = view + path = paths.get(vname) or view_file(vname) + out.setdefault(path, []).append(view) return out vname = sanitize_name(model.get("name", "model"), "Model", set()) @@ -756,7 +760,7 @@ def _build_views(model, model_stash, cube_names, relationships, datasets, cube_names, relationships, cube_names[_pick_base_cube(model.get("name", ""), datasets, relationships, base_cube)]) - out[view_file(vname)] = view + out[view_file(vname)] = [view] return out diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 5b34565f..7232f2e4 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -1087,6 +1087,53 @@ def test_foreign_extensions_survive_once_a_view_is_mapped(): assert "SNOWFLAKE" in vendors +_TWO_VIEWS_ONE_FILE = { + "model/all.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + "views:\n" + " - name: alpha\n" + " description: A\n" + " cubes:\n" + " - join_path: orders\n" + " includes: '*'\n" + " - name: beta\n" + " description: B\n" + ), +} + + +def test_several_views_in_one_file_all_survive(): + """Views were keyed one-per-path on export, so two sharing a file meant the second + overwrote the first. The lost one here is `alpha` -- the *mapped* view, which is + where the model's own description and AI context live.""" + ossie, _ = convert_cube_to_ossie(_TWO_VIEWS_ONE_FILE, view="alpha") + back, _ = convert_ossie_to_cube(ossie) + assert set(back) == {"model/all.yml"} + rebuilt = parse(back["model/all.yml"]) + # Declaration order preserved, both present. + assert [v["name"] for v in rebuilt["views"]] == ["alpha", "beta"] + assert [c["name"] for c in rebuilt["cubes"]] == ["orders"] + assert parse_files(back) == parse_files(_TWO_VIEWS_ONE_FILE) + + +def test_the_mapped_view_in_a_shared_file_still_carries_model_metadata(): + """The mapped view is the model's home for description and AI context, so it has + to be the one updated -- not whichever view happens to be written last.""" + ossie, _ = convert_cube_to_ossie(_TWO_VIEWS_ONE_FILE, view="alpha") + model = model_of(ossie) + assert model["name"] == "alpha" + assert model["description"] == "A" + + model["description"] = "edited" + files, _ = convert_ossie_to_cube(dump_yaml({ + "version": "0.2.0.dev0", "semantic_model": [model]})) + views = by_name(parse(files["model/all.yml"])["views"]) + assert views["alpha"]["description"] == "edited" + assert views["beta"]["description"] == "B" + + def test_both_views_are_restored_on_export(): ossie, _ = convert_cube_to_ossie(_TWO_VIEWS, view="b") back, _ = convert_ossie_to_cube(ossie) From a377cab9b85c715f178087456b37aa4410d56e9f Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 17:33:21 +0500 Subject: [PATCH 17/46] Mark a Cube primary key only on a dimension that actually is that column `primary_key: true` in Cube declares that dimension's own `sql` to be the key, but coverage was satisfied by name equality alone. Two ways that declared the wrong thing, both reproduced: primary_key: [id] + dimension `id` sql `LOWER(email)` -> {name: id, sql: LOWER(email), primary_key: true} the lowercased email became the key primary_key: [location] + merged geo dimension `location` -> {name: location, type: geo, primary_key: true} a dimension with two sql expressions and no single one Coverage now requires the dimension to be *scalar* -- its expression a single source column -- reachable either by that column's name or by its own name. A computed dimension and a merged geo dimension qualify under neither. Removing the name match outright would have regressed the round trip: import records the *dimension name* in `primary_key`, not the column, so a Cube dimension `order_id` with `sql: id` comes back as `primary_key: [order_id]`. Gating the name match on the dimension being scalar keeps that working while excluding the two bad cases -- verified against a Cube -> Ossie -> Cube trip that stays exact. Synthesizing the replacement now avoids collisions. The obvious name is the key entry itself, but a computed or geo dimension may already own it, and emitting a second dimension of that name would produce an invalid cube while overwriting would lose a member. A `_pk` suffix is added until the name is free, and the issue says so when it happens. 255 tests, 97% line coverage. `git diff --check` clean. Co-Authored-By: Claude Opus 5 --- converters/cube/README.md | 2 +- converters/cube/src/ossie_cube/osi_to_cube.py | 84 +++++++++---- converters/cube/tests/test_edge_cases.py | 112 ++++++++++++++++++ 3 files changed, 173 insertions(+), 25 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index ef5771e0..ecb19241 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -121,7 +121,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | `dataset.source` (`SELECT ...`) | `sql` | Cube requires exactly one of `sql` / `sql_table`. | | `dataset.description` | cube `description` | | | `dataset.ai_context` | cube `meta.ai_context` | Preserved for the round trip, but **inert in Cube** -- its agent ignores cube-level `ai_context`. Recorded as an issue. | -| `dataset.primary_key` | dimension(s) with `primary_key: true` | Composite = several. Export: a key column no field covers becomes a `public: false` dimension. | +| `dataset.primary_key` | dimension(s) with `primary_key: true` | Composite = several. Export marks a dimension only when it is **scalar** — a single source column — since `primary_key: true` declares that dimension's own `sql` to be the key; a computed dimension or a merged `geo` one would declare the wrong thing even if its name matches. Anything left uncovered becomes a `public: false` scalar dimension, suffixed (`id_pk`) if the obvious name is taken. | | `dataset.unique_keys` | `meta.ossie.unique_keys` | No native Cube slot; parked rather than dropped. | | field | `dimensions[]` entry | Export: a name that is not a valid Cube identifier is sanitized; a case-insensitive collision is an error, never a silent merge. | | `field.expression` | dimension `sql` | Dataset-scoped, so `{CUBE}.col` <-> `col`. Export emits `{CUBE}.column` for a raw column and `{CUBE.member}` for a declared member, and never spells the cube's own name (which would break under `extends`). | diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 3033c940..8ac3d7b1 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -271,25 +271,35 @@ def _build_cube(ds, cname, dim_names, inline_sql, joins, measures, dialect, "Cube's agent reads ai_context only on views and members, " "so this cube-level value has no effect in Cube") - dimensions, covered = _build_dimensions( + dimensions, by_name_scalar, by_column = _build_dimensions( ds, cname, dim_names, inline_sql, dialect, issues) - # A primary-key column no field covers still has to exist as a dimension for - # Cube to join or roll up the cube. + # Resolve each `primary_key` entry to the dimension Cube should mark. A + # dimension only qualifies when it is *scalar* -- backed by a single source + # column -- because `primary_key: true` in Cube declares that dimension's own + # sql to be the key. A computed dimension would declare the wrong expression, + # and a merged geo dimension has no single sql at all, so neither counts even + # when its name matches. Anything left uncovered gets a private dimension. pk_names = [] - for col in (ds.get("primary_key") or []): - col = str(col) - if col in covered: - pk_names.append(covered[col]) + taken = {d["name"].lower() for d in dimensions} + for entry in (ds.get("primary_key") or []): + entry = str(entry) + # Import records the *dimension name*, so the name match is checked first; + # a hand-authored model naming the source column resolves by column. + match = by_name_scalar.get(entry) or by_column.get(entry) + if match: + pk_names.append(match) continue - issues.add(IssueType.APPROXIMATED, scope, - f"primary key column '{col}' has no field; emitted as a " - f"non-public dimension with type 'string' (Cube requires a type " - f"and Ossie carries none here)") - synth = {"name": col, "sql": col, "type": "string", - "primary_key": True, "public": False} - dimensions.append(synth) - covered[col] = col - pk_names.append(col) + name = _unique_pk_dimension_name(entry, taken) + taken.add(name.lower()) + detail = (f"primary key '{entry}' is not backed by a scalar dimension; " + f"emitted as a non-public dimension with type 'string' (Cube " + f"requires a type and Ossie carries none here)") + if name != entry: + detail += f", named '{name}' to avoid colliding with the existing member" + issues.add(IssueType.APPROXIMATED, scope, detail) + dimensions.append({"name": name, "sql": entry, "type": "string", + "primary_key": True, "public": False}) + pk_names.append(name) for dim in dimensions: if dim["name"] in pk_names: dim["primary_key"] = True @@ -381,14 +391,19 @@ def _resolve_dimension_names(ds, scope): def _build_dimensions(ds, cname, dim_names, inline_sql, dialect, issues): """Build a cube's dimensions from an Ossie dataset's fields. - Returns (dimensions, {column or field name: dimension name}) -- the second - value is what primary-key resolution matches against. Fields carrying a `geo` - stash are re-merged into the single Cube dimension they were split from. + Returns (dimensions, by_name_scalar, by_column) -- the two maps are what + primary-key resolution matches against, and both hold only *scalar* dimensions + (those whose expression is a single source column). A computed dimension and a + merged geo dimension are deliberately absent from both: Cube's + `primary_key: true` declares that dimension's own sql to be the key, so marking + either would declare something other than the column Ossie named. Fields + carrying a `geo` stash are re-merged into the single Cube dimension they were + split from. Dimension names come from `dim_names` (see `_resolve_dimension_names`) rather than being sanitized again here. """ ds_name = ds["name"] - covered = {} + by_name_scalar, by_column = {}, {} # Built by target dimension name rather than by list position: a geo dimension # is assembled from two fields that may appear in either order and need not be # adjacent, so an insertion index computed mid-loop is not a safe way to hold @@ -440,9 +455,11 @@ def _build_dimensions(ds, cname, dim_names, inline_sql, dialect, issues): dim[key] = value built[dname] = dim - covered[dname] = dname if is_simple_identifier(expr): - covered[expr.strip()] = dname + # Scalar: this dimension is exactly one source column, so Cube can mark + # it as the key. Reachable by its own name and by that column's name. + by_name_scalar[dname] = dname + by_column.setdefault(expr.strip(), dname) # Both halves are guaranteed present by _resolve_dimension_names, which # validates the pair before anything is built. @@ -453,11 +470,30 @@ def _build_dimensions(ds, cname, dim_names, inline_sql, dialect, issues): for key, value in (slot.get("host") or {}).items(): dim[key] = value built[base] = dim - covered[base] = base # A name in `order` with nothing built is a field dropped for want of a usable # dialect; it simply does not appear. - return [built[n] for n in order if n in built], covered + return [built[n] for n in order if n in built], by_name_scalar, by_column + + +def _unique_pk_dimension_name(entry, taken): + """A valid, unused Cube identifier for a synthesized primary-key dimension. + + The obvious name is the primary-key entry itself, but a computed or geo + dimension may already own it -- in which case emitting a second dimension of + that name would produce an invalid cube, and overwriting the existing one would + lose a member. So a suffix is added until the name is free. + """ + base = sanitize_name(entry, "primary key", set()) + if base.lower() not in taken: + return base + for n in range(1, 100): + candidate = f"{base}_pk" if n == 1 else f"{base}_pk_{n}" + if candidate.lower() not in taken: + return candidate + raise ConversionError( + f"cannot find a free dimension name for primary key '{entry}'; rename the " + f"colliding members in the Ossie model.") def _dimension_type(field, stash, scope, issues): diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 7232f2e4..a86656d1 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -919,6 +919,118 @@ def _ossie_fields(*specs): return out +def _ossie_pk(primary_key, *specs): + """An Ossie model with a primary_key and (name, expression, geo part) fields.""" + out = ("version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: public.orders\n" + " primary_key:\n") + for col in primary_key: + out += f" - {col}\n" + out += " fields:\n" + for fname, expr, part in specs: + out += (f" - name: {fname}\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + f" expression: {expr}\n" + " datatype: String\n") + if part: + out += (" custom_extensions:\n" + " - vendor_name: CUBE\n" + f" data: '{_geo_stash(part, of=fname.rsplit('_', 1)[0])}'\n") + return out + + +def _dims(files): + return parse(files["model/cubes/orders.yml"])["cubes"][0]["dimensions"] + + +def test_a_computed_dimension_does_not_cover_a_primary_key(): + """`primary_key: true` in Cube declares that dimension's own sql to be the key. + Marking a computed dimension would declare `LOWER(email)` as the key when Ossie + named the `id` column -- so a name match alone must not count as coverage.""" + files, issues = convert_ossie_to_cube( + _ossie_pk(["id"], ("id", "LOWER(email)", None))) + dims = by_name(_dims(files)) + assert "primary_key" not in dims["id"] + assert dims["id"]["sql"] == "LOWER(email)" + # A private scalar dimension carries the key instead, under a free name. + assert dims["id_pk"] == {"name": "id_pk", "sql": "id", "type": "string", + "primary_key": True, "public": False} + assert issues.of_type(IssueType.APPROXIMATED) + + +def test_a_merged_geo_dimension_does_not_cover_a_primary_key(): + """A geo dimension has two sql expressions and no single one, so it cannot be + the key even though its name matches.""" + files, _ = convert_ossie_to_cube(_ossie_pk( + ["location"], + ("location_latitude", "lat", "latitude"), + ("location_longitude", "lon", "longitude"))) + dims = by_name(_dims(files)) + assert dims["location"]["type"] == "geo" + assert "primary_key" not in dims["location"] + assert dims["location_pk"] == { + "name": "location_pk", "sql": "location", "type": "string", + "primary_key": True, "public": False} + + +def test_a_scalar_dimension_backed_by_the_key_column_covers_it(): + """The legitimate case: a differently-named dimension whose sql *is* the key + column. It stays the key, and nothing is synthesized alongside it.""" + files, issues = convert_ossie_to_cube( + _ossie_pk(["id"], ("order_id", "id", None))) + dims = _dims(files) + assert len(dims) == 1 + assert dims[0]["name"] == "order_id" + assert dims[0]["primary_key"] is True + assert not issues.of_type(IssueType.APPROXIMATED) + + +def test_a_scalar_dimension_named_as_the_key_covers_it(): + """Import records the *dimension name* in `primary_key`, not the column, so a + scalar dimension matching by name has to keep covering it -- otherwise + `Cube -> Ossie -> Cube` would synthesize a bogus duplicate key.""" + src = _files(orders=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: order_id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + )) + ossie, _ = convert_cube_to_ossie(src) + assert by_name(model_of(ossie)["datasets"])["orders"]["primary_key"] == [ + "order_id"] + back, _ = convert_ossie_to_cube(ossie) + assert parse_files(back) == parse_files(src) + + +def test_a_synthesized_key_name_avoids_every_existing_member(): + """Suffixing has to keep going while names are taken, and the result must still + be a single non-public scalar dimension.""" + files, _ = convert_ossie_to_cube(_ossie_pk( + ["id"], + ("id", "LOWER(email)", None), + ("id_pk", "UPPER(email)", None), + ("id_pk_2", "TRIM(email)", None))) + dims = by_name(_dims(files)) + keys = [n for n, d in dims.items() if d.get("primary_key")] + assert keys == ["id_pk_3"] + assert dims["id_pk_3"] == {"name": "id_pk_3", "sql": "id", "type": "string", + "primary_key": True, "public": False} + # Nothing was overwritten. + assert dims["id"]["sql"] == "LOWER(email)" + assert dims["id_pk"]["sql"] == "UPPER(email)" + assert dims["id_pk_2"]["sql"] == "TRIM(email)" + + def test_geo_halves_may_appear_in_any_order_without_clobbering_a_dimension(): """The geo dimension is assembled from two fields that need not be adjacent and may come in either order. Holding its place with a list index computed mid-loop From 27938071bad2165e74310d87d216e33d39278c14 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Mon, 3 Aug 2026 16:02:38 +0500 Subject: [PATCH 18/46] Stop stashing what is not Cube-specific (first pass) Groundwork for the interop critique: the stash was dominated by round-trip bookkeeping rather than Cube features, and every entry becomes a warning-and-discard in converters that do not read foreign extensions. Measured on Cube -> Ossie -> Databricks, 32 of the warnings were purely this converter's own stash. Three sources removed: - A member's exact `sql` spelling. Only a *member* reference (`{CUBE.member}`) genuinely needs it, because Cube inlines the referenced member's own SQL; a plain `{CUBE}.column` regenerates faithfully. Added `sql_is_reversible` to tell them apart. - A `many_to_one` join declared on the many side, which is exactly what Ossie's `from`(many) -> `to`(one) already says. A legacy spelling (`belongsTo`) is still kept exact, so preserving it costs the modern spelling nothing. - The join `sql` in the common case, by emitting the alias-dot raw-column form on both sides (`{users}.id`) rather than a member reference (`{users.id}`). Ossie's from_columns/to_columns name columns, so this is also the more faithful form. Consequence, accepted deliberately: `{CUBE}.column` and a bare `column` mean the same thing and are no longer distinguished, so Cube -> Ossie -> Cube is now lossless *semantically* rather than byte-for-byte. Expressed in the tests as one narrow normalization in `_util.canon_sql`, which leaves `{CUBE.member}` alone precisely because that one is not equivalent. Fixture stash entries: fixtureA 18 -> 13, and Databricks foreign-extension warnings 32 -> 17 on the TPC-DS model. 255 tests, 96% line coverage. `git diff --check` clean. Co-Authored-By: Claude Opus 5 --- converters/cube/src/ossie_cube/_common.py | 27 +++ converters/cube/src/ossie_cube/cube_to_osi.py | 31 +++- converters/cube/src/ossie_cube/osi_to_cube.py | 2 +- converters/cube/tests/_roundtrip_helpers.py | 5 +- converters/cube/tests/_util.py | 42 ++++- .../cube/tests/fixtures/fixtureA_ossie.yaml | 38 ++--- .../cube/tests/fixtures/tpcds_ossie.yaml | 159 +++++++----------- converters/cube/tests/test_cube_to_osi.py | 7 +- converters/cube/tests/test_edge_cases.py | 5 +- converters/cube/tests/test_osi_to_cube.py | 6 +- 10 files changed, 170 insertions(+), 152 deletions(-) diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index 5cb4ab28..f6db16e1 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -385,6 +385,33 @@ def repl(m): return out, changed +def sql_is_reversible(sql): + """True if translating this Cube SQL to Ossie and back reproduces it. + + Only `{CUBE}.column` / `{TABLE}.column` -- a raw physical column of the owning + cube -- survives the trip, because Ossie expressions address columns and the + exporter re-emits them bare. A *member* reference (`{CUBE.member}`, `{member}`, + `{other.member}`) does not: Cube inlines the referenced member's own SQL, which + can differ from a column of that name, so the original spelling has to be kept. + + Used to decide whether the exact Cube `sql` needs stashing at all -- most + dimensions reference plain columns, so most need nothing. + """ + if not isinstance(sql, str): + sql = str(sql) + protected = sql.replace("\\{", "").replace("\\}", "") + for m in _CUBE_REF_RE.finditer(protected): + body = m.group(1).strip() + if body not in _SELF_REFS: + return False + # `{CUBE}` on its own (no trailing `.column`) is the cube's alias, which an + # Ossie expression cannot express either. + rest = protected[m.end():] + if not rest.startswith("."): + return False + return True + + def requalify_self_refs(sql, cube_name): """Rewrite `{CUBE}` / `{TABLE}` in a Cube SQL snippet to name `cube_name`. diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 2508942d..c313e62d 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -56,6 +56,7 @@ require_str, snake, snake_keys, + sql_is_reversible, view_file, read_stash, write_stash, @@ -516,11 +517,14 @@ def _convert_dimension(cname, dname, dim, issues): # No `sql` means the same-named physical column. expr = dname else: - expr, changed = cube_sql_to_ossie(sql, cname) - if changed or str(sql).strip() == dname: - # Stashed when the Ossie expression differs from the Cube sql, and also - # when the sql is an explicit same-named bare column -- which export - # would otherwise normalize away to the implicit form. + expr, _ = cube_sql_to_ossie(sql, cname) + if not sql_is_reversible(sql): + # Only a *member* reference needs the original spelling kept: Cube + # inlines the referenced member's own SQL, which a bare column name in + # the Ossie expression would not reproduce. A plain `{CUBE}.column` (or + # a bare column) regenerates faithfully, so nothing is stashed -- which + # is the common case, and stashing it only added noise for every other + # converter reading the model. stash["sql"] = sql field = { @@ -645,7 +649,17 @@ def _convert_joins(cubes, skipped_files, issues): from_cube, to_cube = cname, target from_cols = [p[0] for p in pairs] to_cols = [p[1] for p in pairs] - stash = {"declared_on": cname, "relationship": raw_rel} + # A `many_to_one` join declared on the many side is exactly what Ossie's + # `from`(many) -> `to`(one) already says, so nothing is stashed for the + # common case. Only an orientation Ossie cannot express on its own -- + # one_to_many (flipped) or one_to_one (no many side) -- needs recording. + stash = {} + if (rel_type != "many_to_one" or cname != from_cube + or raw_rel != "many_to_one"): + # The last clause keeps a legacy spelling (`belongsTo`) exact + # without costing the modern spelling a stash entry. + stash["declared_on"] = cname + stash["relationship"] = raw_rel if rel_type == "one_to_many": from_cube, to_cube = to_cube, from_cube from_cols, to_cols = to_cols, from_cols @@ -730,9 +744,10 @@ def _ref_target(side, own_cube, target): def _rebuild_join_sql(target, pairs): """The canonical form export emits, used to decide whether the original has to be stashed. The own side is always `{CUBE}` so the join keeps working when the - cube is extended.""" + cube is extended, and both sides use the alias-dot raw-column form because + Ossie's from_columns/to_columns name columns, not members.""" return " AND ".join( - "{CUBE}." + own + " = {" + target + "." + other + "}" + "{CUBE}." + own + " = {" + target + "}." + other for own, other in pairs ) diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 8ac3d7b1..80845f80 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -570,7 +570,7 @@ def _build_joins(relationships, cube_names, issues): join["sql"] = stash["sql"] else: join["sql"] = " AND ".join( - "{CUBE}." + str(a) + " = {" + other + "." + str(b) + "}" + "{CUBE}." + str(a) + " = {" + other + "}." + str(b) for a, b in zip(own_cols, other_cols)) for key, value in stash.items(): if key not in ("declared_on", "relationship", "sql"): diff --git a/converters/cube/tests/_roundtrip_helpers.py b/converters/cube/tests/_roundtrip_helpers.py index f7d20f16..2343f683 100644 --- a/converters/cube/tests/_roundtrip_helpers.py +++ b/converters/cube/tests/_roundtrip_helpers.py @@ -185,7 +185,10 @@ def _build_dimension(rnd, name): def _parse_files(files): - return {name: load_yaml(text, name) for name, text in files.items()} + # Same documented normalization the fixture tests use; see _util.canon_sql. + from _util import canon_sql + return {name: canon_sql(load_yaml(text, name)) + for name, text in files.items()} def assert_cube_roundtrip_is_lossless(files): diff --git a/converters/cube/tests/_util.py b/converters/cube/tests/_util.py index bf11b481..5e18bd28 100644 --- a/converters/cube/tests/_util.py +++ b/converters/cube/tests/_util.py @@ -20,6 +20,7 @@ import copy import json import pathlib +import re from ossie_cube._common import load_yaml # src is on sys.path via conftest.py @@ -46,17 +47,46 @@ def parse(yaml_str): return load_yaml(yaml_str) +_ALIAS_DOT_RE = re.compile(r"\$?\{\s*(?:CUBE|TABLE)\s*\}\s*\.") + + +def canon_sql(node): + """Canonicalize the one documented Cube SQL normalization, in place-ish. + + `{CUBE}.column` and a bare `column` are the same thing -- a raw physical column + of the owning cube -- so the converter no longer stashes the original spelling + just to reproduce it. Round-trip assertions therefore compare with the alias + prefix removed. + + Deliberately narrow: `{CUBE.member}` is a *member* reference and means something + else, so it is left alone (and is still stashed, so it round-trips exactly). + """ + if isinstance(node, dict): + return {k: (_ALIAS_DOT_RE.sub("", v) + if k == "sql" and isinstance(v, str) else canon_sql(v)) + for k, v in node.items()} + if isinstance(node, list): + return [canon_sql(v) for v in node] + return node + + def parse_files(files): - """Parse every file of a Cube model dict for structural comparison. + """Parse a Cube model dict into the form round-trip fidelity is asserted on. + + Comments and key order are not part of the data model, so comparison happens on + parsed structures. A non-YAML file (a `.js` model preserved verbatim) is + compared as text. - Comments and key order are not part of the data model, so round-trip fidelity - is asserted on the parsed structures. A non-YAML file (a `.js` model preserved - verbatim) is compared as text. + Also applies `canon_sql`: the converter no longer stashes a member's exact SQL + spelling just to reproduce `{CUBE}.column` over a bare `column`, since the two + mean the same thing and stashing it put noise into every other converter's view + of the model. That spelling is therefore a documented normalization, not a + difference worth failing on. """ out = {} for name, text in files.items(): - out[name] = (load_yaml(text, name) if name.lower().endswith((".yml", ".yaml")) - else text) + out[name] = (canon_sql(load_yaml(text, name)) + if name.lower().endswith((".yml", ".yaml")) else text) return out diff --git a/converters/cube/tests/fixtures/fixtureA_ossie.yaml b/converters/cube/tests/fixtures/fixtureA_ossie.yaml index d78abf3d..8247d6c2 100644 --- a/converters/cube/tests/fixtures/fixtureA_ossie.yaml +++ b/converters/cube/tests/fixtures/fixtureA_ossie.yaml @@ -25,6 +25,14 @@ semantic_model: ai_context: instructions: | Primary view for revenue analysis. Use it for any question about sales, orders, or customer spend. + relationships: + - name: orders_to_users + from: orders + to: users + from_columns: + - user_id + to_columns: + - id datasets: - name: orders source: public.orders @@ -37,7 +45,7 @@ semantic_model: expression: id custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "id", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: user_id expression: dialects: @@ -45,7 +53,7 @@ semantic_model: expression: user_id custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "user_id", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: status expression: dialects: @@ -56,9 +64,6 @@ semantic_model: description: Current order status ai_context: instructions: Values are pending, shipped, and completed. - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "status"}' - name: created_at expression: dialects: @@ -67,18 +72,12 @@ semantic_model: datatype: DateTime dimension: is_time: true - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "created_at"}' - name: is_large expression: dialects: - dialect: ANSI_SQL expression: amount > 500 datatype: Boolean - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "{CUBE}.amount > 500"}' primary_key: - id - name: users @@ -91,16 +90,13 @@ semantic_model: expression: id custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "id", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: city expression: dialects: - dialect: ANSI_SQL expression: city datatype: String - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "city"}' - name: location_latitude expression: dialects: @@ -125,18 +121,6 @@ semantic_model: - vendor_name: CUBE data: '{"_v": 1, "cube_extras": {"segments": [{"name": "active", "sql": "{CUBE}.status = ''active''"}]}}' - relationships: - - name: orders_to_users - from: orders - to: users - from_columns: - - user_id - to_columns: - - id - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "declared_on": "orders", "relationship": "many_to_one", "sql": - "{CUBE}.user_id = {users}.id"}' metrics: - name: orders__count expression: diff --git a/converters/cube/tests/fixtures/tpcds_ossie.yaml b/converters/cube/tests/fixtures/tpcds_ossie.yaml index 5ee7e1ae..0e6ad634 100644 --- a/converters/cube/tests/fixtures/tpcds_ossie.yaml +++ b/converters/cube/tests/fixtures/tpcds_ossie.yaml @@ -26,6 +26,47 @@ semantic_model: sales, customer, product, and store data from the TPC-DS benchmark. The model supports time-based analysis, customer segmentation, product performance, and store operations metrics. + relationships: + - name: store_sales_to_date_dim + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - d_date_sk + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "{CUBE}.ss_sold_date_sk = {date_dim.d_date_sk}"}' + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: + - ss_customer_sk + to_columns: + - c_customer_sk + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "{CUBE}.ss_customer_sk = {customer.c_customer_sk}"}' + - name: store_sales_to_item + from: store_sales + to: item + from_columns: + - ss_item_sk + to_columns: + - i_item_sk + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "{CUBE}.ss_item_sk = {item.i_item_sk}"}' + - name: store_sales_to_store + from: store_sales + to: store + from_columns: + - ss_store_sk + to_columns: + - s_store_sk + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "{CUBE}.ss_store_sk = {store.s_store_sk}"}' datasets: - name: store_sales source: tpcds.public.store_sales @@ -52,7 +93,7 @@ semantic_model: - transaction date custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "ss_sold_date_sk", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: ss_item_sk expression: dialects: @@ -65,7 +106,7 @@ semantic_model: - item custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "ss_item_sk", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: ss_customer_sk expression: dialects: @@ -78,7 +119,7 @@ semantic_model: - buyer custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "ss_customer_sk", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: ss_store_sk expression: dialects: @@ -91,7 +132,7 @@ semantic_model: - location custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "ss_store_sk", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: ss_quantity expression: dialects: @@ -104,7 +145,7 @@ semantic_model: - quantity custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "ss_quantity", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: ss_sales_price expression: dialects: @@ -117,7 +158,7 @@ semantic_model: - price custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "ss_sales_price", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: ss_ext_sales_price expression: dialects: @@ -130,7 +171,7 @@ semantic_model: - line total custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "ss_ext_sales_price", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: ss_net_profit expression: dialects: @@ -143,7 +184,7 @@ semantic_model: - margin custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "ss_net_profit", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: ss_ticket_number expression: dialects: @@ -152,7 +193,7 @@ semantic_model: datatype: String custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "ss_ticket_number", "public": false}' + data: '{"_v": 1, "public": false}' primary_key: - ss_item_sk - ss_ticket_number @@ -175,7 +216,7 @@ semantic_model: description: Surrogate key for date custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "d_date_sk", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: d_date expression: dialects: @@ -189,9 +230,6 @@ semantic_model: synonyms: - date - calendar date - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "d_date"}' - name: d_year expression: dialects: @@ -203,7 +241,7 @@ semantic_model: - year custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "d_year", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: d_quarter_name expression: dialects: @@ -217,9 +255,6 @@ semantic_model: synonyms: - quarter - fiscal quarter - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "d_quarter_name"}' - name: d_month_name expression: dialects: @@ -232,9 +267,6 @@ semantic_model: ai_context: synonyms: - month - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "d_month_name"}' primary_key: - d_date_sk - name: customer @@ -256,7 +288,7 @@ semantic_model: description: Surrogate key for customer custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "c_customer_sk", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: c_customer_id expression: dialects: @@ -268,9 +300,6 @@ semantic_model: synonyms: - customer ID - customer number - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "c_customer_id"}' - name: c_first_name expression: dialects: @@ -278,9 +307,6 @@ semantic_model: expression: c_first_name datatype: String description: Customer first name - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "c_first_name"}' - name: c_last_name expression: dialects: @@ -288,9 +314,6 @@ semantic_model: expression: c_last_name datatype: String description: Customer last name - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "c_last_name"}' - name: customer_full_name expression: dialects: @@ -313,9 +336,6 @@ semantic_model: synonyms: - email - contact - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "c_email_address"}' primary_key: - c_customer_sk - name: item @@ -337,7 +357,7 @@ semantic_model: description: Surrogate key for item custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "i_item_sk", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: i_item_id expression: dialects: @@ -350,9 +370,6 @@ semantic_model: - item ID - product ID - SKU - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "i_item_id"}' - name: i_item_desc expression: dialects: @@ -364,9 +381,6 @@ semantic_model: synonyms: - product description - item name - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "i_item_desc"}' - name: i_brand expression: dialects: @@ -378,9 +392,6 @@ semantic_model: synonyms: - brand - manufacturer - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "i_brand"}' - name: i_category expression: dialects: @@ -392,9 +403,6 @@ semantic_model: synonyms: - product category - department - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "i_category"}' - name: i_current_price expression: dialects: @@ -407,7 +415,7 @@ semantic_model: - list price custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "i_current_price", "type": "number"}' + data: '{"_v": 1, "type": "number"}' primary_key: - i_item_sk - name: store @@ -429,7 +437,7 @@ semantic_model: description: Surrogate key for store custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "s_store_sk", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: s_store_id expression: dialects: @@ -441,9 +449,6 @@ semantic_model: synonyms: - store ID - store number - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "s_store_id"}' - name: s_store_name expression: dialects: @@ -455,9 +460,6 @@ semantic_model: synonyms: - store name - location name - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "s_store_name"}' - name: s_city expression: dialects: @@ -469,9 +471,6 @@ semantic_model: synonyms: - city - location - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "s_city"}' - name: s_state expression: dialects: @@ -483,9 +482,6 @@ semantic_model: synonyms: - state - region - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "s_state"}' - name: s_number_employees expression: dialects: @@ -498,50 +494,9 @@ semantic_model: - staff size custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "s_number_employees", "type": "number"}' + data: '{"_v": 1, "type": "number"}' primary_key: - s_store_sk - relationships: - - name: store_sales_to_date_dim - from: store_sales - to: date_dim - from_columns: - - ss_sold_date_sk - to_columns: - - d_date_sk - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "declared_on": "store_sales", "relationship": "many_to_one"}' - - name: store_sales_to_customer - from: store_sales - to: customer - from_columns: - - ss_customer_sk - to_columns: - - c_customer_sk - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "declared_on": "store_sales", "relationship": "many_to_one"}' - - name: store_sales_to_item - from: store_sales - to: item - from_columns: - - ss_item_sk - to_columns: - - i_item_sk - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "declared_on": "store_sales", "relationship": "many_to_one"}' - - name: store_sales_to_store - from: store_sales - to: store - from_columns: - - ss_store_sk - to_columns: - - s_store_sk - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "declared_on": "store_sales", "relationship": "many_to_one"}' metrics: - name: total_sales expression: diff --git a/converters/cube/tests/test_cube_to_osi.py b/converters/cube/tests/test_cube_to_osi.py index 66da6106..fcf2335e 100644 --- a/converters/cube/tests/test_cube_to_osi.py +++ b/converters/cube/tests/test_cube_to_osi.py @@ -155,9 +155,10 @@ def test_many_to_one_join_becomes_a_relationship(model_a): assert rel["to"] == "users" assert rel["from_columns"] == ["user_id"] assert rel["to_columns"] == ["id"] - # The declaring side and the exact Cube spelling round-trip via the stash. - assert stash_of(rel)["declared_on"] == "orders" - assert stash_of(rel)["relationship"] == "many_to_one" + # Nothing is stashed: a many_to_one join declared on the many side is exactly + # what `from`(many) -> `to`(one) already says, so recording it again would only + # add a custom_extension for every other converter to warn about and discard. + assert stash_of(rel) == {} def test_one_to_many_join_is_flipped_to_many_side_first(): diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index a86656d1..e7e92beb 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -281,8 +281,9 @@ def test_field_and_metric_foreign_extensions_survive_the_round_trip(): field = by_name(by_name(model["datasets"])["orders"]["fields"])["status"] exts = {e["vendor_name"]: e["data"] for e in field["custom_extensions"]} assert exts["SNOWFLAKE"] == '{"collation": "en"}' - # The CUBE stash is written first, foreign entries appended -- as for datasets. - assert field["custom_extensions"][0]["vendor_name"] == "CUBE" + # A plain scalar field needs no CUBE stash at all any more, so the foreign + # extension is the only entry -- which is the point of the reduction. + assert list(exts) == ["SNOWFLAKE"] metric = by_name(model["metrics"])["total"] mexts = {e["vendor_name"]: e["data"] for e in metric["custom_extensions"]} diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 44af252b..f2ee960b 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -306,7 +306,9 @@ def test_preferred_dialect_wins_over_ansi(): def test_relationship_lands_on_the_many_side_as_many_to_one(): files, _ = convert_ossie_to_cube(_ossie(_TWO_DATASETS, _REL)) join = _cubes(files)["orders"]["joins"][0] - assert join == {"name": "users", "sql": "{CUBE}.user_id = {users.id}", + # Alias-dot on both sides: Ossie's from_columns/to_columns name columns, so the + # far side is a raw column reference too, not a member reference. + assert join == {"name": "users", "sql": "{CUBE}.user_id = {users}.id", "relationship": "many_to_one"} # The one side declares nothing; Cube needs the join on one side only. assert "joins" not in _cubes(files, "model/cubes/users.yml")["users"] @@ -319,7 +321,7 @@ def test_composite_relationship_becomes_an_and_chain(): " to_columns: [id, region]\n") files, _ = convert_ossie_to_cube(_ossie(_TWO_DATASETS, rel)) assert _cubes(files)["orders"]["joins"][0]["sql"] == ( - "{CUBE}.user_id = {users.id} AND {CUBE}.region = {users.region}") + "{CUBE}.user_id = {users}.id AND {CUBE}.region = {users}.region") def test_relationship_ai_context_is_reported_as_dropped_not_parked(): From a397e85df4c5b3a913d0e41ce1084272a7b49a15 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Mon, 3 Aug 2026 16:19:32 +0500 Subject: [PATCH 19/46] Map datatypes natively and narrow member references Two more stash sources removed, and both come out better than the stash they replace rather than merely smaller. Cube's dimension `type` was stashed because it is coarser than Ossie's `datatype` -- Integer, Decimal and Float all become `number` -- so no mapping back is exact. Omitting the datatype and stashing `number` gave every other spoke a warning and no type information at all. Now `number` maps to `Decimal`, the safe reading for the money and quantity columns it overwhelmingly holds, and export parks the *precise* datatype under `meta.ossie.datatype` whenever the default would not recover it. `meta.ossie` is Cube-side, so it costs the Ossie document nothing. The result is strictly better in both directions: all nine Ossie datatypes now survive Ossie -> Cube -> Ossie exactly, where `Integer` and `Float` were previously lost outright, and the Ossie document carries a real datatype for other converters to act on. Measure operands stopped stashing their spelling too. `{CUBE.member}` is only required when the member is *not* just its own same-named column, because that form makes Cube inline the member's SQL; for a plain member the raw `{CUBE}.column` form is identical and regenerates. Cross-cube references are deliberately left in member form -- that is what makes Cube add the implicit join, so the two are not interchangeable there. Regenerated tpcds_cube from examples/tpcds_semantic_model.yaml so the fixture reflects what the converter now emits rather than an older shape. CUBE stash entries: TPC-DS 41 -> 7, fixtureA 18 -> 10. Foreign-extension warnings: databricks 32 -> 2, snowflake 41 -> 7, omni 41 -> 7, wisdom 41 -> 7. 255 tests, 96% line coverage. `git diff --check` clean. Co-Authored-By: Claude Opus 5 --- converters/cube/src/ossie_cube/_common.py | 59 +++++++++---- converters/cube/src/ossie_cube/cube_to_osi.py | 50 +++++++---- converters/cube/src/ossie_cube/osi_to_cube.py | 53 +++++++++--- .../cube/tests/fixtures/fixtureA_ossie.yaml | 14 +--- .../tpcds_cube/model/cubes/store_sales.yml | 18 ++-- .../cube/tests/fixtures/tpcds_ossie.yaml | 82 +++++-------------- converters/cube/tests/test_cube_to_osi.py | 12 +-- converters/cube/tests/test_edge_cases.py | 2 +- converters/cube/tests/test_osi_to_cube.py | 14 ++-- 9 files changed, 167 insertions(+), 137 deletions(-) diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index f6db16e1..b25aa2ca 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -385,30 +385,45 @@ def repl(m): return out, changed -def sql_is_reversible(sql): +def sql_is_reversible(sql, plain_members=(), own_cube=None): """True if translating this Cube SQL to Ossie and back reproduces it. - Only `{CUBE}.column` / `{TABLE}.column` -- a raw physical column of the owning - cube -- survives the trip, because Ossie expressions address columns and the - exporter re-emits them bare. A *member* reference (`{CUBE.member}`, `{member}`, - `{other.member}`) does not: Cube inlines the referenced member's own SQL, which - can differ from a column of that name, so the original spelling has to be kept. + `{CUBE}.column` / `{TABLE}.column` -- a raw physical column of the owning cube -- + always survives, because Ossie expressions address columns and the exporter + re-emits them in that form. - Used to decide whether the exact Cube `sql` needs stashing at all -- most - dimensions reference plain columns, so most need nothing. + A *member* reference (`{CUBE.member}`, `{member}`) survives only when the member + is **plain**: its own `sql` is just the same-named column, so the reference and + the raw column are the same thing. Otherwise Cube inlines the member's own SQL, + which a bare column name would not reproduce, and the original spelling has to be + kept. + + A **cross-cube** reference never survives: `{other.member}` is what makes Cube + add the implicit join, and the raw `{other}.column` form does not, so the two are + not interchangeable. """ if not isinstance(sql, str): sql = str(sql) + plain = set(plain_members) protected = sql.replace("\\{", "").replace("\\}", "") for m in _CUBE_REF_RE.finditer(protected): body = m.group(1).strip() - if body not in _SELF_REFS: - return False - # `{CUBE}` on its own (no trailing `.column`) is the cube's alias, which an - # Ossie expression cannot express either. - rest = protected[m.end():] - if not rest.startswith("."): - return False + head, _, rest = body.partition(".") + if not rest: + if body in _SELF_REFS or (own_cube and body == own_cube): + # A bare alias only makes sense followed by `.column`. + if not protected[m.end():].startswith("."): + return False + continue + # `{member}` -- an unqualified own-cube member reference. + if body not in plain: + return False + continue + if head in _SELF_REFS or (own_cube and head == own_cube): + if rest not in plain: + return False + continue + return False # cross-cube reference; carries join semantics return True @@ -530,8 +545,22 @@ def join_source(cube, cube_name): "boolean": "Boolean", "time": "DateTime", "switch": "String", + # Cube collapses Integer/Decimal/Float into one type, so no mapping back is + # exact. `Decimal` is chosen over omitting a datatype because a downstream + # converter can use it: exact base-10 is the safe reading for the money and + # quantity columns `number` overwhelmingly holds, and asserting it beats + # emitting nothing plus a Cube-only extension no other spoke reads. When the + # model came from Ossie in the first place, the precise datatype is recovered + # from `meta.ossie.datatype` instead of guessed. + "number": "Decimal", } +# The datatype each Cube type maps back to by default. Export parks the original in +# `meta.ossie.datatype` only when it is *not* the default -- Cube cannot hold the +# distinction, and `meta.ossie` is Cube-side, so this keeps Ossie -> Cube -> Ossie +# exact without putting anything in `custom_extensions`. +DEFAULT_DATATYPE_FOR_CUBE_TYPE = dict(DIM_TYPE_TO_DATATYPE) + # Ossie `datatype` -> Cube dimension `type`, which is required on every # dimension. Lossy in the numeric and temporal directions by construction. DATATYPE_TO_DIM_TYPE = { diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index c313e62d..c2e5f0f1 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -432,6 +432,23 @@ def _restore_parked_extensions(obj, meta): # --- cubes ---------------------------------------------------------------------- +def _plain_members(cube, cname): + """Dimension names whose `sql` is just the same-named column. + + For those, `{CUBE.member}`, `{CUBE}.member` and a bare `member` all mean the + same thing, so the spelling carries no information worth stashing. Any other + member inlines its own SQL when referenced, which a column name would not + reproduce. + """ + plain = set() + for dim in _as_named_list(cube.get("dimensions"), f"cube '{cname}' dimensions"): + name = dim.get("name") + sql = dim.get("sql") + if name and (sql is None or str(sql).strip() == name): + plain.add(name) + return plain + + def _primary_key_of(cube, cname): """The names of a cube's `primary_key: true` dimensions. @@ -445,6 +462,7 @@ def _primary_key_of(cube, cname): def _convert_cube(cname, cube, extra_joins, extra_measures, issues): + plain = _plain_members(cube, cname) """Build one Ossie dataset from a Cube cube.""" scope = f"cube '{cname}'" ds = {"name": cname} @@ -469,7 +487,7 @@ def _convert_cube(cname, cube, extra_joins, extra_measures, issues): fields = [] for dim in _as_named_list(cube.get("dimensions"), f"{scope} dimensions"): dname = require_str(dim, "name", f"{scope}: dimension") - fields.extend(_convert_dimension(cname, dname, dim, issues)) + fields.extend(_convert_dimension(cname, dname, dim, plain, issues)) if fields: ds["fields"] = fields primary_key = _primary_key_of(cube, cname) @@ -500,7 +518,7 @@ def _convert_cube(cname, cube, extra_joins, extra_measures, issues): return ds -def _convert_dimension(cname, dname, dim, issues): +def _convert_dimension(cname, dname, dim, plain, issues): """Build the Ossie field(s) for one Cube dimension. Returns a list because a `type: geo` dimension carries two SQL expressions @@ -518,7 +536,7 @@ def _convert_dimension(cname, dname, dim, issues): expr = dname else: expr, _ = cube_sql_to_ossie(sql, cname) - if not sql_is_reversible(sql): + if not sql_is_reversible(sql, plain, cname): # Only a *member* reference needs the original spelling kept: Cube # inlines the referenced member's own SQL, which a bare column name in # the Ossie expression would not reproduce. A plain `{CUBE}.column` (or @@ -532,16 +550,13 @@ def _convert_dimension(cname, dname, dim, issues): "expression": {"dialects": [{"dialect": DIALECT_ANSI, "expression": expr}]}, } datatype = DIM_TYPE_TO_DATATYPE.get(dtype) - if datatype: - field["datatype"] = datatype - elif dtype == "number": - # Cube collapses Integer/Decimal/Float into `number`, so no Ossie datatype - # is asserted -- the spec says to omit it when unknown. The original type - # rides in the stash so export reproduces it. - stash["type"] = dtype - else: + if not datatype: raise ConversionError( f"cube '{cname}': dimension '{dname}' has unknown type '{dtype}'") + # A precise datatype parked by a previous export wins over the default the Cube + # type maps to, since Cube itself cannot hold the distinction. + parked_dt = ((dim.get("meta") or {}).get("ossie") or {}).get("datatype") + field["datatype"] = parked_dt or datatype if dtype == "time": field["dimension"] = {"is_time": True} if dim.get("title"): @@ -924,7 +939,8 @@ def _convert_measures(cubes, pk_by_cube, fanned_out, issues): f"colliding measures in Cube") seen.add(metric_name) metric = _convert_measure(cname, mname, metric_name, measure, resolver, - fanned_out, issues) + fanned_out, _plain_members(cube, cname), + issues) if metric is not None: metrics.append(metric) else: @@ -934,7 +950,7 @@ def _convert_measures(cubes, pk_by_cube, fanned_out, issues): def _convert_measure(cname, mname, metric_name, measure, resolver, fanned_out, - issues): + plain, issues): scope = f"{cname}.{mname}" expr = resolver.expression(cname, mname) if expr is None: @@ -985,10 +1001,10 @@ def _convert_measure(cname, mname, metric_name, measure, resolver, fanned_out, snake(k): v for k, v in measure.items() if snake(k) not in ("description", "meta") } - elif sql is not None: - # The operand's exact Cube spelling: `{CUBE}.city` and `{CUBE.city}` are - # equivalent but not interchangeable byte-for-byte, and export cannot tell - # which one the author wrote from the Ossie expression alone. + elif sql is not None and not sql_is_reversible(sql, plain, cname): + # Only a reference export cannot regenerate needs the original spelling: a + # non-plain member (whose own SQL is inlined) or a cross-cube reference + # (which is what adds the implicit join). stash["sql"] = sql if metric_name != mname: stash["name"] = mname diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 80845f80..beaee615 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -36,6 +36,7 @@ from ._common import ( DATATYPE_TO_DIM_TYPE, + DEFAULT_DATATYPE_FOR_CUBE_TYPE, OSSIE_FUNC_TO_AGG, OSSIE_VERSION, ConversionError, @@ -146,7 +147,9 @@ def _convert_model(model, dialect, base_cube, issues): cname = cube_names[ds_name] dim_names_by_cube[cname], inline_sql_by_cube[cname] = ( _resolve_dimension_names(ds, f"Model '{name}': dataset '{ds_name}'")) - members_by_cube[cname] = set(dim_names_by_cube[cname].values()) + # Not every member: only those the `{CUBE.member}` form is required for. + members_by_cube[cname] = _reference_members( + ds, dim_names_by_cube[cname], dialect) pk_by_cube[cname] = [str(c) for c in (ds.get("primary_key") or [])] joins_by_cube = _build_joins(relationships, cube_names, issues) @@ -161,8 +164,9 @@ def _convert_model(model, dialect, base_cube, issues): for ds_name, ds in datasets.items(): cname = cube_names[ds_name] cube = _build_cube(ds, cname, dim_names_by_cube[cname], - inline_sql_by_cube[cname], joins_by_cube.get(cname), - measures_by_cube.get(cname), dialect, issues) + inline_sql_by_cube[cname], members_by_cube[cname], + joins_by_cube.get(cname), measures_by_cube.get(cname), + dialect, issues) path = stashed_paths.get(cname) or cube_file(cname) files_content.setdefault(path, {}).setdefault("cubes", []).append(cube) @@ -241,8 +245,8 @@ def _ordered(obj, order): # --- cubes ---------------------------------------------------------------------- -def _build_cube(ds, cname, dim_names, inline_sql, joins, measures, dialect, - issues): +def _build_cube(ds, cname, dim_names, inline_sql, ref_members, joins, measures, + dialect, issues): ds_name = ds["name"] scope = f"dataset '{ds_name}'" stash = read_stash(ds) @@ -272,7 +276,7 @@ def _build_cube(ds, cname, dim_names, inline_sql, joins, measures, dialect, "so this cube-level value has no effect in Cube") dimensions, by_name_scalar, by_column = _build_dimensions( - ds, cname, dim_names, inline_sql, dialect, issues) + ds, cname, dim_names, inline_sql, ref_members, dialect, issues) # Resolve each `primary_key` entry to the dimension Cube should mark. A # dimension only qualifies when it is *scalar* -- backed by a single source # column -- because `primary_key: true` in Cube declares that dimension's own @@ -327,6 +331,26 @@ def _build_cube(ds, cname, dim_names, inline_sql, joins, measures, dialect, return _ordered(cube, _CUBE_KEY_ORDER) +def _reference_members(ds, dim_names, dialect): + """Members that must be addressed as `{CUBE.member}` rather than `{CUBE}.column`. + + Only a member whose expression is something other than its own same-named column + needs the reference form, because that form makes Cube inline the member's SQL. A + plain member is identical either way, and the raw-column form is what survives a + round trip without stashing the spelling. + """ + needed = set() + for field in (ds.get("fields") or []): + fname = field.get("name") + dname = dim_names.get(fname) + if not dname: + continue + expr = pick_expression(field.get("expression"), dialect) + if expr is None or not is_simple_identifier(expr) or expr.strip() != dname: + needed.add(dname) + return needed + + def _resolve_dimension_names(ds, scope): """Map each of a dataset's fields to the Cube dimension name it becomes. @@ -388,7 +412,8 @@ def _resolve_dimension_names(ds, scope): return names, inline_sql -def _build_dimensions(ds, cname, dim_names, inline_sql, dialect, issues): +def _build_dimensions(ds, cname, dim_names, inline_sql, ref_members, dialect, + issues): """Build a cube's dimensions from an Ossie dataset's fields. Returns (dimensions, by_name_scalar, by_column) -- the two maps are what @@ -436,8 +461,7 @@ def _build_dimensions(ds, cname, dim_names, inline_sql, dialect, issues): dim["sql"] = stash["sql"] else: dim["sql"] = ossie_expr_to_cube_sql( - expr, cname, set(dim_names.values()), (), - inline_sql={cname: inline_sql}) + expr, cname, ref_members, (), inline_sql={cname: inline_sql}) dim["type"] = _dimension_type(field, stash, f"{ds_name}.{fname}", issues) if field.get("label"): dim["title"] = field["label"] @@ -447,6 +471,14 @@ def _build_dimensions(ds, cname, dim_names, inline_sql, dialect, issues): foreign = foreign_vendor_extensions(field) if foreign: parked["custom_extensions"] = foreign + # Cube's `type` is coarser than Ossie's `datatype` (Integer/Decimal/Float all + # become `number`), so the precise one is parked whenever importing would not + # recover it. `meta.ossie` is Cube-side, so this costs the Ossie document + # nothing -- unlike a custom_extension, which every other spoke would warn + # about and discard. + dt = field.get("datatype") + if dt and DEFAULT_DATATYPE_FOR_CUBE_TYPE.get(dim["type"]) != dt: + parked["datatype"] = dt extras = {k: v for k, v in stash.items() if k not in ("sql", "type", "meta")} meta = _build_meta(field.get("ai_context"), stash.get("meta"), parked) if meta: @@ -499,8 +531,7 @@ def _unique_pk_dimension_name(entry, taken): def _dimension_type(field, stash, scope, issues): """Choose the Cube `type`, which every dimension must declare.""" if "type" in stash: - # Cube collapses Integer/Decimal/Float into `number`, so import parks the - # original rather than asserting an Ossie datatype; restore it here. + # An older stash from before datatypes were mapped natively. return stash["type"] datatype = field.get("datatype") explicit_is_time = (field.get("dimension") or {}).get("is_time") diff --git a/converters/cube/tests/fixtures/fixtureA_ossie.yaml b/converters/cube/tests/fixtures/fixtureA_ossie.yaml index 8247d6c2..96efcf5d 100644 --- a/converters/cube/tests/fixtures/fixtureA_ossie.yaml +++ b/converters/cube/tests/fixtures/fixtureA_ossie.yaml @@ -43,17 +43,13 @@ semantic_model: dialects: - dialect: ANSI_SQL expression: id - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' + datatype: Decimal - name: user_id expression: dialects: - dialect: ANSI_SQL expression: user_id - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' + datatype: Decimal - name: status expression: dialects: @@ -88,9 +84,7 @@ semantic_model: dialects: - dialect: ANSI_SQL expression: id - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' + datatype: Decimal - name: city expression: dialects: @@ -179,7 +173,7 @@ semantic_model: datatype: Integer custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "cube": "users", "sql": "{CUBE}.city"}' + data: '{"_v": 1, "cube": "users"}' custom_extensions: - vendor_name: CUBE data: '{"_v": 1, "views": {"sales": {"name": "sales", "cubes": [{"join_path": diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml index 48f2b70f..45eb7c44 100644 --- a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml @@ -39,16 +39,16 @@ cubes: - POS data joins: - name: date_dim - sql: '{CUBE}.ss_sold_date_sk = {date_dim.d_date_sk}' + sql: '{CUBE}.ss_sold_date_sk = {date_dim}.d_date_sk' relationship: many_to_one - name: customer - sql: '{CUBE}.ss_customer_sk = {customer.c_customer_sk}' + sql: '{CUBE}.ss_customer_sk = {customer}.c_customer_sk' relationship: many_to_one - name: item - sql: '{CUBE}.ss_item_sk = {item.i_item_sk}' + sql: '{CUBE}.ss_item_sk = {item}.i_item_sk' relationship: many_to_one - name: store - sql: '{CUBE}.ss_store_sk = {store.s_store_sk}' + sql: '{CUBE}.ss_store_sk = {store}.s_store_sk' relationship: many_to_one dimensions: - name: ss_sold_date_sk @@ -147,7 +147,7 @@ cubes: public: false measures: - name: total_sales - sql: '{CUBE.ss_ext_sales_price}' + sql: '{CUBE}.ss_ext_sales_price' type: sum description: Total sales revenue across all transactions meta: @@ -159,7 +159,7 @@ cubes: - gross sales - sales amount - name: total_profit - sql: '{CUBE.ss_net_profit}' + sql: '{CUBE}.ss_net_profit' type: sum description: Total net profit from store sales meta: @@ -171,7 +171,7 @@ cubes: - total earnings - profit - name: customer_lifetime_value - sql: SUM({CUBE.ss_ext_sales_price}) / COUNT(DISTINCT {customer.c_customer_sk}) + sql: SUM({CUBE}.ss_ext_sales_price) / COUNT(DISTINCT {customer.c_customer_sk}) type: number description: Average lifetime sales value per customer meta: @@ -184,7 +184,7 @@ cubes: - customer value - lifetime revenue - name: sales_by_brand - sql: '{CUBE.ss_ext_sales_price}' + sql: '{CUBE}.ss_ext_sales_price' type: sum description: Total sales by brand (requires grouping by item.i_brand) meta: @@ -196,7 +196,7 @@ cubes: - brand performance - brand revenue - name: store_productivity - sql: SUM({CUBE.ss_ext_sales_price}) / NULLIF(SUM({store.s_number_employees}), + sql: SUM({CUBE}.ss_ext_sales_price) / NULLIF(SUM({store.s_number_employees}), 0) type: number description: Sales per employee across stores diff --git a/converters/cube/tests/fixtures/tpcds_ossie.yaml b/converters/cube/tests/fixtures/tpcds_ossie.yaml index 0e6ad634..0b323fb9 100644 --- a/converters/cube/tests/fixtures/tpcds_ossie.yaml +++ b/converters/cube/tests/fixtures/tpcds_ossie.yaml @@ -34,9 +34,6 @@ semantic_model: - ss_sold_date_sk to_columns: - d_date_sk - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "{CUBE}.ss_sold_date_sk = {date_dim.d_date_sk}"}' - name: store_sales_to_customer from: store_sales to: customer @@ -44,9 +41,6 @@ semantic_model: - ss_customer_sk to_columns: - c_customer_sk - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "{CUBE}.ss_customer_sk = {customer.c_customer_sk}"}' - name: store_sales_to_item from: store_sales to: item @@ -54,9 +48,6 @@ semantic_model: - ss_item_sk to_columns: - i_item_sk - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "{CUBE}.ss_item_sk = {item.i_item_sk}"}' - name: store_sales_to_store from: store_sales to: store @@ -64,9 +55,6 @@ semantic_model: - ss_store_sk to_columns: - s_store_sk - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "{CUBE}.ss_store_sk = {store.s_store_sk}"}' datasets: - name: store_sales source: tpcds.public.store_sales @@ -86,105 +74,89 @@ semantic_model: dialects: - dialect: ANSI_SQL expression: ss_sold_date_sk + datatype: Decimal description: Foreign key to date dimension ai_context: synonyms: - sale date - transaction date - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: ss_item_sk expression: dialects: - dialect: ANSI_SQL expression: ss_item_sk + datatype: Decimal description: Foreign key to item dimension ai_context: synonyms: - product - item - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: ss_customer_sk expression: dialects: - dialect: ANSI_SQL expression: ss_customer_sk + datatype: Decimal description: Foreign key to customer dimension ai_context: synonyms: - customer - buyer - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: ss_store_sk expression: dialects: - dialect: ANSI_SQL expression: ss_store_sk + datatype: Decimal description: Foreign key to store dimension ai_context: synonyms: - store - location - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: ss_quantity expression: dialects: - dialect: ANSI_SQL expression: ss_quantity + datatype: Decimal description: Quantity of items sold ai_context: synonyms: - units sold - quantity - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: ss_sales_price expression: dialects: - dialect: ANSI_SQL expression: ss_sales_price + datatype: Decimal description: Sales price per unit ai_context: synonyms: - unit price - price - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: ss_ext_sales_price expression: dialects: - dialect: ANSI_SQL expression: ss_ext_sales_price + datatype: Decimal description: Extended sales price (quantity * price) ai_context: synonyms: - total price - line total - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: ss_net_profit expression: dialects: - dialect: ANSI_SQL expression: ss_net_profit + datatype: Decimal description: Net profit from the sale ai_context: synonyms: - profit - margin - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: ss_ticket_number expression: dialects: @@ -213,10 +185,8 @@ semantic_model: dialects: - dialect: ANSI_SQL expression: d_date_sk + datatype: Decimal description: Surrogate key for date - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: d_date expression: dialects: @@ -235,13 +205,11 @@ semantic_model: dialects: - dialect: ANSI_SQL expression: d_year + datatype: Decimal description: Year ai_context: synonyms: - year - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: d_quarter_name expression: dialects: @@ -285,10 +253,8 @@ semantic_model: dialects: - dialect: ANSI_SQL expression: c_customer_sk + datatype: Decimal description: Surrogate key for customer - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: c_customer_id expression: dialects: @@ -354,10 +320,8 @@ semantic_model: dialects: - dialect: ANSI_SQL expression: i_item_sk + datatype: Decimal description: Surrogate key for item - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: i_item_id expression: dialects: @@ -408,14 +372,12 @@ semantic_model: dialects: - dialect: ANSI_SQL expression: i_current_price + datatype: Decimal description: Current price of the item ai_context: synonyms: - price - list price - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' primary_key: - i_item_sk - name: store @@ -434,10 +396,8 @@ semantic_model: dialects: - dialect: ANSI_SQL expression: s_store_sk + datatype: Decimal description: Surrogate key for store - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: s_store_id expression: dialects: @@ -487,14 +447,12 @@ semantic_model: dialects: - dialect: ANSI_SQL expression: s_number_employees + datatype: Decimal description: Number of employees at the store ai_context: synonyms: - employee count - staff size - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' primary_key: - s_store_sk metrics: @@ -511,7 +469,7 @@ semantic_model: - sales amount custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "cube": "store_sales", "sql": "{CUBE.ss_ext_sales_price}"}' + data: '{"_v": 1, "cube": "store_sales"}' - name: total_profit expression: dialects: @@ -525,7 +483,7 @@ semantic_model: - profit custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "cube": "store_sales", "sql": "{CUBE.ss_net_profit}"}' + data: '{"_v": 1, "cube": "store_sales"}' - name: customer_lifetime_value expression: dialects: @@ -541,7 +499,7 @@ semantic_model: custom_extensions: - vendor_name: CUBE data: '{"_v": 1, "cube": "store_sales", "measure": {"name": "customer_lifetime_value", - "sql": "SUM({CUBE.ss_ext_sales_price}) / COUNT(DISTINCT {customer.c_customer_sk})", + "sql": "SUM({CUBE}.ss_ext_sales_price) / COUNT(DISTINCT {customer.c_customer_sk})", "type": "number"}}' - name: sales_by_brand expression: @@ -556,7 +514,7 @@ semantic_model: - brand revenue custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "cube": "store_sales", "sql": "{CUBE.ss_ext_sales_price}"}' + data: '{"_v": 1, "cube": "store_sales"}' - name: store_productivity expression: dialects: @@ -572,7 +530,7 @@ semantic_model: custom_extensions: - vendor_name: CUBE data: '{"_v": 1, "cube": "store_sales", "measure": {"name": "store_productivity", - "sql": "SUM({CUBE.ss_ext_sales_price}) / NULLIF(SUM({store.s_number_employees}), + "sql": "SUM({CUBE}.ss_ext_sales_price) / NULLIF(SUM({store.s_number_employees}), 0)", "type": "number"}}' custom_extensions: - vendor_name: CUBE diff --git a/converters/cube/tests/test_cube_to_osi.py b/converters/cube/tests/test_cube_to_osi.py index fcf2335e..aa9c7874 100644 --- a/converters/cube/tests/test_cube_to_osi.py +++ b/converters/cube/tests/test_cube_to_osi.py @@ -111,13 +111,15 @@ def test_dimension_types_map_to_datatypes(model_a): assert fields["created_at"]["dimension"]["is_time"] is True -def test_number_dimension_asserts_no_datatype(model_a): - """Cube collapses Integer/Decimal/Float into `number`, so the converter omits - `datatype` rather than assert a precision the model does not carry.""" +def test_number_dimension_maps_to_a_native_datatype(model_a): + """Cube collapses Integer/Decimal/Float into `number`, so no mapping back is + exact. `Decimal` is asserted anyway, because a downstream converter can act on it + -- where omitting it and stashing Cube's `type` in a custom_extension gave every + other spoke a warning and nothing else.""" model, _ = model_a fields = by_name(by_name(model["datasets"])["orders"]["fields"]) - assert "datatype" not in fields["id"] - assert stash_of(fields["id"])["type"] == "number" + assert fields["id"]["datatype"] == "Decimal" + assert stash_of(fields["id"]) == {} def test_dimension_title_becomes_label_and_ai_context_maps(model_a): diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index e7e92beb..eebf5d6b 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -878,7 +878,7 @@ def test_a_geo_half_reference_is_requalified_when_it_crosses_cubes(): # `{users}.lat` names the cube explicitly, since `{CUBE}` here would mean # `orders`. `{CUBE.amount}` stays a member reference because `amount` is a # declared field of the cube the measure lands on. - assert measures[0]["sql"] == "AVG({users}.lat) - MIN({CUBE.amount})" + assert measures[0]["sql"] == "AVG({users}.lat) - MIN({CUBE}.amount)" assert measures[0]["type"] == "number" diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index f2ee960b..b1e128e7 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -349,14 +349,14 @@ def _metric(name, expr): @pytest.mark.parametrize("expr,expected", [ - ("SUM(orders.amount)", {"type": "sum", "sql": "{CUBE.amount}"}), - ("AVG(orders.amount)", {"type": "avg", "sql": "{CUBE.amount}"}), - ("MIN(orders.amount)", {"type": "min", "sql": "{CUBE.amount}"}), - ("MAX(orders.amount)", {"type": "max", "sql": "{CUBE.amount}"}), + ("SUM(orders.amount)", {"type": "sum", "sql": "{CUBE}.amount"}), + ("AVG(orders.amount)", {"type": "avg", "sql": "{CUBE}.amount"}), + ("MIN(orders.amount)", {"type": "min", "sql": "{CUBE}.amount"}), + ("MAX(orders.amount)", {"type": "max", "sql": "{CUBE}.amount"}), ("COUNT(DISTINCT orders.amount)", - {"type": "count_distinct", "sql": "{CUBE.amount}"}), + {"type": "count_distinct", "sql": "{CUBE}.amount"}), ("APPROX_COUNT_DISTINCT(orders.amount)", - {"type": "count_distinct_approx", "sql": "{CUBE.amount}"}), + {"type": "count_distinct_approx", "sql": "{CUBE}.amount"}), ]) def test_aggregate_expressions_become_structured_measures(expr, expected): files, _ = convert_ossie_to_cube(_ossie(_ORDERS, metrics=_metric("m", expr))) @@ -389,7 +389,7 @@ def test_ratio_becomes_a_calculated_measure(): _metric("aov", "SUM(orders.amount) / COUNT(DISTINCT users.id)"))) measure = _cubes(files)["orders"]["measures"][0] assert measure["type"] == "number" - assert measure["sql"] == "SUM({CUBE.amount}) / COUNT(DISTINCT {users.id})" + assert measure["sql"] == "SUM({CUBE}.amount) / COUNT(DISTINCT {users.id})" assert any("spans several datasets" in i.detail for i in issues.of_type(IssueType.APPROXIMATED)) From 1de9e118eb97f9dd7b034b768f183fc6c6c081b7 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Mon, 3 Aug 2026 16:27:26 +0500 Subject: [PATCH 20/46] Report a source that other Ossie converters will reject Found by running Cube -> Ossie -> every other spoke. A Cube `sql_table` of `public.orders` is perfectly ordinary and converts cleanly here, but three converters reject the resulting Ossie source outright: databricks source 'public.orders' must be a 3-part catalog.schema.table gsf source must resolve to database.schema.table snowflake must be a fully qualified db.schema.table or a subquery So a model can pass every test in this converter and still be unable to reach half the ecosystem. Import now reports SOURCE_NOT_FULLY_QUALIFIED, naming the converters and what to change, at the point the Ossie document is produced rather than three hops later. Deliberately a report and not a fix: inventing a catalog name would be guessing at the user's warehouse. Cube's own `sql_table` is legitimately one- or two-part, so this is a portability limit of the hub rather than a defect on either side -- which is why it gets its own issue type instead of being folded into the loss categories. Dots inside quoted identifiers are not path separators, so `"My.Catalog".public.orders` counts as three parts, not four. 261 tests, 96% line coverage. `git diff --check` clean. Co-Authored-By: Claude Opus 5 --- converters/cube/README.md | 27 ++++++++++++++ converters/cube/src/ossie_cube/_common.py | 21 +++++++++++ .../cube/src/ossie_cube/converter_issues.py | 6 ++++ converters/cube/src/ossie_cube/cube_to_osi.py | 14 ++++++++ converters/cube/tests/test_edge_cases.py | 35 +++++++++++++++++++ 5 files changed, 103 insertions(+) diff --git a/converters/cube/README.md b/converters/cube/README.md index ecb19241..4ca68bac 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -220,6 +220,32 @@ AVG(users.home_latitude) - MIN(orders.amt) -> sql: AVG({users}.lat) - MIN({CUB One documented normalization follows: after a round trip such a metric names the column the half actually reads (`users.lat`) rather than the Ossie-only field name (`users.home_latitude`). Same reference, and it is the form Cube can express. +## Onward conversion + +Ossie is a hub, so the useful question is not only whether `Cube → Ossie → Cube` +round-trips but whether the Ossie model then reaches the other spokes. Two things +matter in practice. + +**Keep Cube-only detail out of `custom_extensions`.** Converters that do not read +foreign extensions warn about and discard every one, so anything placed there is +noise to them. This converter therefore stashes only what is genuinely Cube-specific +— segments, pre-aggregations, hierarchies, view curation, geo reconstruction — and +maps everything else natively. On the TPC-DS model that is 7 stash entries rather +than 41, and 2 Databricks warnings rather than 32. + +**Qualify your `sql_table`.** Cube accepts `orders` or `public.orders`, but the +Databricks, Snowflake and NVIDIA GSF converters all require a three-part +`catalog.schema.table` and reject anything shorter: + +``` +Error: Dataset 'orders': source 'public.orders' must be a 3-part catalog.schema.table +Error: Dataset 'orders' source must resolve to database.schema.table +Error: Source 'public.orders' must be a fully qualified db.schema.table or a subquery +``` + +Import reports this as `SOURCE_NOT_FULLY_QUALIFIED` rather than guessing a catalog +name, so it surfaces where the Ossie document is produced instead of three hops later. + ## Conversion issues `convert_cube_to_ossie` returns `(yaml, IssueLog)`. Each issue carries a type, the @@ -233,6 +259,7 @@ element it concerns, and a detail string. | `GEO_DIMENSION_SPLIT` | A `type: geo` dimension became two Ossie fields | | `TEMPLATED_FILE_SKIPPED` | Jinja templating anywhere in a file, or a `.js`/`.ts` model file. Detected per file, as Cube's own tooling does, so the file is preserved whole rather than half-converted | | `NO_USABLE_DIALECT` | Export: no `ANSI_SQL` or preferred-dialect expression | +| `SOURCE_NOT_FULLY_QUALIFIED` | A `sql_table` shorter than `catalog.schema.table`. Valid Cube and nothing is lost, but the Databricks, Snowflake and NVIDIA GSF converters reject such a source, so the model cannot convert onward — see [Onward conversion](#onward-conversion) | | `PARKED_IN_META` | Preserved in the stash or under `meta.ossie` — invisible to Cube, but intact through a round trip | | `DROPPED_NO_CUBE_EQUIVALENT` | **Gone from the output.** Cube has nowhere to hold it and it cannot be parked: relationship `ai_context` (a Cube join entry has no `meta`), a `dimension.is_time` role or opt-out that Cube expresses only through `type`, and the second and later `semantic_model` entries | | `APPROXIMATED` | Emitted, but not an exact equivalent: a value Cube requires and Ossie does not carry (so the converter chose one), or a construct rendered in the nearest form Cube has | diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index b25aa2ca..6ca0666a 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -385,6 +385,27 @@ def repl(m): return out, changed +def source_part_count(source): + """How many identifier parts a dotted dataset `source` has, or None for a query. + + Dots inside double quotes or backticks belong to a quoted identifier, not to the + path -- `"My.Catalog".public.t` is three parts, not four. + """ + s = str(source).strip() + if re.match(r"(?i)(select|with)\b", s): + return None + parts, quote = 1, None + for ch in s: + if quote: + if ch == quote: + quote = None + elif ch in '"`': + quote = ch + elif ch == ".": + parts += 1 + return parts + + def sql_is_reversible(sql, plain_members=(), own_cube=None): """True if translating this Cube SQL to Ossie and back reproduces it. diff --git a/converters/cube/src/ossie_cube/converter_issues.py b/converters/cube/src/ossie_cube/converter_issues.py index 93df8842..f0306d2f 100644 --- a/converters/cube/src/ossie_cube/converter_issues.py +++ b/converters/cube/src/ossie_cube/converter_issues.py @@ -62,6 +62,12 @@ class IssueType(Enum): # An Ossie field or metric with no usable expression dialect (export). NO_USABLE_DIALECT = "NO_USABLE_DIALECT" + # A dataset `source` that is a valid Cube `sql_table` but not a three-part + # `catalog.schema.table`. Nothing is lost and Cube is happy, but several other + # Ossie converters reject such a source outright, so the model will not travel + # past this hub. Reported so that is discovered here rather than downstream. + SOURCE_NOT_FULLY_QUALIFIED = "SOURCE_NOT_FULLY_QUALIFIED" + # An Ossie construct Cube has no slot for, parked under `meta.ossie` -- so the # value survives the round trip even though Cube itself cannot read it. PARKED_IN_META = "PARKED_IN_META" diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index c2e5f0f1..c5bb38f3 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -56,6 +56,7 @@ require_str, snake, snake_keys, + source_part_count, sql_is_reversible, view_file, read_stash, @@ -469,6 +470,19 @@ def _convert_cube(cname, cube, extra_joins, extra_measures, issues): stash = {} ds["source"] = join_source(cube, cname) + parts = source_part_count(ds["source"]) + if parts is not None and parts < 3: + # Cube accepts a one- or two-part `sql_table`, but the Ossie spec describes + # `source` as `database.schema.table` and the Databricks, Snowflake and NVIDIA + # GSF converters all reject anything shorter -- so a model that converts + # cleanly here still cannot reach them. Better to say so at the point the + # Ossie document is produced than to have it fail three hops later. + ds_scope = f"cube '{cname}'" + issues.add(IssueType.SOURCE_NOT_FULLY_QUALIFIED, ds_scope, + f"source '{ds['source']}' has {parts} part(s); several Ossie " + f"converters (Databricks, Snowflake, NVIDIA GSF) require a " + f"3-part catalog.schema.table, so qualify the cube's `sql_table` " + f"if the model needs to convert onward") if cube.get("description"): ds["description"] = cube["description"] diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index eebf5d6b..0577528b 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -291,6 +291,41 @@ def test_field_and_metric_foreign_extensions_survive_the_round_trip(): assert metric["custom_extensions"][0]["vendor_name"] == "CUBE" +@pytest.mark.parametrize("sql_table,parts,warns", [ + ("orders", 1, True), + ("public.orders", 2, True), + ("tpcds.public.orders", 3, False), + ('"My.Catalog".public.orders', 3, False), # dots inside quotes are not parts + ("a.b.c.d", 4, False), +]) +def test_a_source_that_other_converters_reject_is_reported(sql_table, parts, warns): + """Cube is happy with a one- or two-part `sql_table`, but the Databricks, + Snowflake and NVIDIA GSF converters all reject a source shorter than + `catalog.schema.table` -- so a model that converts cleanly here still cannot + travel. Reported at the point the Ossie document is produced, rather than being + discovered three hops later.""" + files = _files(orders=( + "cubes:\n" + " - name: orders\n" + # Single-quoted so a value containing double quotes stays one YAML scalar. + f" sql_table: '{sql_table}'\n")) + _, issues = convert_cube_to_ossie(files) + reported = issues.of_type(IssueType.SOURCE_NOT_FULLY_QUALIFIED) + assert bool(reported) is warns + if warns: + assert f"{parts} part(s)" in reported[0].detail + + +def test_a_sql_defined_cube_is_not_reported_as_unqualified(): + """A `sql:` cube is a query, not a table path, and every converter accepts one.""" + files = _files(orders=( + "cubes:\n" + " - name: orders\n" + " sql: SELECT * FROM public.orders\n")) + _, issues = convert_cube_to_ossie(files) + assert not issues.of_type(IssueType.SOURCE_NOT_FULLY_QUALIFIED) + + # --- join orientation, both ways ------------------------------------------------ _ONE_TO_MANY = _files(m=( From de748f15444486fd764e6a7cf8186d5000eccba2 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Mon, 3 Aug 2026 16:46:17 +0500 Subject: [PATCH 21/46] Split a composite metric into one measure per aggregate An Ossie metric such as SUM(store_sales.amount) / COUNT(DISTINCT customer.id) used to become a single Cube measure of `type: number` holding the whole expression. Cube corrects row multiplication per measure, keyed on the cube the measure sits on, so one calculated measure gets one correction for an expression whose aggregates read different cubes. Export now emits one `public: false` measure per aggregate, each declared on the cube its own operand reads, plus the public measure referencing them. Each aggregate is then corrected on its own terms. Parts carry `meta.ossie.part_of`, and import skips them and inlines their SQL back through the references, so the original expression is recovered exactly. Locating the aggregates uses sqlglot rather than a regex, since an expression can nest them -- SUM(x) / NULLIF(SUM(y), 0). It is already a runtime dependency of the dbt and NVIDIA GSF converters for the same purpose. Also drops the parentheses import used to add around every inlined measure reference: a lone aggregate is one term already, and keeping them off is what makes the round trip exact. --- converters/cube/README.md | 22 ++- converters/cube/pyproject.toml | 4 + converters/cube/src/ossie_cube/cube_to_osi.py | 23 ++- converters/cube/src/ossie_cube/expressions.py | 176 ++++++++++++++++++ converters/cube/src/ossie_cube/osi_to_cube.py | 78 +++++++- .../cube/tests/fixtures/fixtureA_ossie.yaml | 2 +- converters/cube/tests/test_cube_to_osi.py | 4 +- converters/cube/tests/test_edge_cases.py | 47 +++-- converters/cube/tests/test_osi_to_cube.py | 51 ++++- converters/cube/uv.lock | 15 +- 10 files changed, 390 insertions(+), 32 deletions(-) create mode 100644 converters/cube/src/ossie_cube/expressions.py diff --git a/converters/cube/README.md b/converters/cube/README.md index 4ca68bac..71897937 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -55,7 +55,9 @@ pip install apache-ossie-cube # once published to PyPI pip install -e . ``` -The only runtime dependency is `PyYAML`. Python 3.11+. +Runtime dependencies are `PyYAML` and `sqlglot` (already a runtime dependency of +the dbt and NVIDIA GSF converters, used here to locate the aggregate calls inside a +composite metric). Python 3.11+. ## Usage @@ -125,7 +127,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | `dataset.unique_keys` | `meta.ossie.unique_keys` | No native Cube slot; parked rather than dropped. | | field | `dimensions[]` entry | Export: a name that is not a valid Cube identifier is sanitized; a case-insensitive collision is an error, never a silent merge. | | `field.expression` | dimension `sql` | Dataset-scoped, so `{CUBE}.col` <-> `col`. Export emits `{CUBE}.column` for a raw column and `{CUBE.member}` for a declared member, and never spells the cube's own name (which would break under `extends`). | -| `field.datatype` | dimension `type` (**required**) | `String`->`string`, `Boolean`->`boolean`, `Date`/`Time`/`DateTime`/`DateTimeTz`->`time`, `Integer`/`Decimal`/`Float`->`number`, `Opaque`->`string`. Import maps back except for `number`, where it **omits `datatype`** -- Cube collapses three Ossie types into one, and the spec says to omit rather than assert. The original `type` is stashed. | +| `field.datatype` | dimension `type` (**required**) | `String`->`string`, `Boolean`->`boolean`, `Date`/`Time`/`DateTime`/`DateTimeTz`->`time`, `Integer`/`Decimal`/`Float`->`number`, `Opaque`->`string`. Import maps back, choosing `Decimal` for `number` -- Cube collapses three Ossie types into one, so any single answer is a guess, and a stated datatype is what another converter can act on. Export parks the exact one in `meta.ossie.datatype`, which import prefers when present, so `Integer` and `Float` still survive a round trip. | | `field.dimension.is_time` | `type: time` | Import sets `is_time: true` for a time dimension. | | `field.label` / `description` | dimension `title` / `description` | | | `field.ai_context.instructions` | dimension `meta.ai_context` | Cube's documented AI-only context field. | @@ -137,6 +139,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | `COUNT(DISTINCT x)` | `type: count_distinct` | | | `APPROX_COUNT_DISTINCT(x)` | `type: count_distinct_approx` | Cube resolves the warehouse-specific function itself. | | `COUNT(DISTINCT )` | bare `type: count` | See [Fan-out](#fan-out) -- the primary key is load-bearing here. | +| several aggregates in one expression | one `public: false` measure per aggregate + a `type: number` measure referencing them | Each part is declared on the cube its own operand reads, so Cube corrects row multiplication per aggregate rather than once for the whole expression. The parts carry `meta.ossie.part_of`, and import skips them and inlines their SQL back through the references -- recovering the original expression exactly. | | anything else | `type: number` (calculated) | A `{other_measure}` reference is **inlined**, because that is what Cube itself does; Ossie has no metric-to-metric reference. | | — | measure `filters` | Folded into `CASE WHEN … THEN … END` inside the aggregate, exactly as Cube's own `applyMeasureFilters` renders it. | | `metric.datatype` | — | Import emits `Integer` for the count family, whose result type Cube does know, and omits it otherwise. | @@ -191,6 +194,21 @@ Because a bare `count` maps through the primary key, a cube carrying one **must* declare `primary_key: true` on a dimension; its absence is an error, not a different number. +Going the other way, an Ossie metric combining several aggregates is **decomposed** +rather than emitted as one calculated measure, so Cube's correction applies to each +aggregate on its own cube: + +```yaml +# Ossie # Cube +SUM(store_sales.amount) store_sales: clv_part_1 (sum, public: false) + / COUNT(DISTINCT customer.id) customer: clv_part_2 (count, public: false) + store_sales: clv = {CUBE.clv_part_1} + / {customer.clv_part_2} +``` + +A single aggregate reading two datasets cannot be split this way and still lands on +one cube. + > Ossie has no additivity or grain declaration to record this properly -- dbt's > `non_additive_dimension` is the nearest precedent, and this repo's dbt converter > already loses the same information. Worth raising on `dev@`. diff --git a/converters/cube/pyproject.toml b/converters/cube/pyproject.toml index a78e12b8..72a087d4 100644 --- a/converters/cube/pyproject.toml +++ b/converters/cube/pyproject.toml @@ -38,6 +38,10 @@ classifiers = [ ] dependencies = [ "PyYAML>=6.0", + # Expression handling: locating the aggregate calls inside a composite metric so + # each can become its own Cube measure. Already a runtime dependency of the dbt + # and NVIDIA GSF converters, which use it for the same purpose. + "sqlglot>=20.0", ] [dependency-groups] diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index c5bb38f3..136cf114 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -63,6 +63,7 @@ write_stash, ) from .converter_issues import IssueLog, IssueType +from .expressions import has_top_level_operator # Cube keys the converter maps natively at the cube level; everything else is # stashed verbatim in the dataset's `cube_extras` and restored on export. @@ -903,7 +904,10 @@ def _inline(self, body, cname, stack): raise ConversionError( f"measure '{cname}': references '{target_cube}.{target_name}', " f"which has no static Ossie form") - return f"({inner})" + # A lone `SUM(x)` needs no parentheses; only a term with its own top-level + # operators does. Keeping them off means a decomposed metric inlines back to + # exactly the expression it was split from. + return f"({inner})" if has_top_level_operator(inner) else inner def _operand(self, cname, sql, stack): """Translate an aggregate's operand into an Ossie reference. @@ -919,6 +923,12 @@ def _operand(self, cname, sql, stack): return translated +def _is_generated_part(measure): + """True for a `public: false` measure a previous export created to hold one + aggregate of a composite metric (marked `meta.ossie.part_of`).""" + return bool(((measure.get("meta") or {}).get("ossie") or {}).get("part_of")) + + def _convert_measures(cubes, pk_by_cube, fanned_out, issues): """Hoist every cube's measures into Ossie model-level metrics. @@ -935,8 +945,9 @@ def _convert_measures(cubes, pk_by_cube, fanned_out, issues): resolver = _MeasureResolver(cubes, pk_by_cube, issues) counts = {} - for (_, mname) in resolver.measures(): - counts[mname] = counts.get(mname, 0) + 1 + for (cname, mname), measure in resolver.measures().items(): + if not _is_generated_part(measure): + counts[mname] = counts.get(mname, 0) + 1 metrics = [] extra_measures = {} @@ -946,6 +957,12 @@ def _convert_measures(cubes, pk_by_cube, fanned_out, issues): _as_named_list(cube.get("measures"), f"cube '{cname}' measures")): mname = measure["name"] + if _is_generated_part(measure): + # Emitted by a previous export to split a composite metric across + # cubes. It has no Ossie metric of its own -- the public measure's + # references inline back to the whole expression -- and export + # regenerates it, so it is not stashed either. + continue metric_name = mname if counts[mname] == 1 else f"{cname}__{mname}" if metric_name in seen: raise ConversionError( diff --git a/converters/cube/src/ossie_cube/expressions.py b/converters/cube/src/ossie_cube/expressions.py new file mode 100644 index 00000000..68243c0f --- /dev/null +++ b/converters/cube/src/ossie_cube/expressions.py @@ -0,0 +1,176 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Reading the structure of an Ossie metric expression. + +Cube expects a measure to *be* an aggregation -- `type: sum` over a column -- and +falls back to a calculated `type: number` measure whose sql carries the whole +aggregate. A composite Ossie metric such as + + SUM(store_sales.amount) / COUNT(DISTINCT customer.id) + +can be emitted either way, and the difference matters: as one calculated measure +Cube sees a single opaque expression, whereas as two `public: false` measures on +their own cubes plus a ratio referencing them, **Cube applies its row-multiplication +correction to each aggregate independently**. So decomposition is a correctness +improvement for cross-dataset metrics, not a formatting choice. + +Locating the aggregate calls is done with sqlglot rather than a regex, since an +expression can nest them (`SUM(x) / NULLIF(SUM(y), 0)`) and string matching cannot +tell a top-level call from one inside another argument. sqlglot is already a runtime +dependency of the dbt and NVIDIA GSF converters for the same purpose. +""" + +import sqlglot +import sqlglot.expressions as exp + +# sqlglot node types for the aggregates this converter maps to a Cube measure type. +# `Count` covers COUNT / COUNT(DISTINCT x); ApproxDistinct covers +# APPROX_COUNT_DISTINCT. +_AGGREGATE_NODES = ( + exp.Sum, exp.Avg, exp.Min, exp.Max, exp.Count, exp.ApproxDistinct, +) + + +def parse(expr): + """Parse an Ossie expression, or None when sqlglot cannot. + + An unparseable expression is not an error: the converter falls back to treating + it as one opaque calculated measure, which is what it did for everything before. + """ + try: + return sqlglot.parse_one(str(expr).strip()) + except Exception: + return None + + +def is_single_aggregate(expr): + """True if the whole expression is exactly one aggregate call. + + Those already map to a structured Cube measure (`type: sum` + `sql`), so they + are never decomposed. + """ + tree = parse(expr) + return tree is not None and isinstance(tree, _AGGREGATE_NODES) + + +# The aggregate call names this converter maps to a Cube measure type. Scanned for +# in the source text: sqlglot renames some when it renders (`APPROX_COUNT_DISTINCT` +# comes back as `APPROX_DISTINCT`), and two calls of the same name render +# identically, so node text cannot be used to find them in the original string. +_AGGREGATE_NAMES = ( + "APPROX_COUNT_DISTINCT", "APPROX_DISTINCT", + "COUNT", "SUM", "AVG", "MIN", "MAX", +) + + +def aggregate_spans(expr): + """The outermost aggregate calls in `expr`, as (start, end) offsets. + + Offsets index the original string so a caller can substitute each span in place. + That matters because the surrounding text may carry Cube `{...}` references, + which sqlglot would not reproduce verbatim if the expression were re-rendered. + + Spans are found by scanning for an aggregate name followed by a balanced + parenthesis group, then confirmed with sqlglot -- which is also what rules out a + malformed expression. Nesting is resolved on the offsets themselves: a span + inside another span is not returned, so `SUM(x) / NULLIF(SUM(y), 0)` gives two + and `SUM(SUM(x))` gives one. Returns [] when the expression does not parse, or is + itself a single aggregate needing no decomposition. + """ + text = str(expr) + if parse(text) is None or is_single_aggregate(text): + return [] + + candidates = [] + upper = text.upper() + for name in _AGGREGATE_NAMES: + at = 0 + while True: + at = upper.find(name, at) + if at < 0: + break + start, after = at, at + len(name) + at = after + # A call, not part of a longer identifier: boundary before, `(` after. + if start and (text[start - 1].isalnum() or text[start - 1] == "_"): + continue + probe = after + while probe < len(text) and text[probe].isspace(): + probe += 1 + if probe >= len(text) or text[probe] != "(": + continue + close = _match_paren(text, probe) + if close is None: + continue + end = close + 1 + # Confirm the slice really is an aggregate and not, say, a UDF that + # happens to share a prefix. + node = parse(text[start:end]) + if isinstance(node, _AGGREGATE_NODES): + candidates.append((start, end)) + + # Drop any span contained within another: only the outermost becomes a measure. + candidates.sort() + out = [] + for start, end in candidates: + if any(s <= start and end <= e for s, e in out): + continue + out.append((start, end)) + return out + + +def _match_paren(text, open_at): + """Index of the `)` closing the `(` at `open_at`, honouring quotes.""" + depth, quote = 0, None + for i in range(open_at, len(text)): + ch = text[i] + if quote: + if ch == quote: + quote = None + elif ch in "'\"": + quote = ch + elif ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth == 0: + return i + return None + + +def has_top_level_operator(expr): + """True if `expr` is not a single self-contained term. + + Used to decide whether inlining it back into a larger expression needs + parentheses: a lone `SUM(x)` does not, `SUM(x) / 2` does. + """ + depth, quote = 0, None + for ch in str(expr): + if quote: + if ch == quote: + quote = None + elif ch in "'\"": + quote = ch + elif ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + elif depth == 0 and (ch in "+-*/%<>=|&" or ch.isspace()): + # Whitespace at depth 0 also implies structure (`CASE WHEN ...`). + return True + return False diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index beaee615..889b3fce 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -36,6 +36,7 @@ from ._common import ( DATATYPE_TO_DIM_TYPE, + DOTTED_REF_RE, DEFAULT_DATATYPE_FOR_CUBE_TYPE, OSSIE_FUNC_TO_AGG, OSSIE_VERSION, @@ -58,6 +59,7 @@ view_file, ) from .converter_issues import IssueLog, IssueType +from .expressions import aggregate_spans # An aggregate call the exporter can turn back into a structured Cube measure. _AGG_CALL_RE = re.compile( @@ -666,15 +668,83 @@ def resolve_base(): } target = stash.get("cube") or ( next(iter(referenced)) if len(referenced) == 1 else resolve_base()) - measure = _measure_from_expression( - expr, target, mname, stash, members_by_cube.get(target, set()), - inline_sql_by_cube, pk_by_cube.get(target, []), sanitized, scope, - issues) + + spans = [] if stash.get("sql") else aggregate_spans(expr) + if len(spans) > 1: + # A composite metric: give each aggregate its own measure on the cube its + # operand belongs to, and let the public measure reference them. Cube then + # applies its row-multiplication correction per aggregate instead of + # seeing one opaque expression -- see _decompose_measure. + public_sql = _decompose_measure( + expr, spans, mname, target, measures_by_cube, members_by_cube, + inline_sql_by_cube, pk_by_cube, sanitized, name, scope, issues) + measure = {"name": mname, "sql": public_sql, "type": "number"} + else: + measure = _measure_from_expression( + expr, target, mname, stash, members_by_cube.get(target, set()), + inline_sql_by_cube, pk_by_cube.get(target, []), sanitized, scope, + issues) _apply_measure_metadata(metric, measure, stash) _place(measures_by_cube, target, measure, name) return measures_by_cube +def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, + members_by_cube, inline_sql_by_cube, pk_by_cube, sanitized, + model_name, scope, issues): + """Emit one `public: false` measure per aggregate; return the sql referencing them. + + Cube corrects for row multiplication per measure, keyed on the cube that measure + sits on. A cross-dataset ratio emitted as a single calculated measure gets one + correction for the whole expression; split into a measure per aggregate, each on + the cube its operand comes from, each aggregate is corrected on its own terms. + That is why this is a correctness change and not a formatting one. + + Each part carries `meta.ossie.part_of` so import knows it is generated and skips + it, recovering the original expression by inlining the references instead. + """ + # A part name has to be free on whichever cube it lands on, and a Cube member name + # is unique across dimensions and measures alike -- so the check is over both, and + # over every cube rather than the one part it happens to land on. + taken = {m["name"].lower() for ms in measures_by_cube.values() for m in ms} + taken |= {n.lower() for ns in members_by_cube.values() for n in ns} + out, cursor, index = [], 0, 0 + for start, end in spans: + piece = expr[start:end] + # Each aggregate lands on the cube its own operand references. + refs = {m.group(1) for m in DOTTED_REF_RE.finditer(piece) + if m.group(1) in sanitized} + part_target = next(iter(refs)) if len(refs) == 1 else fallback + + index += 1 + part_name = f"{mname}_part_{index}" + while part_name.lower() in taken: + index += 1 + part_name = f"{mname}_part_{index}" + taken.add(part_name.lower()) + + part = _measure_from_expression( + piece, part_target, part_name, {}, + members_by_cube.get(part_target, set()), inline_sql_by_cube, + pk_by_cube.get(part_target, []), sanitized, scope, issues) + part["public"] = False + part["meta"] = {"ossie": {"part_of": mname}} + _place(measures_by_cube, part_target, part, model_name) + + out.append(ossie_expr_to_cube_sql( + expr[cursor:start], fallback, members_by_cube.get(fallback, set()), + sanitized, inline_sql=inline_sql_by_cube)) + # `{CUBE.x}` for a part on the same cube as the public measure: an explicit + # name pins the reference to this cube and breaks if it is extended. + qualifier = "CUBE" if part_target == fallback else part_target + out.append("{" + f"{qualifier}.{part_name}" + "}") + cursor = end + out.append(ossie_expr_to_cube_sql( + expr[cursor:], fallback, members_by_cube.get(fallback, set()), sanitized, + inline_sql=inline_sql_by_cube)) + return "".join(out) + + def _place(measures_by_cube, target, measure, model_name): bucket = measures_by_cube.setdefault(target, []) if any(m["name"].lower() == measure["name"].lower() for m in bucket): diff --git a/converters/cube/tests/fixtures/fixtureA_ossie.yaml b/converters/cube/tests/fixtures/fixtureA_ossie.yaml index 96efcf5d..577fd8ed 100644 --- a/converters/cube/tests/fixtures/fixtureA_ossie.yaml +++ b/converters/cube/tests/fixtures/fixtureA_ossie.yaml @@ -151,7 +151,7 @@ semantic_model: expression: dialects: - dialect: ANSI_SQL - expression: (SUM(orders.amount)) / (COUNT(DISTINCT orders.id)) + expression: SUM(orders.amount) / COUNT(DISTINCT orders.id) custom_extensions: - vendor_name: CUBE data: '{"_v": 1, "cube": "orders", "measure": {"name": "avg_order_value", "sql": diff --git a/converters/cube/tests/test_cube_to_osi.py b/converters/cube/tests/test_cube_to_osi.py index aa9c7874..a3609e98 100644 --- a/converters/cube/tests/test_cube_to_osi.py +++ b/converters/cube/tests/test_cube_to_osi.py @@ -326,7 +326,9 @@ def test_calculated_measure_inlines_its_measure_references(model_a): aggregate SQL; Ossie has no metric-to-metric reference, so it is inlined.""" model, _ = model_a metric = by_name(model["metrics"])["avg_order_value"] - assert expr_of(metric) == "(SUM(orders.amount)) / (COUNT(DISTINCT orders.id))" + # No redundant parentheses: a lone aggregate is already a single term, so an + # inlined reference reads exactly as the expression it stands for. + assert expr_of(metric) == "SUM(orders.amount) / COUNT(DISTINCT orders.id)" def test_measure_reference_cycle_is_rejected(): diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 0577528b..6cdd1b50 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -893,10 +893,9 @@ def test_a_metric_referencing_a_geo_half_inlines_its_sql(): "longitude": {"sql": "{CUBE}.lon"}}] -def test_a_geo_half_reference_is_requalified_when_it_crosses_cubes(): - """`{CUBE}` means "the cube this is declared on", so inlining a snippet into - another cube's SQL has to name the original cube explicitly.""" - model = _GEO_MODEL.replace( +def _two_cube_geo_model(expression): + """`_GEO_MODEL` plus an `orders.amount` field, and `expression` as the metric.""" + return _GEO_MODEL.replace( " - name: users\n", " - name: orders\n source: public.orders\n" " fields:\n" " - name: amount\n" @@ -907,14 +906,40 @@ def test_a_geo_half_reference_is_requalified_when_it_crosses_cubes(): " datatype: Decimal\n" " - name: users\n", 1 ).replace(" expression: AVG(users.home_latitude)\n", - " expression: AVG(users.home_latitude) - MIN(orders.amount)\n") + f" expression: {expression}\n") + + +def test_a_geo_half_reference_is_requalified_when_it_crosses_cubes(): + """`{CUBE}` means "the cube this is declared on", so inlining a snippet into + another cube's SQL has to name the original cube explicitly. + + One aggregate reading two datasets cannot be decomposed, so it lands on the base + cube and the `users` half travels there with it. + """ + model = _two_cube_geo_model("AVG(users.home_latitude - orders.amount)") + files, _ = convert_ossie_to_cube(model, base_cube="orders") + cube = parse(files["model/cubes/orders.yml"])["cubes"][0] + assert cube["measures"] == [ + {"name": "avg_lat", "sql": "{users}.lat - {CUBE}.amount", "type": "avg"}] + + +def test_a_decomposed_part_lands_on_the_cube_its_operand_reads(): + """Two aggregates over two datasets: each part is declared on the cube it reads, + which is what lets Cube correct row multiplication for each independently. So the + geo half needs no requalification -- its part lives on `users` already.""" + model = _two_cube_geo_model( + "AVG(users.home_latitude) - MIN(orders.amount)") files, _ = convert_ossie_to_cube(model, base_cube="orders") - measures = parse(files["model/cubes/orders.yml"])["cubes"][0]["measures"] - # `{users}.lat` names the cube explicitly, since `{CUBE}` here would mean - # `orders`. `{CUBE.amount}` stays a member reference because `amount` is a - # declared field of the cube the measure lands on. - assert measures[0]["sql"] == "AVG({users}.lat) - MIN({CUBE}.amount)" - assert measures[0]["type"] == "number" + on_users = by_name(parse(files["model/cubes/users.yml"])["cubes"][0]["measures"]) + on_orders = by_name(parse(files["model/cubes/orders.yml"])["cubes"][0]["measures"]) + assert on_users["avg_lat_part_1"]["sql"] == "{CUBE}.lat" + assert on_users["avg_lat_part_1"]["public"] is False + assert on_orders["avg_lat_part_2"]["sql"] == "{CUBE}.amount" + # The public measure stays on the base cube, naming the foreign part by its cube + # and its own with `{CUBE.x}`. + assert on_orders["avg_lat"]["sql"] == ( + "{users.avg_lat_part_1} - {CUBE.avg_lat_part_2}") + assert "public" not in on_orders["avg_lat"] def test_geo_half_references_normalize_to_the_underlying_column(): diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index b1e128e7..985ebca5 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -18,9 +18,14 @@ """Apache Ossie semantic model -> Cube data model.""" import pytest -from _util import by_name, parse +from _util import by_name, expr_of, model_of, parse -from ossie_cube import ConversionError, IssueType, convert_ossie_to_cube +from ossie_cube import ( + ConversionError, + IssueType, + convert_cube_to_ossie, + convert_ossie_to_cube, +) from ossie_cube._common import OSSIE_VERSION @@ -383,15 +388,43 @@ def test_declared_member_gets_a_member_reference_and_a_raw_column_does_not(): assert _cubes(files)["orders"]["measures"][0]["sql"] == "{CUBE}.shipping_fee" -def test_ratio_becomes_a_calculated_measure(): - files, issues = convert_ossie_to_cube(_ossie( +def test_a_ratio_is_split_into_one_measure_per_aggregate(): + """Each aggregate becomes its own `public: false` measure on the cube its operand + comes from, and the public measure references them. Cube corrects for row + multiplication per measure, so splitting is what lets each aggregate be corrected + on its own cube instead of the whole ratio being one opaque expression.""" + files, _ = convert_ossie_to_cube(_ossie( _TWO_DATASETS, _REL, _metric("aov", "SUM(orders.amount) / COUNT(DISTINCT users.id)"))) - measure = _cubes(files)["orders"]["measures"][0] - assert measure["type"] == "number" - assert measure["sql"] == "SUM({CUBE}.amount) / COUNT(DISTINCT {users.id})" - assert any("spans several datasets" in i.detail - for i in issues.of_type(IssueType.APPROXIMATED)) + orders = by_name(_cubes(files)["orders"]["measures"]) + users = by_name(parse(files["model/cubes/users.yml"])["cubes"][0]["measures"]) + + assert orders["aov_part_1"] == { + "name": "aov_part_1", "sql": "{CUBE}.amount", "type": "sum", + "meta": {"ossie": {"part_of": "aov"}}, "public": False} + # `users.id` is that cube's primary key, so its aggregate is a bare Cube count -- + # the form Cube corrects for fan-out. + assert users["aov_part_2"] == { + "name": "aov_part_2", "type": "count", + "meta": {"ossie": {"part_of": "aov"}}, "public": False} + # `{CUBE.aov_part_1}` rather than `{orders.aov_part_1}`: an own-cube reference + # stays correct when the cube is extended. + assert orders["aov"] == { + "name": "aov", "type": "number", + "sql": "{CUBE.aov_part_1} / {users.aov_part_2}"} + + +def test_a_split_ratio_comes_back_as_the_metric_it_was_split_from(): + """The split is an implementation detail of the Cube side: the parts are marked + generated, so import skips them and inlines their SQL back through the public + measure's references, recovering the original expression verbatim.""" + expression = "SUM(orders.amount) / COUNT(DISTINCT users.id)" + files, _ = convert_ossie_to_cube( + _ossie(_TWO_DATASETS, _REL, _metric("aov", expression))) + ossie, _ = convert_cube_to_ossie(files, strict_fanout=False) + metrics = model_of(ossie)["metrics"] + assert [m["name"] for m in metrics] == ["aov"] + assert expr_of(metrics[0]) == expression def test_metric_lands_on_the_dataset_its_expression_references(): diff --git a/converters/cube/uv.lock b/converters/cube/uv.lock index 6b158087..05578dff 100644 --- a/converters/cube/uv.lock +++ b/converters/cube/uv.lock @@ -8,6 +8,7 @@ version = "0.2.0.dev0" source = { editable = "." } dependencies = [ { name = "pyyaml" }, + { name = "sqlglot" }, ] [package.dev-dependencies] @@ -18,7 +19,10 @@ dev = [ ] [package.metadata] -requires-dist = [{ name = "pyyaml", specifier = ">=6.0" }] +requires-dist = [ + { name = "pyyaml", specifier = ">=6.0" }, + { name = "sqlglot", specifier = ">=20.0" }, +] [package.metadata.requires-dev] dev = [ @@ -390,6 +394,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, ] +[[package]] +name = "sqlglot" +version = "30.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/cd/39a94f0f98076ee8e7c7c38fd4bba8d7845b0c629ff967057c64ef2c0989/sqlglot-30.14.0.tar.gz", hash = "sha256:df2ef5d2b8ca814313781f4ff35bf63e58f821ef517eeddbd523c19a61fa9bb9", size = 5944410, upload-time = "2026-07-27T11:23:30.698Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/ec/a729883ceda22dcd9117ce182f64d884bf494e72c4dfce00c2ad0a5978e1/sqlglot-30.14.0-py3-none-any.whl", hash = "sha256:fc768e24889d63a5e1237dea7ad305e5ffb4356a98b0bed828f89591ebcd3636", size = 719007, upload-time = "2026-07-27T11:23:28.637Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" From 8e7131815d9bf936caee58a09a989b930c06b45f Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Mon, 3 Aug 2026 16:55:36 +0500 Subject: [PATCH 22/46] Convert a fan-out-unsafe metric and report it, rather than refusing Refusing mirrored Cube's own refusal, but it is the wrong default for a hub-and-spoke converter: a model with one fanning join and one SUM failed outright, so the spoke on the other side got nothing at all -- and the metrics most worth converting are exactly the ones that trip it. Import now emits the metric and records FANOUT_UNSAFE_METRIC, naming the metric, the dataset and the relationship responsible. `--strict-fanout` (was `--no-strict-fanout`, inverted) restores the refusal for a caller who would rather have nothing than a number that disagrees with Cube. --- converters/cube/README.md | 15 +++++++---- converters/cube/src/ossie_cube/cli.py | 18 +++++++------ .../cube/src/ossie_cube/converter_issues.py | 9 ++++--- converters/cube/src/ossie_cube/cube_to_osi.py | 10 ++++--- converters/cube/tests/test_cli.py | 10 +++---- converters/cube/tests/test_cube_to_osi.py | 27 ++++++++++--------- converters/cube/tests/test_edge_cases.py | 10 +++---- converters/cube/tests/test_osi_to_cube.py | 2 +- 8 files changed, 57 insertions(+), 44 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index 71897937..d95c6dd2 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -65,7 +65,7 @@ composite metric). Python 3.11+. ```bash ossie-cube import -i model/ [-o model.yaml] [--name my_model] [--view sales] - [--no-strict-fanout] + [--strict-fanout] ossie-cube export -i model.yaml -o model/ [--dialect SNOWFLAKE] [--base-cube orders] ``` @@ -185,10 +185,15 @@ emit a silently-wrong one: | `sum`, `avg`, `count` + `sql` | `SUM(x)`, `AVG(x)`, `COUNT(x)` | **No** | Only the last row is at risk, and only when its own cube is the `to` (one) side of -a relationship in the model. The converter computes that from the Ossie graph and, -**by default, refuses** -- mirroring Cube's own refusal. Pass -`--no-strict-fanout` to emit the metric with a `FANOUT_UNSAFE_METRIC` issue -instead, naming the metric, the dataset, and the relationship responsible. +a relationship in the model. The converter computes that from the Ossie graph and +**records a `FANOUT_UNSAFE_METRIC` issue** naming the metric, the dataset and the +relationship responsible -- refusing a whole model over one such metric would leave +the spoke on the other side with nothing. Pass `--strict-fanout` to refuse instead, +mirroring Cube's own refusal. + +The issue is reported to the caller, not written into the Ossie model: the spec has +no additivity declaration to write it into (see below), and a `custom_extensions` +entry would only give every other converter something to warn about and discard. Because a bare `count` maps through the primary key, a cube carrying one **must** declare `primary_key: true` on a dimension; its absence is an error, not a diff --git a/converters/cube/src/ossie_cube/cli.py b/converters/cube/src/ossie_cube/cli.py index ce4804b1..d665be0c 100644 --- a/converters/cube/src/ossie_cube/cli.py +++ b/converters/cube/src/ossie_cube/cli.py @@ -28,10 +28,12 @@ always needs `-o` (a directory). Conversions that could not carry something across print an issue list to stderr. -By default a metric whose value a static Ossie expression cannot keep correct -under row multiplication is refused on import, mirroring Cube's own refusal to -answer such a query; pass `--no-strict-fanout` to emit it with a recorded issue -instead. +A metric whose value a static Ossie expression cannot keep correct under row +multiplication is converted with a `FANOUT_UNSAFE_METRIC` issue naming the metric, +the dataset and the relationship responsible -- a hub-and-spoke converter that +refuses a whole model over one such metric is not much use to the spoke on the +other side. Pass `--strict-fanout` to refuse instead, mirroring Cube's own refusal +to answer such a query. """ import argparse @@ -64,10 +66,10 @@ def _build_parser(): imp.add_argument("--view", help="view whose name/description/AI context map onto the " "Ossie model (default: the sole view, if there is one)") - imp.add_argument("--no-strict-fanout", dest="strict_fanout", - action="store_false", default=True, - help="record fan-out-unsafe metrics as issues instead of " - "refusing the conversion") + imp.add_argument("--strict-fanout", dest="strict_fanout", + action="store_true", default=False, + help="refuse the conversion when a metric is fan-out-unsafe, " + "instead of converting it and recording an issue") exp = sub.add_parser( "export", help="Apache Ossie semantic model -> Cube data model directory") diff --git a/converters/cube/src/ossie_cube/converter_issues.py b/converters/cube/src/ossie_cube/converter_issues.py index f0306d2f..d0285cd7 100644 --- a/converters/cube/src/ossie_cube/converter_issues.py +++ b/converters/cube/src/ossie_cube/converter_issues.py @@ -103,10 +103,11 @@ def __str__(self): class IssueLog: """Collects issues during a conversion. - `strict_types` names the issue types that should abort the conversion - instead of being recorded. The CLI puts `FANOUT_UNSAFE_METRIC` in there by - default, mirroring Cube's own refusal to answer a query whose measures - reference cubes that lead to row multiplication. + `strict_types` names the issue types that should abort the conversion instead of + being recorded. Nothing is in there by default: a converter that refuses a whole + model over one metric leaves the spoke on the other side with nothing. Passing + `--strict-fanout` adds `FANOUT_UNSAFE_METRIC`, mirroring Cube's own refusal to + answer a query whose measures reference cubes that lead to row multiplication. """ issues: list = field(default_factory=list) diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 136cf114..a0fbcc6c 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -98,15 +98,17 @@ _AND_SPLIT_RE = re.compile(r"\s+AND\s+", re.IGNORECASE) -def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=True): +def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=False): """Convert Cube model files ({relative filename: YAML str}) to Ossie YAML. Returns (ossie_yaml_str, IssueLog). `model_name` overrides the Ossie model name (default: the mapped view's name, else 'cube_model'). `view` names the view whose name/description/AI context map onto the Ossie model when the - directory holds more than one. `strict_fanout` refuses metrics whose value a - static Ossie expression cannot keep correct under row multiplication -- see - README "Fan-out". + directory holds more than one. + + A metric whose value a static Ossie expression cannot keep correct under row + multiplication is converted with a FANOUT_UNSAFE_METRIC issue; `strict_fanout` + refuses it instead -- see README "Fan-out". """ if not isinstance(files, dict) or not files: raise ConversionError("expected a non-empty mapping of {filename: YAML}") diff --git a/converters/cube/tests/test_cli.py b/converters/cube/tests/test_cli.py index e608bd4f..2314280d 100644 --- a/converters/cube/tests/test_cli.py +++ b/converters/cube/tests/test_cli.py @@ -227,7 +227,7 @@ def test_issues_go_to_stderr_so_stdout_stays_pipeable(tmp_path, capsys): parse(captured.out) # stdout is still clean YAML -def test_fanout_refusal_exits_nonzero_and_the_flag_downgrades_it(tmp_path, capsys): +def test_fanout_warns_by_default_and_the_flag_exits_nonzero(tmp_path, capsys): model = _write(tmp_path / "model", **{"cubes|m.yml": ( "cubes:\n" " - name: orders\n" @@ -248,14 +248,14 @@ def test_fanout_refusal_exits_nonzero_and_the_flag_downgrades_it(tmp_path, capsy " sql: \"{CUBE}.ltv\"\n" " type: sum\n" )}) - assert main(["import", "-i", str(model)]) == 1 - assert "FANOUT_UNSAFE_METRIC" in capsys.readouterr().err - - assert main(["import", "-i", str(model), "--no-strict-fanout"]) == 0 + assert main(["import", "-i", str(model)]) == 0 captured = capsys.readouterr() assert "FANOUT_UNSAFE_METRIC" in captured.err assert parse(captured.out)["semantic_model"][0]["metrics"] + assert main(["import", "-i", str(model), "--strict-fanout"]) == 1 + assert "FANOUT_UNSAFE_METRIC" in capsys.readouterr().err + def test_view_and_name_flags_take_effect(tmp_path, capsys): model = _write(tmp_path / "model", **{ diff --git a/converters/cube/tests/test_cube_to_osi.py b/converters/cube/tests/test_cube_to_osi.py index a3609e98..15f6f26d 100644 --- a/converters/cube/tests/test_cube_to_osi.py +++ b/converters/cube/tests/test_cube_to_osi.py @@ -398,17 +398,13 @@ def test_multi_stage_measure_is_dropped_with_an_issue(): } -def test_fanout_unsafe_metric_is_refused_by_default(): +def test_fanout_unsafe_metric_is_recorded_by_default(): """`users` is the one side of a many-to-one join, so summing over it after the - join over-counts. Cube deduplicates on the primary key at query time; a static - Ossie expression cannot, so the default is to refuse rather than emit a number - that silently disagrees with Cube.""" - with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): - convert_cube_to_ossie(_FANOUT_MODEL) - - -def test_fanout_unsafe_metric_is_recorded_when_not_strict(): - out, issues = convert_cube_to_ossie(_FANOUT_MODEL, strict_fanout=False) + join over-counts. Cube deduplicates on the primary key at query time and a static + Ossie expression cannot -- so the metric converts and the risk is reported, named + down to the relationship responsible. Refusing the whole model over one metric + would leave the spoke on the other side with nothing to convert.""" + out, issues = convert_cube_to_ossie(_FANOUT_MODEL) metric = by_name(model_of(out)["metrics"])["lifetime_value"] assert expr_of(metric) == "SUM(users.ltv)" recorded = issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) @@ -417,10 +413,17 @@ def test_fanout_unsafe_metric_is_recorded_when_not_strict(): assert "over-count" in recorded[0].detail +def test_fanout_unsafe_metric_is_refused_under_strict_fanout(): + """Mirrors Cube's own refusal, for a caller who would rather have nothing than a + number that disagrees with Cube.""" + with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): + convert_cube_to_ossie(_FANOUT_MODEL, strict_fanout=True) + + def test_idempotent_aggregates_are_never_flagged(fixture_a): """count / count_distinct / min / max are unaffected by duplicate rows, so a - fanned-out dataset carrying only those converts cleanly under strict mode.""" - _, issues = convert_cube_to_ossie(fixture_a) + fanned-out dataset carrying only those raises nothing even under strict mode.""" + _, issues = convert_cube_to_ossie(fixture_a, strict_fanout=True) assert not issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 6cdd1b50..0f83c4ac 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -42,7 +42,7 @@ def _files(**named): def _roundtrip(files): - ossie, issues = convert_cube_to_ossie(files, strict_fanout=False) + ossie, issues = convert_cube_to_ossie(files) back, _ = convert_ossie_to_cube(ossie) return ossie, back, issues @@ -150,10 +150,10 @@ def test_count_over_an_expression_is_fanout_unsafe(): " sql: \"{CUBE}.email\"\n" " type: count\n" )) - with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): - convert_cube_to_ossie(files) - _, issues = convert_cube_to_ossie(files, strict_fanout=False) + _, issues = convert_cube_to_ossie(files) assert issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) + with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): + convert_cube_to_ossie(files, strict_fanout=True) _MULTI_STAGE = _files(orders=( @@ -425,7 +425,7 @@ def test_a_many_to_one_join_still_makes_its_target_fanned_out(): " type: sum\n" )) with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): - convert_cube_to_ossie(files) + convert_cube_to_ossie(files, strict_fanout=True) def test_one_to_one_keeps_its_declared_orientation(): diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 985ebca5..3fd94eca 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -421,7 +421,7 @@ def test_a_split_ratio_comes_back_as_the_metric_it_was_split_from(): expression = "SUM(orders.amount) / COUNT(DISTINCT users.id)" files, _ = convert_ossie_to_cube( _ossie(_TWO_DATASETS, _REL, _metric("aov", expression))) - ossie, _ = convert_cube_to_ossie(files, strict_fanout=False) + ossie, _ = convert_cube_to_ossie(files) metrics = model_of(ossie)["metrics"] assert [m["name"] for m in metrics] == ["aov"] assert expr_of(metrics[0]) == expression From a103b0c11d3bce4f67b838dd207a12c186fec400 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Mon, 3 Aug 2026 18:52:56 +0500 Subject: [PATCH 23/46] Add the interop matrix the README's claims are measured with The README asserts that the stash reduction cut Databricks warnings from 32 to 2 and that three spokes reject a two-part source. Both were measured with a throwaway script, which is no use to a reviewer -- so it is now a committed tool. `tools/interop_matrix.py` converts a Cube model to Ossie, checks that intermediate against the repo's own validation/validate.py, then hands it to all nine Python spokes and reports result, warning count, and how many of those warnings exist only because of a foreign `custom_extensions` entry. Stdlib only, and outside pytest -- it drives the other converters' environments, not this one's. Nothing about it is Cube-specific past the first hop. If it is useful repo-wide it belongs somewhere like compliance/, which is a dev@ question. --- converters/cube/README.md | 51 +++++ converters/cube/tools/interop_matrix.py | 251 ++++++++++++++++++++++++ 2 files changed, 302 insertions(+) create mode 100644 converters/cube/tools/interop_matrix.py diff --git a/converters/cube/README.md b/converters/cube/README.md index d95c6dd2..d5833dc0 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -269,6 +269,52 @@ Error: Source 'public.orders' must be a fully qualified db.schema.table or a sub Import reports this as `SOURCE_NOT_FULLY_QUALIFIED` rather than guessing a catalog name, so it surfaces where the Ossie document is produced instead of three hops later. +### Measuring it + +Both claims above are measurements, so they are reproducible: + +```bash +uv run tools/interop_matrix.py # the committed TPC-DS fixture +uv run tools/interop_matrix.py path/to/cube/model # any Cube model directory +uv run tools/interop_matrix.py --spokes omni --keep # one spoke, keep its output +``` + +It converts a Cube model to Ossie, checks that intermediate against the repo's own +`validation/validate.py`, then hands it to every other converter and reports what +each made of it: + +``` +model: converters/cube/tests/fixtures/tpcds_cube +Ossie: 539 lines, 7 CUBE stash entries +issues: 5x CUBE_LEVEL_AI_CONTEXT_INERT +spec: valid (validation/validate.py) + +spoke result warns foreign note +---------------------------------------------------------------------------- +databricks OK 22 2 +dbt FAIL 0 0 AttributeError: 'PydanticSemanticManifes +gooddata OK 0 0 +gsf OK 0 0 +honeydew OK 0 0 +omni OK 15 7 +orionbelt OK 2 0 +snowflake OK 7 7 +wisdom OK 47 7 +polaris -- Java converter, needs Maven +salesforce -- Java converter, needs Maven +``` + +`foreign` counts warnings that name a `custom_extensions` vendor — the cost this +converter imposes on the others by stashing, and the number to watch when deciding +whether something belongs in a stash at all. The dbt `FAIL` is unrelated to this +converter: its CLI crashes on every input, including this repo's own examples +([#296](https://github.com/apache/ossie/issues/296)). + +Each spoke runs in its own `uv` environment, so the first run resolves that +converter's dependencies; the script is stdlib-only and needs none of its own. Nothing +about it is Cube-specific except the first hop — if it is useful repo-wide it belongs +somewhere like `compliance/`, which is a question for `dev@`. + ## Conversion issues `convert_cube_to_ossie` returns `(yaml, IssueLog)`. Each issue carries a type, the @@ -343,6 +389,11 @@ document, and Hypothesis property-based round-trip tests over generated Cube models -- which fall back to a seeded sweep when `hypothesis` is unavailable, so the properties still run. +`tools/interop_matrix.py` checks the other half of the job — whether the Ossie this +converter emits is any use to the other spokes. It is not part of `pytest`, because +it drives the other converters' environments rather than this one's. See +[Measuring it](#measuring-it). + ## Future effort Both the Apache Ossie specification and Cube's data model are still evolving. As diff --git a/converters/cube/tools/interop_matrix.py b/converters/cube/tools/interop_matrix.py new file mode 100644 index 00000000..2bc2ec3b --- /dev/null +++ b/converters/cube/tools/interop_matrix.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +# +# /// script +# requires-python = ">=3.11" +# /// + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Does a converted Cube model actually reach the other spokes? + +Ossie is a hub: `Cube -> Ossie` is only half the point, and a converter can pass its +own round-trip tests while emitting something the next converter chokes on. This +script runs `Cube -> Ossie -> every other spoke` and prints what each one made of it, +so a change can be judged on interop instead of on self-consistency. + + uv run tools/interop_matrix.py # the committed tpcds fixture + uv run tools/interop_matrix.py path/to/cube/model # any Cube model directory + uv run tools/interop_matrix.py --keep # leave the outputs to read + +Columns: + + result OK / EMPTY (exit 0, nothing written) / FAIL / SKIP (deps not installed) + warns lines the spoke wrote to stderr that read as warnings + foreign those warnings that name a `custom_extensions` vendor -- the cost this + converter imposes on every other spoke by stashing, and the number to + watch when deciding whether something belongs in a stash at all + +Each spoke runs in its own `uv` environment, so the first run for a given spoke +resolves its dependencies (`uv sync` there first to keep this fast) -- which leaves a +`uv.lock` and a `.venv` in that converter's directory. Those belong to the converter, +not to this run: check `git status` before committing. The Java converters (polaris, +salesforce) are listed as unsupported rather than skipped silently; they need Maven, +not uv. + +Stdlib only, so it needs no environment of its own. +""" + +import argparse +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +# (directory under converters/, argv to convert Ossie -> spoke, output is a directory) +# +# The invocations differ per spoke because the CLIs do: some take an `export` +# subcommand, some a named direction, snowflake takes none, gooddata ships no CLI at +# all and is driven through its Python API. +SPOKES = [ + ("databricks", ["ossie-databricks", "export"], False), + ("dbt", ["ossie-dbt", "osi-to-msi"], False), + ("gooddata", None, False), # API-only; see _run_gooddata + ("gsf", ["ossie-gsf", "export"], False), + ("honeydew", ["honeydew-osi", "osi-to-honeydew"], True), + ("omni", ["osi-omni", "export"], True), + ("orionbelt", ["ossie-orionbelt", "osi-to-obml"], False), + ("snowflake", ["ossie-snowflake"], False), + ("wisdom", ["ossie-wisdom", "osi-to-wisdom"], False), +] + +# Converters written in Java: a different toolchain, not a missing dependency. +UNSUPPORTED = ["polaris", "salesforce"] + +_WARN_RE = re.compile(r"warn", re.I) +# Python's `warnings.warn` prints the message and then echoes the calling source +# line, which would otherwise count the same warning twice. +_ECHO_RE = re.compile(r"^\s*warnings\.warn\b") +_FOREIGN_RE = re.compile(r"custom_extension|vendor|foreign", re.I) + + +def repo_root(): + for parent in [Path(__file__).resolve(), *Path(__file__).resolve().parents]: + if (parent / "converters").is_dir() and (parent / "core-spec").is_dir(): + return parent + sys.exit("cannot locate the repository root from this script's path") + + +def run(cwd, argv): + return subprocess.run(argv, cwd=cwd, capture_output=True, text=True) + + +def count_warnings(stderr): + """(warnings, of which are about a foreign vendor extension). + + A line count, so a warning whose message wraps counts more than once. It is a + relative measure -- run it before and after a change -- not an exact tally. + """ + warns = [ln for ln in stderr.splitlines() + if _WARN_RE.search(ln) and not _ECHO_RE.match(ln)] + return len(warns), len([ln for ln in warns if _FOREIGN_RE.search(ln)]) + + +def import_issues(stderr): + """The issue types `ossie-cube import` reported, as {type: count}. + + Its own issues do not read as warnings -- they are `[TYPE] element: detail` lines + -- so they are counted from their structure rather than by keyword. + """ + found = {} + for ln in stderr.splitlines(): + m = re.match(r"\s+\[([A-Z_]+)\]", ln) + if m: + found[m.group(1)] = found.get(m.group(1), 0) + 1 + return found + + +def produced_output(dest, is_dir): + if not dest.exists(): + return False + return any(dest.rglob("*")) if is_dir else dest.stat().st_size > 0 + + +def _run_gooddata(root, ossie, dest): + """gooddata ships no console script, so drive its API the way its README does.""" + script = ( + "import json, sys, yaml\n" + "from ossie_gooddata import osi_to_gooddata\n" + "from ossie_gooddata.models import gd_model_to_dict\n" + "model = yaml.safe_load(open(sys.argv[1]).read())\n" + "out = gd_model_to_dict(osi_to_gooddata(model))\n" + "open(sys.argv[2], 'w').write(json.dumps(out, indent=2, default=str))\n" + ) + return run(root / "converters/gooddata", + ["uv", "run", "--quiet", "python", "-c", script, + str(ossie), str(dest)]) + + +def cube_to_ossie(root, model_dir, dest): + r = run(root / "converters/cube", + ["uv", "run", "--quiet", "ossie-cube", "import", + "-i", str(model_dir), "-o", str(dest)]) + return r + + +def validate_ossie(root, ossie): + """Run the repo's own validator on the intermediate model. + + A spoke rejecting the model is only interesting once the model is known good, so + this is checked before the matrix rather than left to be inferred from it. + """ + return run(root, ["uv", "run", "--quiet", "validation/validate.py", str(ossie)]) + + +def main(): + root = repo_root() + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument( + "model", nargs="?", + default=str(root / "converters/cube/tests/fixtures/tpcds_cube"), + help="Cube model directory (default: the committed tpcds fixture)") + ap.add_argument("--spokes", help="comma-separated subset to run") + ap.add_argument("--keep", action="store_true", + help="keep the converted outputs and print where they are") + args = ap.parse_args() + + model_dir = Path(args.model).expanduser().resolve() + if not model_dir.exists(): + sys.exit(f"no such Cube model: {model_dir}") + + wanted = None + if args.spokes: + wanted = {s.strip() for s in args.spokes.split(",") if s.strip()} + unknown = wanted - {name for name, _, _ in SPOKES} + if unknown: + sys.exit(f"unknown spoke(s): {', '.join(sorted(unknown))}") + + out = Path(tempfile.mkdtemp(prefix="ossie-interop-")) + try: + ossie = out / "from_cube.yaml" + r = cube_to_ossie(root, model_dir, ossie) + if r.returncode != 0: + print(f"Cube -> Ossie FAILED\n{r.stderr}", file=sys.stderr) + return 1 + + text = ossie.read_text() + reported = import_issues(r.stderr) + print(f"model: {model_dir}") + print(f"Ossie: {len(text.splitlines())} lines, " + f"{text.count('vendor_name: CUBE')} CUBE stash entries") + if reported: + print("issues: " + ", ".join( + f"{n}x {kind}" for kind, n in sorted(reported.items()))) + + v = validate_ossie(root, ossie) + print(f"spec: {'valid' if v.returncode == 0 else 'INVALID'} " + f"(validation/validate.py)") + if v.returncode != 0: + print(v.stdout.strip() or v.stderr.strip()) + + print() + print(f"{'spoke':<12} {'result':<7} {'warns':>5} {'foreign':>8} note") + print("-" * 76) + + failures = 0 + for name, argv, is_dir in SPOKES: + if wanted and name not in wanted: + continue + dest = out / (name if is_dir else f"{name}.out") + if argv is None: + r = _run_gooddata(root, ossie, dest) + else: + r = run(root / "converters" / name, + ["uv", "run", "--quiet", *argv, + "-i", str(ossie), "-o", str(dest)]) + + warns, foreign = count_warnings(r.stderr) + note = "" + if r.returncode != 0: + tail = (r.stderr.strip().splitlines() or [""])[-1] + # A missing environment is not the converter rejecting the model. + skipped = "No solution found" in r.stderr or "no such command" in tail + result = "SKIP" if skipped else "FAIL" + note = tail[:40] + failures += result == "FAIL" + else: + result = "OK" if produced_output(dest, is_dir) else "EMPTY" + print(f"{name:<12} {result:<7} {warns:>5} {foreign:>8} {note}") + + if not wanted: + for name in UNSUPPORTED: + print(f"{name:<12} {'--':<7} {'':>5} {'':>8} " + "Java converter, needs Maven") + + if args.keep: + print(f"\noutputs: {out}") + return 1 if failures else 0 + finally: + if not args.keep: + shutil.rmtree(out, ignore_errors=True) + + +if __name__ == "__main__": + sys.exit(main()) From 410abb9e293a3035b17b6135a06a1ca0655a7e30 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Mon, 3 Aug 2026 19:55:14 +0500 Subject: [PATCH 24/46] Fix two expression-handling defects found in review **String literals were rewritten as if they were code.** Cube compiles a YAML `sql` as a Python f-string (`f""` in YamlCompiler), so `{...}` interpolates anywhere in the value -- SQL's own quotes mean nothing to it. Export was rewriting inside literals, which means Cube then replaced the literal's own text with a column reference: SUM(orders.amount) || ' per COUNT(users.id) unit' -> {CUBE.label_part_1} || ' per {users.label_part_2} unit' Three silent consequences: the literal's text changed, a measure reference appeared inside it, and `users` looked like a second dataset -- which also decides which cube the measure lands on. `quoted_runs`/`referenced_datasets` now confine every export-side rewrite to the code outside quoted runs, and `aggregate_spans` skips a name found inside one. Import deliberately does NOT do this, and now says so: because of the f-string compilation, a reference inside a literal is one Cube really resolves, so skipping it there would lose it. Both directions are pinned by tests. **A cross-dataset metric stopped being reported.** The check lived only in the calculated-measure fallback, so the two shapes a cross-dataset metric now normally takes -- a decomposed pair, or a recognized single aggregate -- reported nothing, even though decomposition produces *more* cross-cube references than the shape that did report. Moved to where the dataset set is computed, so it fires whatever shape the measure takes, and no test had ever asserted it. Also folds the three copies of the dotted-reference regex into one helper, and drops the parameters `_measure_from_expression` no longer needs. 298 tests, 96% line coverage (expressions.py 82% -> 96%). --- converters/cube/README.md | 9 ++ converters/cube/src/ossie_cube/_common.py | 74 +++++++++++++- converters/cube/src/ossie_cube/expressions.py | 9 ++ converters/cube/src/ossie_cube/osi_to_cube.py | 44 ++++----- converters/cube/tests/test_edge_cases.py | 98 +++++++++++++++++++ converters/cube/tests/test_osi_to_cube.py | 49 ++++++++++ 6 files changed, 258 insertions(+), 25 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index d5833dc0..f3489dec 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -161,6 +161,15 @@ Ossie dialect enum has no `CUBE` entry -- so import emits `ANSI_SQL`, and export prefers `ANSI_SQL` with `--dialect` prepending a warehouse dialect (e.g. `SNOWFLAKE` for a Snowflake-backed Cube model). +**String literals** are handled asymmetrically, on purpose. Cube compiles a YAML +`sql` value as a Python f-string (`f""` in `YamlCompiler`), so `{CUBE}.col` +interpolates *anywhere* in the value -- SQL's own quotes mean nothing to it. So on +import a reference inside a literal is a real reference and is translated, while on +export nothing is rewritten inside a literal: emitting `{CUBE.col}` there would make +Cube replace the literal's own text with a column reference. The same rule decides +which dataset a metric belongs to, so a name mentioned only inside a literal does not +attribute the metric or make it look cross-dataset. + ## Fan-out This is the one place where Cube carries semantics an Ossie expression cannot, and diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index 6ca0666a..200ef0fa 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -385,6 +385,73 @@ def repl(m): return out, changed +def quoted_runs(sql): + """Split SQL into (text, is_quoted) runs, delimiters included in the quoted run. + + Used by the **export** direction only, to keep a rewrite out of string literals + and delimited identifiers. Import deliberately does not do this: a Cube YAML + `sql` is compiled as a Python f-string (`f""` in YamlCompiler), so `{CUBE}` + interpolates anywhere in the value -- SQL's own quotes are ordinary characters to + it. Skipping quoted text on the way in would therefore *lose* a reference Cube + really does resolve. + + `'`, `"` and backtick all open a run; a run is closed by its own delimiter, and + an unterminated one runs to the end (reported quoted, so nothing in it is + rewritten). SQL's `''` doubling needs no special case: it reads as a close + immediately followed by an open, leaving an empty unquoted run between. + """ + runs, buf, quote = [], [], None + for ch in str(sql): + if quote: + buf.append(ch) + if ch == quote: + runs.append(("".join(buf), True)) + buf, quote = [], None + elif ch in "'\"`": + if buf: + runs.append(("".join(buf), False)) + buf, quote = [ch], ch + else: + buf.append(ch) + if buf: + runs.append(("".join(buf), quote is not None)) + return runs + + +def sub_outside_quotes(sql, transform): + """Apply `transform` to the parts of `sql` outside quoted runs.""" + return "".join(text if quoted else transform(text) + for text, quoted in quoted_runs(sql)) + + +def referenced_datasets(expr, known): + """The dataset names an Ossie expression references, ignoring quoted text. + + Decides which cube a measure lands on and whether it crosses cubes, so a name + that only appears inside a string literal must not count -- otherwise + `SUM(orders.amount) || ' per users.id unit'` reads as a two-dataset metric and + gets attributed to the base cube rather than to `orders`. + """ + found = set() + for text, quoted in quoted_runs(expr): + if quoted: + continue + found |= {m.group(1) for m in DOTTED_REF_RE.finditer(text) + if m.group(1) in known} + return found + + +def quoted_char_mask(sql): + """One flag per character of `sql`: True where it sits inside a quoted run. + + For a caller that needs offsets into the original string rather than a rewrite. + """ + mask = [] + for text, quoted in quoted_runs(sql): + mask.extend([quoted] * len(text)) + return mask + + def source_part_count(source): """How many identifier parts a dotted dataset `source` has, or None for a query. @@ -488,6 +555,11 @@ def ossie_expr_to_cube_sql(expr, own_cube, own_members=(), cube_names=(), exists only in Ossie -- Cube has neither a column nor a member by that name -- so a reference to it becomes the half's own SQL (`{CUBE}.lat`), requalified when it crosses cubes. + + A dotted token inside a string literal is left alone. This matters more than it + looks: Cube compiles a YAML `sql` as a Python f-string, so a `{...}` it emitted + into a literal would be interpolated at compile time and replace the literal's + own text with a column reference. """ escaped = str(expr).replace("{", "\\{").replace("}", "\\}") known = set(cube_names) @@ -510,7 +582,7 @@ def repl(m): # reference or an unrelated dotted token. Leave it alone. return m.group(0) - return DOTTED_REF_RE.sub(repl, escaped) + return sub_outside_quotes(escaped, lambda run: DOTTED_REF_RE.sub(repl, run)) # --- source --------------------------------------------------------------------- diff --git a/converters/cube/src/ossie_cube/expressions.py b/converters/cube/src/ossie_cube/expressions.py index 68243c0f..12f70e44 100644 --- a/converters/cube/src/ossie_cube/expressions.py +++ b/converters/cube/src/ossie_cube/expressions.py @@ -38,6 +38,8 @@ import sqlglot import sqlglot.expressions as exp +from ._common import quoted_char_mask + # sqlglot node types for the aggregates this converter maps to a Cube measure type. # `Count` covers COUNT / COUNT(DISTINCT x); ApproxDistinct covers # APPROX_COUNT_DISTINCT. @@ -91,6 +93,10 @@ def aggregate_spans(expr): inside another span is not returned, so `SUM(x) / NULLIF(SUM(y), 0)` gives two and `SUM(SUM(x))` gives one. Returns [] when the expression does not parse, or is itself a single aggregate needing no decomposition. + + A name inside a string literal is not a call: `SUM(x) || ' per COUNT(y) unit'` + has one aggregate, not two. Taking the second would splice a measure reference + into the literal. """ text = str(expr) if parse(text) is None or is_single_aggregate(text): @@ -98,6 +104,7 @@ def aggregate_spans(expr): candidates = [] upper = text.upper() + quoted = quoted_char_mask(text) for name in _AGGREGATE_NAMES: at = 0 while True: @@ -106,6 +113,8 @@ def aggregate_spans(expr): break start, after = at, at + len(name) at = after + if quoted[start]: + continue # A call, not part of a longer identifier: boundary before, `(` after. if start and (text[start - 1].isalnum() or text[start - 1] == "_"): continue diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 889b3fce..762ae00f 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -36,7 +36,6 @@ from ._common import ( DATATYPE_TO_DIM_TYPE, - DOTTED_REF_RE, DEFAULT_DATATYPE_FOR_CUBE_TYPE, OSSIE_FUNC_TO_AGG, OSSIE_VERSION, @@ -53,6 +52,7 @@ pick_expression, primary_key_operand, read_stash, + referenced_datasets, require_str, sanitize_name, synonyms_of, @@ -660,15 +660,22 @@ def resolve_base(): "no ANSI_SQL or preferred-dialect expression; metric dropped") continue - referenced = { - m.group(1) for m in re.finditer( - r"(? 1: + # Cube resolves a cross-cube member reference by adding an implicit join, + # so the model needs a join path between these cubes -- which Ossie's + # expression does not state and this converter cannot verify. Reported for + # every shape the measure can take: it used to be raised only from the + # calculated-measure fallback, so a decomposed metric (the shape with the + # *most* cross-cube references) reported nothing at all. + issues.add(IssueType.APPROXIMATED, scope, + f"expression spans datasets {', '.join(sorted(referenced))}; " + f"Cube reaches the others from '{target}' through an implicit " + f"join, so verify a join path exists") + spans = [] if stash.get("sql") else aggregate_spans(expr) if len(spans) > 1: # A composite metric: give each aggregate its own measure on the cube its @@ -677,13 +684,12 @@ def resolve_base(): # seeing one opaque expression -- see _decompose_measure. public_sql = _decompose_measure( expr, spans, mname, target, measures_by_cube, members_by_cube, - inline_sql_by_cube, pk_by_cube, sanitized, name, scope, issues) + inline_sql_by_cube, pk_by_cube, sanitized, name) measure = {"name": mname, "sql": public_sql, "type": "number"} else: measure = _measure_from_expression( expr, target, mname, stash, members_by_cube.get(target, set()), - inline_sql_by_cube, pk_by_cube.get(target, []), sanitized, scope, - issues) + inline_sql_by_cube, pk_by_cube.get(target, []), sanitized) _apply_measure_metadata(metric, measure, stash) _place(measures_by_cube, target, measure, name) return measures_by_cube @@ -691,7 +697,7 @@ def resolve_base(): def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, members_by_cube, inline_sql_by_cube, pk_by_cube, sanitized, - model_name, scope, issues): + model_name): """Emit one `public: false` measure per aggregate; return the sql referencing them. Cube corrects for row multiplication per measure, keyed on the cube that measure @@ -712,8 +718,7 @@ def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, for start, end in spans: piece = expr[start:end] # Each aggregate lands on the cube its own operand references. - refs = {m.group(1) for m in DOTTED_REF_RE.finditer(piece) - if m.group(1) in sanitized} + refs = referenced_datasets(piece, sanitized) part_target = next(iter(refs)) if len(refs) == 1 else fallback index += 1 @@ -726,7 +731,7 @@ def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, part = _measure_from_expression( piece, part_target, part_name, {}, members_by_cube.get(part_target, set()), inline_sql_by_cube, - pk_by_cube.get(part_target, []), sanitized, scope, issues) + pk_by_cube.get(part_target, []), sanitized) part["public"] = False part["meta"] = {"ossie": {"part_of": mname}} _place(measures_by_cube, part_target, part, model_name) @@ -755,7 +760,7 @@ def _place(measures_by_cube, target, measure, model_name): def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_cube, - primary_key, sanitized, scope, issues): + primary_key, sanitized): """Turn an Ossie metric expression back into a structured Cube measure. `COUNT(DISTINCT )` is Cube's bare `type: count` -- @@ -796,15 +801,6 @@ def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_ measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( expr, target, members, sanitized, inline_sql=inline_sql_by_cube) measure["type"] = "number" - if len({ - ref for ref in re.findall( - r"(? 1: - issues.add(IssueType.APPROXIMATED, scope, - f"expression spans several datasets; emitted as a calculated " - f"measure on cube '{target}' -- verify the join path") return measure diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 0f83c4ac..ebcb59fa 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -1413,3 +1413,101 @@ def test_several_semantic_models_convert_the_first_with_an_issue(): # The other models are not preserved anywhere, so this is a drop. dropped = issues.of_type(IssueType.DROPPED_NO_CUBE_EQUIVALENT) assert any("only the first is converted" in i.detail for i in dropped) + + +# --- string literals ------------------------------------------------------------ +# +# The two directions are deliberately asymmetric, so both are pinned here. A Cube +# YAML `sql` is compiled as a Python f-string (`f""` in YamlCompiler), which +# interpolates `{...}` anywhere in the value -- SQL's own quotes mean nothing to it. +# So on import a reference inside a literal is a real reference, while on export a +# rewrite must stop at the quotes or it would destroy the literal's text. + +@pytest.mark.parametrize("sql,expected", [ + ("a = 'x'", [("a = ", False), ("'x'", True)]), + ("'x' = a", [("'x'", True), (" = a", False)]), + ("'it''s'", [("'it'", True), ("'s'", True)]), + ('"col" = `c`', [('"col"', True), (" = ", False), ("`c`", True)]), + ("a = 'unterminated", [("a = ", False), ("'unterminated", True)]), + ("plain", [("plain", False)]), +]) +def test_quoted_runs_splits_sql_into_code_and_quoted_text(sql, expected): + from ossie_cube._common import quoted_runs + assert quoted_runs(sql) == expected + + +@pytest.mark.parametrize("expr,expected", [ + ("SUM(orders.amount)", {"orders"}), + ("SUM(orders.amount) / COUNT(users.id)", {"orders", "users"}), + ("SUM(orders.amount) || ' per users.id unit'", {"orders"}), + ("'orders.amount'", set()), + ("SUM(ghost.amount)", set()), +]) +def test_referenced_datasets_ignores_quoted_text(expr, expected): + from ossie_cube._common import referenced_datasets + assert referenced_datasets(expr, {"orders", "users"}) == expected + + +def test_a_reference_inside_a_literal_is_still_translated_on_import(): + """Not an oversight: Cube would have interpolated it, so dropping it would lose a + reference the model really does resolve.""" + files = _files(orders=( + "cubes:\n" + " - name: orders\n" + " sql_table: a.b.orders\n" + " dimensions:\n" + " - name: note\n" + " sql: \"CONCAT({CUBE}.status, ' {CUBE}.status ')\"\n" + " type: string\n" + )) + ossie, _ = convert_cube_to_ossie(files) + field = by_name(by_name(model_of(ossie)["datasets"])["orders"]["fields"])["note"] + assert expr_of(field) == "CONCAT(status, ' status ')" + + +# --- aggregate span scanning ----------------------------------------------------- +# +# The scanner decides whether a metric is decomposed into one measure per aggregate, +# so its rejection paths matter as much as its matches: a false positive splices a +# measure reference into text that was never a call. + +@pytest.mark.parametrize("expr,expected", [ + # Two aggregates -- the case decomposition exists for. + ("SUM(a.x) / COUNT(b.y)", ["SUM(a.x)", "COUNT(b.y)"]), + # Only the outermost of a nested pair. + ("SUM(a.x) / NULLIF(SUM(b.y), 0)", ["SUM(a.x)", "SUM(b.y)"]), + # A closing paren inside a literal does not end the call. + ("SUM(a.x || ')') / COUNT(b.y)", ["SUM(a.x || ')')", "COUNT(b.y)"]), + # Part of a longer identifier, not a call. + ("MY_SUM(a.x) / 2", []), + ("SUMMARY(a.x) - MIN(b.y)", ["MIN(b.y)"]), + # A name with no argument list at all. + ("a.count / b.total", []), + # Whitespace between the name and its parens is still a call. + ("SUM (a.x) - MIN (b.y)", ["SUM (a.x)", "MIN (b.y)"]), + # Unbalanced parens: not a span, and not a crash. + ("SUM(a.x / MIN(b.y)", []), + # A single aggregate needs no decomposition. + ("SUM(a.x)", []), + # Unparseable input falls back to one opaque measure. + ("SUM(a.x) /// COUNT(", []), +]) +def test_aggregate_spans_only_matches_real_calls(expr, expected): + from ossie_cube.expressions import aggregate_spans + assert [expr[s:e] for s, e in aggregate_spans(expr)] == expected + + +@pytest.mark.parametrize("expr,expected", [ + ("SUM(a.x)", False), + # One self-contained term: the space is inside the parens, so inlining it into a + # larger expression needs no parentheses. + ("COUNT(DISTINCT a.x)", False), + ("SUM(a.x) / 2", True), + ("CASE WHEN a.x THEN 1 END", True), # a top-level space is structure + ("'a + b'", False), # operators inside a literal are text + ("'a b'", False), + ('"a b"', False), +]) +def test_has_top_level_operator_ignores_quoted_text(expr, expected): + from ossie_cube.expressions import has_top_level_operator + assert has_top_level_operator(expr) is expected diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 3fd94eca..f685b029 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -414,6 +414,55 @@ def test_a_ratio_is_split_into_one_measure_per_aggregate(): "sql": "{CUBE.aov_part_1} / {users.aov_part_2}"} +def test_a_dotted_token_inside_a_string_literal_is_left_alone(): + """Cube compiles a YAML `sql` as a Python f-string, so a `{...}` written into a + string literal is still interpolated -- it would replace the literal's own text + with a column reference. So the rewrite has to stop at the quotes.""" + files, _ = convert_ossie_to_cube(_ossie(_ORDERS, metrics=_metric( + "m", "CONCAT(CAST(SUM(orders.amount) AS VARCHAR), ' orders.amount ')"))) + assert _cubes(files)["orders"]["measures"][0]["sql"] == ( + "CONCAT(CAST(SUM({CUBE}.amount) AS VARCHAR), ' orders.amount ')") + + +def test_an_aggregate_name_inside_a_string_literal_is_not_an_aggregate(): + """Otherwise the literal is treated as a second aggregate and gets a measure + reference spliced into the middle of it.""" + files, issues = convert_ossie_to_cube(_ossie(_TWO_DATASETS, _REL, _metric( + "label", "SUM(orders.amount) || ' per COUNT(users.id) unit'"))) + measures = _cubes(files)["orders"]["measures"] + # One measure, not a decomposed pair, and the literal survives verbatim. + assert [m["name"] for m in measures] == ["label"] + assert measures[0]["sql"] == ( + "SUM({CUBE}.amount) || ' per COUNT(users.id) unit'") + assert "measures" not in _cubes(files, "model/cubes/users.yml")["users"] + # `users` is named only inside the literal, so this is not a cross-cube metric. + assert not issues.of_type(IssueType.APPROXIMATED) + + +@pytest.mark.parametrize("shape,expr", [ + ("decomposed", "SUM(orders.amount) / COUNT(DISTINCT users.id)"), + ("single aggregate", "SUM(orders.amount - users.id)"), + ("calculated", "SUM(orders.amount) + users.id"), +]) +def test_a_cross_dataset_metric_is_reported_whatever_shape_it_takes(shape, expr): + """Cube reaches another cube's members through an implicit join, so the model + needs a join path this converter cannot verify. The report used to come only from + the calculated-measure fallback, which meant the decomposed shape -- the one with + the *most* cross-cube references -- reported nothing.""" + _, issues = convert_ossie_to_cube( + _ossie(_TWO_DATASETS, _REL, _metric("m", expr))) + reported = issues.of_type(IssueType.APPROXIMATED) + assert len(reported) == 1, shape + assert "orders, users" in reported[0].detail + assert "join path" in reported[0].detail + + +def test_a_single_dataset_metric_is_not_reported(): + _, issues = convert_ossie_to_cube(_ossie( + _TWO_DATASETS, _REL, _metric("m", "SUM(orders.amount)"))) + assert not issues.of_type(IssueType.APPROXIMATED) + + def test_a_split_ratio_comes_back_as_the_metric_it_was_split_from(): """The split is an implementation detail of the Cube side: the parts are marked generated, so import skips them and inlines their SQL back through the public From a3a5fbbb6a29e1c6fb80664bbcfc8ef9eb2d333e Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Mon, 3 Aug 2026 20:00:07 +0500 Subject: [PATCH 25/46] Clean up the review findings that were not behaviour - `_convert_cube` had its docstring after the first statement, so `__doc__` was None. The statement it displaced is now passed in instead (see below). - `_MeasureResolver` built a `_dimensions` set for every cube and never read it. - Implemented the memoization its docstring already claimed. A measure referenced from several places was recomputed once per reference, recursively: a 16-deep chain of double references took 262,125 calls and 2.0s, now 49 calls and 0.9s. The docstring now also records what the cache cannot fix -- inlining is exponential in reference depth because Cube's own inlining is, and no limit is imposed since any threshold would reject a legitimate model. - The join stash tested three conditions, two of which could not be false (`from_cube` is `cname` at that point, and an unnormalized `many_to_one` implies a normalized one). Reduced to the one that matters, with the reason stated. - `_plain_members` ran once per measure; hoisted to once per cube and shared with the dimension stage, which was computing the same set again. - Two open-coded copies of `_restore_parked_extensions` replaced with the helper written for exactly that; `ds_scope` folded into the identical `scope`. - Noted why `sanitize_name` is called with an empty `taken` (the target cube is not known yet; `_place` rejects the collision later). tools/interop_matrix.py: a spoke that hangs no longer takes the run down with it (600s timeout), a missing `uv` reports itself instead of raising FileNotFoundError, and the uv-wording match that distinguishes SKIP from FAIL is named and explained rather than inline. 298 tests, 96% line coverage. Interop matrix unchanged. --- converters/cube/src/ossie_cube/cube_to_osi.py | 83 +++++++++++-------- converters/cube/src/ossie_cube/osi_to_cube.py | 3 + converters/cube/tools/interop_matrix.py | 38 +++++++-- 3 files changed, 85 insertions(+), 39 deletions(-) diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index a0fbcc6c..7e5bbedc 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -144,12 +144,16 @@ def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=False fanned_out = _fanned_out_datasets(relationships) pk_by_cube = {cname: _primary_key_of(cube, cname) for cname, cube in cubes.items()} + # Which members regenerate from a bare column name, worked out once per cube: + # both the measure and the dimension stage need the same answer. + plain_by_cube = {cname: _plain_members(cube, cname) + for cname, cube in cubes.items()} metrics, extra_measures = _convert_measures( - cubes, pk_by_cube, fanned_out, issues) + cubes, pk_by_cube, plain_by_cube, fanned_out, issues) model["datasets"] = [ - _convert_cube(cname, cube, extra_joins.get(cname), + _convert_cube(cname, cube, plain_by_cube[cname], extra_joins.get(cname), extra_measures.get(cname), issues) for cname, cube in cubes.items() ] @@ -184,10 +188,7 @@ def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=False # Foreign-vendor extensions a previous export parked on the mapped view are # restored after the stash is written, so the CUBE entry stays first. - parked_exts = ((mapped_view.get("meta") or {}).get("ossie") or {}).get( - "custom_extensions") - if parked_exts: - model.setdefault("custom_extensions", []).extend(parked_exts) + _restore_parked_extensions(model, mapped_view.get("meta")) return dump_yaml({"version": OSSIE_VERSION, "semantic_model": [model]}), issues @@ -465,8 +466,7 @@ def _primary_key_of(cube, cname): if dim.get("primary_key")] -def _convert_cube(cname, cube, extra_joins, extra_measures, issues): - plain = _plain_members(cube, cname) +def _convert_cube(cname, cube, plain, extra_joins, extra_measures, issues): """Build one Ossie dataset from a Cube cube.""" scope = f"cube '{cname}'" ds = {"name": cname} @@ -480,8 +480,7 @@ def _convert_cube(cname, cube, extra_joins, extra_measures, issues): # GSF converters all reject anything shorter -- so a model that converts # cleanly here still cannot reach them. Better to say so at the point the # Ossie document is produced than to have it fail three hops later. - ds_scope = f"cube '{cname}'" - issues.add(IssueType.SOURCE_NOT_FULLY_QUALIFIED, ds_scope, + issues.add(IssueType.SOURCE_NOT_FULLY_QUALIFIED, scope, f"source '{ds['source']}' has {parts} part(s); several Ossie " f"converters (Databricks, Snowflake, NVIDIA GSF) require a " f"3-part catalog.schema.table, so qualify the cube's `sql_table` " @@ -530,8 +529,7 @@ def _convert_cube(cname, cube, extra_joins, extra_measures, issues): # Foreign-vendor extensions parked by a previous export are restored after the # stash is written, so the CUBE entry stays first and both survive. - if parked.get("custom_extensions"): - ds.setdefault("custom_extensions", []).extend(parked["custom_extensions"]) + _restore_parked_extensions(ds, cube.get("meta")) return ds @@ -686,10 +684,10 @@ def _convert_joins(cubes, skipped_files, issues): # common case. Only an orientation Ossie cannot express on its own -- # one_to_many (flipped) or one_to_one (no many side) -- needs recording. stash = {} - if (rel_type != "many_to_one" or cname != from_cube - or raw_rel != "many_to_one"): - # The last clause keeps a legacy spelling (`belongsTo`) exact - # without costing the modern spelling a stash entry. + # Testing the *declared* spelling, not the normalized one: a legacy + # `belongsTo` normalizes to many_to_one but has to come back spelled the + # way it was written, while the modern spelling costs no stash entry. + if raw_rel != "many_to_one": stash["declared_on"] = cname stash["relationship"] = raw_rel if rel_type == "one_to_many": @@ -792,23 +790,26 @@ class _MeasureResolver: Kept as a class because a calculated measure (`type: number`, and the other types in `CALCULATED_MEASURE_TYPES`) can reference other measures, which Cube resolves by inlining their full aggregate SQL -- so producing one measure's - expression may require producing another's first. Results are memoized and - reference cycles are rejected rather than recursed into. + expression may require producing another's first. Each measure's expression is + computed once and cached; a reference cycle is rejected rather than recursed + into. + + Note that inlining is inherently exponential in reference depth -- a chain where + each measure names the previous one twice doubles the SQL at every step -- and + that is Cube's own behaviour, not this converter's choice. The cache makes the + work proportional to the output rather than to the output times the depth; it + cannot make the output smaller. No limit is imposed, since any threshold would + reject a legitimate model to guard against a hand-written pathological one. """ def __init__(self, cubes, pk_by_cube, issues): self._pk = pk_by_cube self._issues = issues self._raw = {} - self._dimensions = {} + self._cache = {} for cname, cube in cubes.items(): for m in _as_named_list(cube.get("measures"), f"cube '{cname}' measures"): self._raw[(cname, require_str(m, "name", f"cube '{cname}': measure"))] = m - self._dimensions[cname] = { - d["name"] - for d in _as_named_list(cube.get("dimensions"), - f"cube '{cname}' dimensions") - } def measures(self): return self._raw @@ -827,6 +828,8 @@ def expression(self, cname, mname, stack=()): if key in stack: chain = " -> ".join(f"{c}.{m}" for c, m in stack + (key,)) raise ConversionError(f"measure reference cycle: {chain}") + if key in self._cache: + return self._cache[key] measure = self._raw[key] scope = f"{cname}.{mname}" mtype = snake(measure.get("type") or "") @@ -840,7 +843,7 @@ def expression(self, cname, mname, stack=()): IssueType.MULTI_STAGE_MEASURE_PARKED, scope, f"multi_stage measure (type '{mtype}'); preserved in " f"custom_extensions only") - return None + return self._remember(key, None) sql = measure.get("sql") filter_exprs = [ self._translate(f["sql"], cname, stack + (key,)) @@ -853,14 +856,14 @@ def expression(self, cname, mname, stack=()): raise ConversionError( f"measure '{scope}': type '{mtype}' requires 'sql'") expr = self._translate(sql, cname, stack + (key,)) - return filtered_operand(expr, filter_exprs) + return self._remember(key, filtered_operand(expr, filter_exprs)) if mtype == "count": if sql is None: - return primary_key_count_expression( - cname, self._pk.get(cname) or [], filter_exprs) + return self._remember(key, primary_key_count_expression( + cname, self._pk.get(cname) or [], filter_exprs)) operand = filtered_operand( self._operand(cname, sql, stack + (key,)), filter_exprs) - return f"COUNT({operand})" + return self._remember(key, f"COUNT({operand})") func = AGG_TO_OSSIE_FUNC.get(mtype) if func is None: raise ConversionError( @@ -870,8 +873,20 @@ def expression(self, cname, mname, stack=()): f"measure '{scope}': type '{mtype}' requires 'sql'") operand = filtered_operand( self._operand(cname, sql, stack + (key,)), filter_exprs) - return (f"COUNT(DISTINCT {operand})" if func == "COUNT_DISTINCT" - else f"{func}({operand})") + return self._remember( + key, f"COUNT(DISTINCT {operand})" if func == "COUNT_DISTINCT" + else f"{func}({operand})") + + def _remember(self, key, expr): + """Cache one measure's expression. + + A calculated measure inlines each reference's full SQL, so a measure + referenced from several places was recomputed once per reference -- and + recursively, so a chain of them cost O(depth * 2**depth) instead of the + O(2**depth) the inlined output is inherently worth. + """ + self._cache[key] = expr + return expr def _translate(self, sql, cname, stack): """Translate a Cube SQL string, inlining any measure reference. @@ -931,7 +946,7 @@ def _is_generated_part(measure): return bool(((measure.get("meta") or {}).get("ossie") or {}).get("part_of")) -def _convert_measures(cubes, pk_by_cube, fanned_out, issues): +def _convert_measures(cubes, pk_by_cube, plain_by_cube, fanned_out, issues): """Hoist every cube's measures into Ossie model-level metrics. A metric name is the measure name when globally unique, else @@ -955,6 +970,7 @@ def _convert_measures(cubes, pk_by_cube, fanned_out, issues): extra_measures = {} seen = set() for cname, cube in cubes.items(): + plain = plain_by_cube[cname] for index, measure in enumerate( _as_named_list(cube.get("measures"), f"cube '{cname}' measures")): @@ -972,8 +988,7 @@ def _convert_measures(cubes, pk_by_cube, fanned_out, issues): f"colliding measures in Cube") seen.add(metric_name) metric = _convert_measure(cname, mname, metric_name, measure, resolver, - fanned_out, _plain_members(cube, cname), - issues) + fanned_out, plain, issues) if metric is not None: metrics.append(metric) else: diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 762ae00f..1e233ca5 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -641,6 +641,9 @@ def resolve_base(): mname_raw = require_str(metric, "name", "metric") scope = f"metric '{mname_raw}'" stash = read_stash(metric) + # An empty `taken` on purpose: a measure name only has to be unique within + # its own cube, and which cube this lands on is not known yet. `_place` + # rejects a collision once the target is decided. mname = stash.get("name") or sanitize_name(mname_raw, scope, set()) if "measure" in stash: diff --git a/converters/cube/tools/interop_matrix.py b/converters/cube/tools/interop_matrix.py index 2bc2ec3b..89939abd 100644 --- a/converters/cube/tools/interop_matrix.py +++ b/converters/cube/tools/interop_matrix.py @@ -86,14 +86,28 @@ def repo_root(): - for parent in [Path(__file__).resolve(), *Path(__file__).resolve().parents]: + for parent in Path(__file__).resolve().parents: if (parent / "converters").is_dir() and (parent / "core-spec").is_dir(): return parent sys.exit("cannot locate the repository root from this script's path") +# Resolving a converter's dependencies on a cold cache is the slow part; a spoke that +# has not finished by then is hung rather than working. Without a timeout one such +# spoke takes the whole run down with it and prints nothing. +_TIMEOUT_S = 600 + + def run(cwd, argv): - return subprocess.run(argv, cwd=cwd, capture_output=True, text=True) + """Run `argv` in `cwd`, or return a synthetic failure rather than raising.""" + try: + return subprocess.run(argv, cwd=cwd, capture_output=True, text=True, + timeout=_TIMEOUT_S) + except subprocess.TimeoutExpired: + return subprocess.CompletedProcess( + argv, 1, "", f"timed out after {_TIMEOUT_S}s") + except FileNotFoundError as e: + return subprocess.CompletedProcess(argv, 1, "", f"{argv[0]}: {e.strerror}") def count_warnings(stderr): @@ -121,6 +135,22 @@ def import_issues(stderr): return found +# uv's wording for "this converter's environment could not be built", which is not +# the converter rejecting the model. Matching on message text is unavoidable (uv exits +# 1 either way) and will drift, so a message that stops matching shows up as a FAIL +# with the reason in the note column rather than as a silent mislabel. +_ENV_FAILURE_MARKERS = ( + "No solution found", + "no such command", + "Failed to spawn", + "does not exist", +) + + +def _is_environment_failure(stderr): + return any(marker in stderr for marker in _ENV_FAILURE_MARKERS) + + def produced_output(dest, is_dir): if not dest.exists(): return False @@ -225,9 +255,7 @@ def main(): note = "" if r.returncode != 0: tail = (r.stderr.strip().splitlines() or [""])[-1] - # A missing environment is not the converter rejecting the model. - skipped = "No solution found" in r.stderr or "no such command" in tail - result = "SKIP" if skipped else "FAIL" + result = "SKIP" if _is_environment_failure(r.stderr) else "FAIL" note = tail[:40] failures += result == "FAIL" else: From 19f3c49e4d32cef268a59aa969887cb969cbdbc9 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Tue, 4 Aug 2026 15:16:08 +0500 Subject: [PATCH 26/46] Fix eleven defects found in review, four of them silent Six from review, five found by checking the converter against Cube's own compiler and docs. All verified to reproduce first. Security: - A stashed file path (`cube_files`, `view_files`, `extra_files`) was joined onto `--output` unchecked, so `../../pwned.yml` wrote outside the target directory. The stash is part of the input document, so those paths are untrusted; `safe_relative_path` now rejects absolute and escaping paths. Cube would refuse the output: - Braces in free text. Cube compiles *every* string in a model as a Python f-string, so an unescaped `{` in a description, an AI context, or a parked JSON blob fails to compile. `ossie-cube export` of this repo's own tpcds fixture produced a model Cube could not compile at all, and the committed tpcds_cube fixture was not valid Cube either -- both are now the escaped form, and both compile. Import undoes the escaping; stashed Cube content is left byte-identical. - An Ossie field and metric of one name became two Cube members of one name ("orders cube: revenue defined more than once"), and a Cube cube with duplicate member names became an Ossie document the spec's own validator rejects. Silently wrong, and invisible to a round-trip test: - A `case` dimension carries conditions instead of `sql`, so it has no column to name -- import emitted the dimension's own name, claiming a column that does not exist. It now maps to a real `CASE WHEN ... END`, and export drops the redundant `sql` Cube would reject alongside `case`. - A computed primary key (`CONCAT(tenant_id, id)`) re-exported as a synthesized dimension reading a nonexistent column, with `primary_key: true` moved onto it -- changing what Cube counts. Import now records which entries are dimension names, since the Ossie document cannot tell them from column names afterwards. - A `switch` dimension came back as `type: string` with an orphaned `case` block. - A metric's `datatype` was dropped; only the count family could be inferred back. - A relationship's foreign-vendor `custom_extensions` vanished with no issue. - `sub_query: true` (a dimension referencing a measure) converted silently. - An empty dimension `sql` produced an empty Ossie expression silently. 317 tests, 96% coverage. Both fixtures compile in Cube, both Ossie fixtures pass validation/validate.py, and Cube generates byte-identical SQL for 10 queries across both fixtures before and after a round trip -- including the fan-out dedup and the multi-fact join split. --- converters/cube/README.md | 24 ++- converters/cube/src/ossie_cube/_common.py | 57 +++++ converters/cube/src/ossie_cube/cube_to_osi.py | 164 +++++++++++++-- converters/cube/src/ossie_cube/osi_to_cube.py | 147 ++++++++++--- .../tpcds_cube/model/cubes/customer.yml | 5 - .../tpcds_cube/model/cubes/date_dim.yml | 5 - .../fixtures/tpcds_cube/model/cubes/item.yml | 5 - .../fixtures/tpcds_cube/model/cubes/store.yml | 5 - .../tpcds_cube/model/cubes/store_sales.yml | 5 - .../model/views/tpcds_retail_model.yml | 41 ++-- converters/cube/tests/test_edge_cases.py | 199 ++++++++++++++++++ converters/cube/tests/test_osi_to_cube.py | 78 +++++++ .../cube/tests/test_roundtrip_properties.py | 11 +- 13 files changed, 654 insertions(+), 92 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index f3489dec..2232a2af 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -123,7 +123,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | `dataset.source` (`SELECT ...`) | `sql` | Cube requires exactly one of `sql` / `sql_table`. | | `dataset.description` | cube `description` | | | `dataset.ai_context` | cube `meta.ai_context` | Preserved for the round trip, but **inert in Cube** -- its agent ignores cube-level `ai_context`. Recorded as an issue. | -| `dataset.primary_key` | dimension(s) with `primary_key: true` | Composite = several. Export marks a dimension only when it is **scalar** — a single source column — since `primary_key: true` declares that dimension's own `sql` to be the key; a computed dimension or a merged `geo` one would declare the wrong thing even if its name matches. Anything left uncovered becomes a `public: false` scalar dimension, suffixed (`id_pk`) if the obvious name is taken. | +| `dataset.primary_key` | dimension(s) with `primary_key: true` | Composite = several. A Cube key can be an *expression*, and then the only name Ossie can carry is the dimension's -- which the Ossie document alone cannot tell apart from a column name afterwards, so import records it (`computed_primary_key`) and export puts `primary_key: true` back on that dimension instead of synthesizing one that reads a column of that name. Export marks a dimension only when it is **scalar** — a single source column — since `primary_key: true` declares that dimension's own `sql` to be the key; a computed dimension or a merged `geo` one would declare the wrong thing even if its name matches. Anything left uncovered becomes a `public: false` scalar dimension, suffixed (`id_pk`) if the obvious name is taken. | | `dataset.unique_keys` | `meta.ossie.unique_keys` | No native Cube slot; parked rather than dropped. | | field | `dimensions[]` entry | Export: a name that is not a valid Cube identifier is sanitized; a case-insensitive collision is an error, never a silent merge. | | `field.expression` | dimension `sql` | Dataset-scoped, so `{CUBE}.col` <-> `col`. Export emits `{CUBE}.column` for a raw column and `{CUBE.member}` for a declared member, and never spells the cube's own name (which would break under `extends`). | @@ -131,6 +131,11 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | `field.dimension.is_time` | `type: time` | Import sets `is_time: true` for a time dimension. | | `field.label` / `description` | dimension `title` / `description` | | | `field.ai_context.instructions` | dimension `meta.ai_context` | Cube's documented AI-only context field. | +| `field.expression` (`CASE WHEN …`) | dimension `case` | A Cube `case` dimension carries conditions instead of `sql` (Cube rejects both together), so it has no column to name. It maps to a real Ossie `CASE WHEN … THEN … ELSE … END`: a string `label` becomes a SQL literal, the `{sql: …}` form becomes that expression. The `case` block still rides in the stash, so export restores the Cube form exactly and drops the generated `sql`. | +| — | `type: switch` dimension | Maps to `String` like an ordinary dimension, and `String` maps back to `string` -- so the Cube type is recorded in the stash, or the dimension would return as a plain string one carrying an orphaned `case` block. | +| — | `sub_query: true` dimension | The sql references a *measure*, which an Ossie field expression has no form for. The reference is emitted as text with an `APPROXIMATED` issue, and the flag rides in the stash so export restores the working Cube form. | +| `metric.datatype` | `meta.ossie.datatype` | Cube has no field for a measure's result type. Import infers one for the count family (whose result type does not depend on the operand) and reads a parked one otherwise, so a `Decimal` sum survives. | +| relationship `custom_extensions` | cube `meta.ossie.join_extensions` | A Cube join entry takes only name/sql/relationship, so a relationship's foreign-vendor extensions ride on the declaring cube keyed by join target. | | — | `type: geo` dimension | An Ossie field holds one expression and a geo dimension has two, so it **splits** into `_latitude` / `_longitude` (`Float`). Reconstruction data rides on the latitude half. See [Geo dimensions](#geo-dimensions). | | relationship | `joins[]` on a cube | `many_to_one` on cube A -> `from: A`(many), `to: B`(one). `one_to_many` is flipped so Ossie's `from` is the many side; the declared side and type are stashed so export restores the original. | | `from_columns` / `to_columns` | join `sql` | Only an AND-chain of equalities between two member references maps. Anything else (non-equi, range, literal, third cube) is preserved verbatim in the stash. | @@ -142,7 +147,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | several aggregates in one expression | one `public: false` measure per aggregate + a `type: number` measure referencing them | Each part is declared on the cube its own operand reads, so Cube corrects row multiplication per aggregate rather than once for the whole expression. The parts carry `meta.ossie.part_of`, and import skips them and inlines their SQL back through the references -- recovering the original expression exactly. | | anything else | `type: number` (calculated) | A `{other_measure}` reference is **inlined**, because that is what Cube itself does; Ossie has no metric-to-metric reference. | | — | measure `filters` | Folded into `CASE WHEN … THEN … END` inside the aggregate, exactly as Cube's own `applyMeasureFilters` renders it. | -| `metric.datatype` | — | Import emits `Integer` for the count family, whose result type Cube does know, and omits it otherwise. | +| `metric.datatype` | — | Import emits `Integer` for the count family, whose result type Cube does know, and reads a parked one otherwise. | | `metric.description` / `ai_context` | measure `description` / `meta.ai_context` | | | `custom_extensions[CUBE]` | everything Cube-only | Import stashes; export restores -- keeping `Cube -> Ossie -> Cube` lossless. | | foreign-vendor `custom_extensions` | `meta.ossie.custom_extensions` | Parked so a multi-vendor Ossie model survives the round trip. | @@ -161,6 +166,14 @@ Ossie dialect enum has no `CUBE` entry -- so import emits `ANSI_SQL`, and export prefers `ANSI_SQL` with `--dialect` prepending a warehouse dialect (e.g. `SNOWFLAKE` for a Snowflake-backed Cube model). +**Braces are escaped in free text.** Cube compiles *every* string in a YAML model as +a Python f-string (`f""` in `YamlCompiler`; only the handful of boolean-ish keys +in the compiler's `nonStringFields` are exempt), so an unescaped `{` in a description, +an AI context, or a parked JSON blob is read as an interpolation and **the whole model +fails to compile**. Export writes `\{` / `\}`, which is Cube's escape for a literal +brace; import undoes it. Content restored from a Cube stash is left byte-identical -- +it was written for Cube already. Only strings sourced from Ossie are escaped. + **String literals** are handled asymmetrically, on purpose. Cube compiles a YAML `sql` value as a Python f-string (`f""` in `YamlCompiler`), so `{CUBE}.col` interpolates *anywhere* in the value -- SQL's own quotes mean nothing to it. So on @@ -352,6 +365,13 @@ Conversion raises a `ConversionError` (rather than guessing or emitting somethin invalid) when an input breaks one of these: - a cube has neither or both of `sql` / `sql_table` (Cube requires exactly one); +- two members of one cube share a name, or an Ossie field and metric map to the same + Cube member -- Cube keeps one namespace per cube for dimensions, measures and + segments alike ("orders cube: revenue defined more than once"), and emitting the + clash produced a model Cube refuses to compile and an Ossie document the spec's own + validator rejects for a duplicate field name; +- a stashed file path is absolute or escapes the output directory: the stash is part + of the input document, so a path in it is untrusted; - a cube uses `extends` -- resolving it means reproducing Cube's definition-merge semantics exactly, so it is refused rather than half-applied; - a bare `type: count` measure's cube declares no primary key; diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index 200ef0fa..649a0e8c 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -385,6 +385,63 @@ def repl(m): return out, changed +def safe_relative_path(path, what): + """Validate a stashed file path before it is used as an output filename. + + The stash is part of the input document, so a path in it is untrusted: an entry + like `../../etc/thing.yml` in `cube_files` would make export write outside the + directory the caller named. Refuses anything that is not a plain relative path + inside the output root. + """ + raw = str(path) + if not raw.strip(): + raise ConversionError(f"{what}: stashed file path is empty") + if raw.startswith(("/", "\\")) or re.match(r"^[A-Za-z]:", raw): + raise ConversionError( + f"{what}: stashed file path '{raw}' is absolute; expected a path " + f"relative to the output directory") + parts = [p for p in raw.replace("\\", "/").split("/") if p not in ("", ".")] + if any(p == ".." for p in parts): + raise ConversionError( + f"{what}: stashed file path '{raw}' escapes the output directory") + if not parts: + raise ConversionError(f"{what}: stashed file path '{raw}' names no file") + return "/".join(parts) + + +def escape_braces_for_cube(value): + """Escape `{`/`}` in every string of `value` (recursing into lists and dicts). + + Cube compiles *every* string in a YAML model as a Python f-string -- only the + handful of boolean-ish keys in the compiler's `nonStringFields` are exempt -- so + an unescaped brace in a description, an AI context, or a parked JSON blob is read + as an interpolation and the model fails to compile. `\\{` / `\\}` is Cube's escape + for a literal brace. + + Applied only to strings this converter puts there from Ossie. Content restored + from a Cube stash is left byte-identical: it was written for Cube in the first + place, so its braces are already whatever Cube needs them to be. + """ + if isinstance(value, str): + return value.replace("{", "\\{").replace("}", "\\}") + if isinstance(value, list): + return [escape_braces_for_cube(v) for v in value] + if isinstance(value, dict): + return {k: escape_braces_for_cube(v) for k, v in value.items()} + return value + + +def unescape_braces_from_cube(value): + """Undo `escape_braces_for_cube` when reading a Cube model back.""" + if isinstance(value, str): + return value.replace("\\{", "{").replace("\\}", "}") + if isinstance(value, list): + return [unescape_braces_from_cube(v) for v in value] + if isinstance(value, dict): + return {k: unescape_braces_from_cube(v) for k, v in value.items()} + return value + + def quoted_runs(sql): """Split SQL into (text, is_quoted) runs, delimiters included in the quoted run. diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 7e5bbedc..804607e3 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -39,6 +39,7 @@ AGG_TO_RESULT_DATATYPE, CALCULATED_MEASURE_TYPES, DIALECT_ANSI, + DATATYPE_TO_DIM_TYPE, DIM_TYPE_TO_DATATYPE, DOTTED_REF_RE, FANOUT_UNSAFE_AGGS, @@ -58,6 +59,7 @@ snake_keys, source_part_count, sql_is_reversible, + unescape_braces_from_cube, view_file, read_stash, write_stash, @@ -129,7 +131,8 @@ def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=False model = {"name": model_name or mapped_name or "cube_model"} if mapped_view.get("description"): - model["description"] = mapped_view["description"] + model["description"] = unescape_braces_from_cube( + mapped_view["description"]) ai = _ai_context_from_meta(mapped_view.get("meta")) if ai: model["ai_context"] = ai @@ -235,6 +238,7 @@ def _collect(files, issues): raise ConversionError( f"cube '{name}' uses `extends`, which this converter does not " f"resolve yet; flatten the cube or exclude the file") + _reject_duplicate_members(name, entry) cubes[name] = entry cube_paths[name] = fname for entry in _as_named_list(parsed.get("views"), f"'{fname}' views"): @@ -248,6 +252,29 @@ def _collect(files, issues): return cubes, cube_paths, views, view_paths, extra_files +def _reject_duplicate_members(cname, cube): + """Refuse a cube whose members collide, which Cube refuses too. + + Cube keeps one namespace per cube for dimensions, measures and segments + ("orders cube: d defined more than once"). Converting such a cube anyway emitted + two Ossie fields of the same name -- a document the spec's own validator rejects + for a duplicate field name -- so it is caught here instead. + """ + seen = {} + for kind in ("dimensions", "measures", "segments"): + for member in _as_named_list(cube.get(kind), f"cube '{cname}' {kind}"): + mname = member.get("name") + if not mname: + continue + key = str(mname).lower() + if key in seen: + raise ConversionError( + f"cube '{cname}': '{mname}' is defined more than once " + f"({seen[key]} and {kind[:-1]}); Cube keeps one member namespace " + f"per cube, so rename one.") + seen[key] = kind[:-1] + + def _as_named_list(value, what): """Normalize a Cube collection to a list of dicts carrying `name`. @@ -377,10 +404,10 @@ def _ai_context_from_meta(meta): """ if not isinstance(meta, dict): return None - parked = (meta.get("ossie") or {}).get("ai_context") + parked = parked_of(meta).get("ai_context") if parked: return parked - text = meta.get("ai_context") + text = unescape_braces_from_cube(meta.get("ai_context")) if isinstance(text, str) and text.strip(): # Kept verbatim rather than stripped: a folded block scalar carries a # trailing newline, and normalizing it away here would make the round trip @@ -389,6 +416,18 @@ def _ai_context_from_meta(meta): return None +def parked_of(meta): + """The `meta.ossie` subtree, with Cube's brace escaping undone. + + Export escapes `{`/`}` in everything it parks, because Cube compiles every string + in a model as a Python f-string and an unescaped brace breaks compilation. Reading + it back has to undo that, or a parked JSON blob comes home with backslashes in it. + """ + if not isinstance(meta, dict): + return {} + return unescape_braces_from_cube(meta.get("ossie") or {}) + + def _meta_without_ai_context(meta): """The part of a Cube `meta` with no Ossie home, for the stash. @@ -430,7 +469,7 @@ def _restore_parked_extensions(obj, meta): parked entries are stripped by `_meta_without_ai_context` and never come back, which would make `Ossie -> Cube -> Ossie` lose them. """ - parked = ((meta or {}).get("ossie") or {}).get("custom_extensions") + parked = parked_of(meta).get("custom_extensions") if parked: obj.setdefault("custom_extensions", []).extend(parked) @@ -486,10 +525,10 @@ def _convert_cube(cname, cube, plain, extra_joins, extra_measures, issues): f"3-part catalog.schema.table, so qualify the cube's `sql_table` " f"if the model needs to convert onward") if cube.get("description"): - ds["description"] = cube["description"] + ds["description"] = unescape_braces_from_cube(cube["description"]) meta = cube.get("meta") if isinstance(cube.get("meta"), dict) else {} - parked = meta.get("ossie") or {} + parked = parked_of(meta) ai = _ai_context_from_meta(meta) if ai: ds["ai_context"] = ai @@ -509,6 +548,15 @@ def _convert_cube(cname, cube, plain, extra_joins, extra_measures, issues): primary_key = _primary_key_of(cube, cname) if primary_key: ds["primary_key"] = primary_key + # Ossie's `primary_key` names columns, but a Cube key can be an expression + # (`CONCAT(tenant_id, id)`), and then the only name there is to write is the + # dimension's. Which of the two an entry is cannot be told from the Ossie + # document afterwards -- a hand-authored model may name a real column that a + # computed field happens to share a name with -- so it is recorded here rather + # than guessed on the way back. + computed = [n for n in primary_key if n not in plain] + if computed: + stash["computed_primary_key"] = computed if extra_joins: stash["extra_joins"] = extra_joins if extra_measures: @@ -546,6 +594,36 @@ def _convert_dimension(cname, dname, dim, plain, issues): stash = {} sql = dim.get("sql") + case = dim.get("case") + if case is not None: + # A `case` dimension carries conditions instead of `sql` (Cube rejects both + # together), so there is no column to name. Ossie expresses this natively as a + # CASE expression -- emitting the dimension's own name instead, as this used to, + # claimed a physical column that does not exist. The `case` block still rides in + # the stash, so export restores the Cube form exactly. + expr = _case_expression(cname, dname, case) + field = { + "name": dname, + "expression": { + "dialects": [{"dialect": DIALECT_ANSI, "expression": expr}]}, + } + return [_finish_dimension_field(cname, dname, dim, field, stash, issues)] + if dim.get("sub_query"): + # `sub_query: true` means the sql references a *measure* (`{users.count}`), + # which Cube resolves by aggregating in a subquery. An Ossie field expression + # is dataset-scoped SQL over columns, so the reference survives as text but + # nothing downstream can resolve it. The flag rides in the stash, so export + # restores the working Cube form. + issues.add(IssueType.APPROXIMATED, f"{cname}.{dname}", + "sub_query dimension references a measure, which an Ossie field " + "expression has no form for; the reference is emitted as text and " + "only Cube can resolve it") + if sql is not None and not str(sql).strip(): + # Cube compiles `sql: ''` without complaint, so this is not refused -- but the + # resulting Ossie expression is empty, which no consumer can evaluate. + issues.add(IssueType.APPROXIMATED, f"{cname}.{dname}", + "dimension sql is empty, so the Ossie expression is empty too; " + "Cube accepts this but no consumer can evaluate it") if sql is None: # No `sql` means the same-named physical column. expr = dname @@ -564,20 +642,32 @@ def _convert_dimension(cname, dname, dim, plain, issues): "name": dname, "expression": {"dialects": [{"dialect": DIALECT_ANSI, "expression": expr}]}, } + return [_finish_dimension_field(cname, dname, dim, field, stash, issues)] + + +def _finish_dimension_field(cname, dname, dim, field, stash, issues): + """Attach the datatype, labels, AI context and stash shared by every dimension.""" + dtype = snake(dim.get("type") or "string") datatype = DIM_TYPE_TO_DATATYPE.get(dtype) if not datatype: raise ConversionError( f"cube '{cname}': dimension '{dname}' has unknown type '{dtype}'") # A precise datatype parked by a previous export wins over the default the Cube # type maps to, since Cube itself cannot hold the distinction. - parked_dt = ((dim.get("meta") or {}).get("ossie") or {}).get("datatype") + parked_dt = parked_of(dim.get("meta")).get("datatype") field["datatype"] = parked_dt or datatype + # `type` is normally regenerated from the datatype, so it costs no stash entry. + # A `switch` dimension is the exception: it maps to String like an ordinary one, + # and String maps back to `string`, so the type has to be recorded or the + # dimension comes back as a plain string one carrying an orphaned `case` block. + if DATATYPE_TO_DIM_TYPE.get(field["datatype"]) != dtype: + stash["dim_type"] = dtype if dtype == "time": field["dimension"] = {"is_time": True} if dim.get("title"): - field["label"] = dim["title"] + field["label"] = unescape_braces_from_cube(dim["title"]) if dim.get("description"): - field["description"] = dim["description"] + field["description"] = unescape_braces_from_cube(dim["description"]) ai = _ai_context_from_meta(dim.get("meta")) if ai: field["ai_context"] = ai @@ -594,7 +684,48 @@ def _convert_dimension(cname, dname, dim, plain, issues): # `meta.ossie` are restored after the stash is written, so the CUBE entry stays # first -- the same ordering datasets use. _restore_parked_extensions(field, dim.get("meta")) - return [field] + return field + + +def _case_expression(cname, dname, case): + """Translate a Cube `case` dimension into an Ossie CASE expression. + + A string `label` becomes a SQL literal; the `{sql: ...}` form becomes that + expression. Both are exactly what Cube itself renders, so nothing is approximated. + """ + if not isinstance(case, dict): + raise ConversionError( + f"cube '{cname}': dimension '{dname}' has a non-mapping `case`") + parts = [] + for branch in (case.get("when") or []): + if not isinstance(branch, dict) or branch.get("sql") is None: + raise ConversionError( + f"cube '{cname}': dimension '{dname}' has a `case.when` entry with " + f"no `sql`") + condition, _ = cube_sql_to_ossie(branch["sql"], cname) + parts.append(f"WHEN {condition} THEN {_case_label(cname, dname, branch)}") + if not parts: + raise ConversionError( + f"cube '{cname}': dimension '{dname}' has a `case` with no `when` " + f"branches") + otherwise = case.get("else") + if isinstance(otherwise, dict) and "label" in otherwise: + parts.append(f"ELSE {_case_label(cname, dname, otherwise)}") + return "CASE " + " ".join(parts) + " END" + + +def _case_label(cname, dname, holder): + """One `label`, as SQL: a plain value is a literal, `{sql: ...}` an expression.""" + label = holder.get("label") + if isinstance(label, dict): + if label.get("sql") is None: + raise ConversionError( + f"cube '{cname}': dimension '{dname}' has a `label` object with no " + f"`sql`") + translated, _ = cube_sql_to_ossie(label["sql"], cname) + return translated + text = str(label if label is not None else "") + return "'" + text.replace("'", "''") + "'" def _convert_geo_dimension(cname, dname, dim, issues): @@ -717,6 +848,12 @@ def _convert_joins(cubes, skipped_files, issues): rel = {"name": name, "from": from_cube, "to": to_cube, "from_columns": from_cols, "to_columns": to_cols} write_stash(rel, stash) + # Foreign-vendor extensions a previous export parked on the declaring + # cube, keyed by join target -- a Cube join entry has no `meta` of its own. + parked_joins = parked_of(cube.get("meta")).get( + "join_extensions") or {} + if parked_joins.get(target): + rel.setdefault("custom_extensions", []).extend(parked_joins[target]) relationships.append(rel) return relationships, extra_joins @@ -1034,11 +1171,14 @@ def _convert_measure(cname, mname, metric_name, measure, resolver, fanned_out, "name": metric_name, "expression": {"dialects": [{"dialect": DIALECT_ANSI, "expression": expr}]}, } - datatype = AGG_TO_RESULT_DATATYPE.get(mtype) + # A datatype parked by a previous export wins: Cube has no field for a measure's + # result type, and only the count family can be inferred from the aggregate. + parked_dt = parked_of(measure.get("meta")).get("datatype") + datatype = parked_dt or AGG_TO_RESULT_DATATYPE.get(mtype) if datatype: metric["datatype"] = datatype if measure.get("description"): - metric["description"] = measure["description"] + metric["description"] = unescape_braces_from_cube(measure["description"]) ai = _ai_context_from_meta(measure.get("meta")) if ai: metric["ai_context"] = ai diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 1e233ca5..7590b0ee 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -35,6 +35,7 @@ from collections import deque from ._common import ( + AGG_TO_RESULT_DATATYPE, DATATYPE_TO_DIM_TYPE, DEFAULT_DATATYPE_FOR_CUBE_TYPE, OSSIE_FUNC_TO_AGG, @@ -42,6 +43,7 @@ ConversionError, cube_file, dump_yaml, + escape_braces_for_cube, examples_of, foreign_vendor_extensions, instructions_of, @@ -54,6 +56,7 @@ read_stash, referenced_datasets, require_str, + safe_relative_path, sanitize_name, synonyms_of, view_file, @@ -154,7 +157,8 @@ def _convert_model(model, dialect, base_cube, issues): ds, dim_names_by_cube[cname], dialect) pk_by_cube[cname] = [str(c) for c in (ds.get("primary_key") or [])] - joins_by_cube = _build_joins(relationships, cube_names, issues) + joins_by_cube, join_parked_by_cube = _build_joins( + relationships, cube_names, issues) measures_by_cube = _build_measures( model, cube_names, members_by_cube, inline_sql_by_cube, pk_by_cube, datasets, relationships, base_cube, dialect, issues) @@ -168,8 +172,10 @@ def _convert_model(model, dialect, base_cube, issues): cube = _build_cube(ds, cname, dim_names_by_cube[cname], inline_sql_by_cube[cname], members_by_cube[cname], joins_by_cube.get(cname), measures_by_cube.get(cname), - dialect, issues) - path = stashed_paths.get(cname) or cube_file(cname) + join_parked_by_cube.get(cname), dialect, issues) + stashed = stashed_paths.get(cname) + path = (safe_relative_path(stashed, f"cube '{cname}'") if stashed + else cube_file(cname)) files_content.setdefault(path, {}).setdefault("cubes", []).append(cube) for vpath, views in _build_views(model, model_stash, cube_names, relationships, @@ -181,7 +187,7 @@ def _convert_model(model, dialect, base_cube, issues): # Files a prior import could not convert (`.js` models, Jinja-templated YAML, # non-model YAML) restore verbatim. for fname, text in (model_stash.get("extra_files") or {}).items(): - files[fname] = text + files[safe_relative_path(fname, "stashed extra file")] = text return files, issues @@ -220,18 +226,24 @@ def _ai_context_to_meta(ai_context): def _build_meta(ai_context, stashed_meta, parked_extra): """Assemble a Cube `meta` from the Ossie AI context, a stashed original meta, - and anything Ossie-only that needs parking.""" + and anything Ossie-only that needs parking. + + Braces are escaped in everything sourced from Ossie: Cube compiles every string in + a model as a Python f-string, so an unescaped `{` -- routine in a parked JSON blob, + and plausible in AI instructions -- makes the whole model fail to compile. The + stashed original meta is left byte-identical; it was written for Cube already. + """ prose, parked_ai = _ai_context_to_meta(ai_context) meta = {} if prose: - meta["ai_context"] = prose + meta["ai_context"] = escape_braces_for_cube(prose) for key, value in (stashed_meta or {}).items(): meta[key] = value parked = dict(parked_extra or {}) if parked_ai is not None: parked["ai_context"] = parked_ai if parked: - meta["ossie"] = parked + meta["ossie"] = escape_braces_for_cube(parked) return meta @@ -248,7 +260,7 @@ def _ordered(obj, order): # --- cubes ---------------------------------------------------------------------- def _build_cube(ds, cname, dim_names, inline_sql, ref_members, joins, measures, - dialect, issues): + join_extensions, dialect, issues): ds_name = ds["name"] scope = f"dataset '{ds_name}'" stash = read_stash(ds) @@ -257,7 +269,7 @@ def _build_cube(ds, cname, dim_names, inline_sql, ref_members, joins, measures, kind, value = parse_source(ds.get("source"), ds_name) cube[kind] = value if ds.get("description"): - cube["description"] = ds["description"] + cube["description"] = escape_braces_for_cube(ds["description"]) parked = {} if ds.get("unique_keys"): @@ -267,6 +279,12 @@ def _build_cube(ds, cname, dim_names, inline_sql, ref_members, joins, measures, foreign = foreign_vendor_extensions(ds) if foreign: parked["custom_extensions"] = foreign + if join_extensions: + parked["join_extensions"] = join_extensions + issues.add(IssueType.PARKED_IN_META, scope, + f"a Cube join carries no metadata field, so relationship " + f"custom_extensions for {', '.join(sorted(join_extensions))} are " + f"parked under meta.ossie.join_extensions") cube_extras = dict(stash.get("cube_extras") or {}) stashed_meta = cube_extras.pop("meta", None) meta = _build_meta(ds.get("ai_context"), stashed_meta, parked) @@ -277,7 +295,7 @@ def _build_cube(ds, cname, dim_names, inline_sql, ref_members, joins, measures, "Cube's agent reads ai_context only on views and members, " "so this cube-level value has no effect in Cube") - dimensions, by_name_scalar, by_column = _build_dimensions( + dimensions, by_name_scalar, by_column, by_name_computed = _build_dimensions( ds, cname, dim_names, inline_sql, ref_members, dialect, issues) # Resolve each `primary_key` entry to the dimension Cube should mark. A # dimension only qualifies when it is *scalar* -- backed by a single source @@ -286,6 +304,7 @@ def _build_cube(ds, cname, dim_names, inline_sql, ref_members, joins, measures, # and a merged geo dimension has no single sql at all, so neither counts even # when its name matches. Anything left uncovered gets a private dimension. pk_names = [] + computed_keys = set(stash.get("computed_primary_key") or []) taken = {d["name"].lower() for d in dimensions} for entry in (ds.get("primary_key") or []): entry = str(entry) @@ -295,6 +314,14 @@ def _build_cube(ds, cname, dim_names, inline_sql, ref_members, joins, measures, if match: pk_names.append(match) continue + # A dimension name import recorded because the Cube key was an expression: + # `primary_key: true` goes back on that dimension, so Cube keys on the same + # expression the source model did. Synthesizing one instead would read a column + # that does not exist. Only entries import flagged qualify -- for anything else + # a name match is not evidence, since Ossie `primary_key` names columns. + if entry in computed_keys and entry in by_name_computed: + pk_names.append(by_name_computed[entry]) + continue name = _unique_pk_dimension_name(entry, taken) taken.add(name.lower()) detail = (f"primary key '{entry}' is not backed by a scalar dimension; " @@ -328,11 +355,36 @@ def _build_cube(ds, cname, dim_names, inline_sql, ref_members, joins, measures, if measures: cube["measures"] = measures + # Cube keeps one namespace per cube for dimensions, measures and segments alike + # ("orders cube: revenue defined more than once"), so a field and a metric of the + # same name make a model Cube refuses to compile. Checked here, where every + # member the cube will carry is known -- including a synthesized primary key, a + # merged geo dimension, and measures restored from the stash. + _reject_member_collisions(cname, dimensions, measures, cube_extras, issues) + for key, value in cube_extras.items(): cube[key] = value return _ordered(cube, _CUBE_KEY_ORDER) +def _reject_member_collisions(cname, dimensions, measures, cube_extras, issues): + seen = {} + groups = [("dimension", dimensions), ("measure", measures), + ("segment", cube_extras.get("segments") or [])] + for kind, members in groups: + for member in members: + if not isinstance(member, dict) or not member.get("name"): + continue + key = str(member["name"]).lower() + if key in seen: + first_kind, first_name = seen[key] + raise ConversionError( + f"Cube '{cname}': {first_kind} '{first_name}' and {kind} " + f"'{member['name']}' share a name; Cube keeps one member " + f"namespace per cube, so rename one in the Ossie model.") + seen[key] = (kind, member["name"]) + + def _reference_members(ds, dim_names, dialect): """Members that must be addressed as `{CUBE.member}` rather than `{CUBE}.column`. @@ -418,19 +470,21 @@ def _build_dimensions(ds, cname, dim_names, inline_sql, ref_members, dialect, issues): """Build a cube's dimensions from an Ossie dataset's fields. - Returns (dimensions, by_name_scalar, by_column) -- the two maps are what - primary-key resolution matches against, and both hold only *scalar* dimensions - (those whose expression is a single source column). A computed dimension and a - merged geo dimension are deliberately absent from both: Cube's - `primary_key: true` declares that dimension's own sql to be the key, so marking - either would declare something other than the column Ossie named. Fields - carrying a `geo` stash are re-merged into the single Cube dimension they were - split from. + Returns (dimensions, by_name_scalar, by_column, by_name_computed). + + The first two maps hold only *scalar* dimensions (those whose expression is a + single source column), which are the ones Cube's `primary_key: true` can mark + without declaring something other than the column Ossie named. `by_name_computed` + holds the rest by name, except merged geo dimensions -- a computed dimension is + still the right thing to mark when Ossie's `primary_key` names it, because import + writes dimension *names* there and a computed key has no column to name instead. + Fields carrying a `geo` stash are re-merged into the single Cube dimension they + were split from. Dimension names come from `dim_names` (see `_resolve_dimension_names`) rather than being sanitized again here. """ ds_name = ds["name"] - by_name_scalar, by_column = {}, {} + by_name_scalar, by_column, by_name_computed = {}, {}, {} # Built by target dimension name rather than by list position: a geo dimension # is assembled from two fields that may appear in either order and need not be # adjacent, so an insertion index computed mid-loop is not a safe way to hold @@ -464,11 +518,17 @@ def _build_dimensions(ds, cname, dim_names, inline_sql, ref_members, dialect, else: dim["sql"] = ossie_expr_to_cube_sql( expr, cname, ref_members, (), inline_sql={cname: inline_sql}) + if stash.get("case") is not None: + # A `case` dimension carries its conditions instead of `sql`, and Cube + # rejects a dimension declaring both ("dimensions.size does not match any + # of the allowed types"). The generated sql is redundant anyway: the CASE + # expression it holds is what `case` says. + dim.pop("sql", None) dim["type"] = _dimension_type(field, stash, f"{ds_name}.{fname}", issues) if field.get("label"): - dim["title"] = field["label"] + dim["title"] = escape_braces_for_cube(field["label"]) if field.get("description"): - dim["description"] = field["description"] + dim["description"] = escape_braces_for_cube(field["description"]) parked = {} foreign = foreign_vendor_extensions(field) if foreign: @@ -481,7 +541,11 @@ def _build_dimensions(ds, cname, dim_names, inline_sql, ref_members, dialect, dt = field.get("datatype") if dt and DEFAULT_DATATYPE_FOR_CUBE_TYPE.get(dim["type"]) != dt: parked["datatype"] = dt - extras = {k: v for k, v in stash.items() if k not in ("sql", "type", "meta")} + # Keys the exporter consumes itself rather than writing onto the dimension: + # `sql`/`type` are an older stash shape, `dim_type` supplies the Cube type, + # and `geo` was used to merge the halves back together. + extras = {k: v for k, v in stash.items() + if k not in ("sql", "type", "dim_type", "meta", "geo")} meta = _build_meta(field.get("ai_context"), stash.get("meta"), parked) if meta: dim["meta"] = meta @@ -494,6 +558,8 @@ def _build_dimensions(ds, cname, dim_names, inline_sql, ref_members, dialect, # it as the key. Reachable by its own name and by that column's name. by_name_scalar[dname] = dname by_column.setdefault(expr.strip(), dname) + else: + by_name_computed[dname] = dname # Both halves are guaranteed present by _resolve_dimension_names, which # validates the pair before anything is built. @@ -507,7 +573,8 @@ def _build_dimensions(ds, cname, dim_names, inline_sql, ref_members, dialect, # A name in `order` with nothing built is a field dropped for want of a usable # dialect; it simply does not appear. - return [built[n] for n in order if n in built], by_name_scalar, by_column + return ([built[n] for n in order if n in built], by_name_scalar, by_column, + by_name_computed) def _unique_pk_dimension_name(entry, taken): @@ -535,6 +602,10 @@ def _dimension_type(field, stash, scope, issues): if "type" in stash: # An older stash from before datatypes were mapped natively. return stash["type"] + if stash.get("dim_type"): + # A Cube type the datatype cannot regenerate (`switch` maps to String like an + # ordinary dimension, and String maps back to `string`), recorded on import. + return stash["dim_type"] datatype = field.get("datatype") explicit_is_time = (field.get("dimension") or {}).get("is_time") if datatype: @@ -567,8 +638,14 @@ def _build_joins(relationships, cube_names, issues): A stashed `declared_on`/`relationship` restores the original declaring side and type. A hand-authored relationship is declared on its `from` (many) cube as `many_to_one`, which is the orientation Ossie already guarantees. + + Returns (joins_by_cube, parked_by_cube). A Cube join entry takes only + name/sql/relationship, so a relationship's foreign-vendor extensions have nowhere + to go on the join itself; they ride on the declaring cube's `meta.ossie` keyed by + the join target, which keeps a multi-vendor model lossless. """ joins_by_cube = {} + parked_by_cube = {} for rel in relationships: rname = rel.get("name", "") from_cols = rel.get("from_columns") or [] @@ -615,9 +692,12 @@ def _build_joins(relationships, cube_names, issues): f"relationship '{rname}'", "a Cube join carries no metadata field, so relationship " "ai_context has nowhere to go and is dropped") + foreign = foreign_vendor_extensions(rel) + if foreign: + parked_by_cube.setdefault(own, {})[other] = foreign joins_by_cube.setdefault(own, []).append( _ordered(join, ["name", "sql", "relationship"])) - return joins_by_cube + return joins_by_cube, parked_by_cube # --- measures ------------------------------------------------------------------- @@ -809,13 +889,19 @@ def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_ def _apply_measure_metadata(metric, measure, stash): if stash.get("title"): - measure["title"] = stash["title"] + measure["title"] = escape_braces_for_cube(stash["title"]) if metric.get("description"): - measure["description"] = metric["description"] + measure["description"] = escape_braces_for_cube(metric["description"]) parked = {} foreign = foreign_vendor_extensions(metric) if foreign: parked["custom_extensions"] = foreign + # Cube has no field for a measure's result type. Import infers one only for the + # count family, whose result type does not depend on the operand, so anything else + # (a `Decimal` sum) would be lost without parking it. + datatype = metric.get("datatype") + if datatype and datatype != AGG_TO_RESULT_DATATYPE.get(measure.get("type")): + parked["datatype"] = datatype meta = _build_meta(metric.get("ai_context"), stash.get("meta"), parked) if meta: measure["meta"] = meta @@ -877,18 +963,21 @@ def _build_views(model, model_stash, cube_names, relationships, datasets, view = dict(view) if vname == mapped: if model.get("description"): - view["description"] = model["description"] + view["description"] = escape_braces_for_cube( + model["description"]) meta = _build_meta(model.get("ai_context"), view.get("meta"), parked) if meta: view["meta"] = meta - path = paths.get(vname) or view_file(vname) + stashed = paths.get(vname) + path = (safe_relative_path(stashed, f"view '{vname}'") if stashed + else view_file(vname)) out.setdefault(path, []).append(view) return out vname = sanitize_name(model.get("name", "model"), "Model", set()) view = {"name": vname} if model.get("description"): - view["description"] = model["description"] + view["description"] = escape_braces_for_cube(model["description"]) meta = _build_meta(model.get("ai_context"), None, parked) if meta: view["meta"] = meta diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/customer.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/customer.yml index 206ea8d8..1cccefd8 100644 --- a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/customer.yml +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/customer.yml @@ -15,11 +15,6 @@ # specific language governing permissions and limitations # under the License. -# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then -# kept as a fixture: the converter guide asks every converter to use the TPC-DS -# model as its baseline. Exercises a five-cube star, cross-cube calculated -# measures, a synthesized primary-key dimension, and meta.ossie parking. - cubes: - name: customer sql_table: tpcds.public.customer diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/date_dim.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/date_dim.yml index 8855e811..f669ef88 100644 --- a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/date_dim.yml +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/date_dim.yml @@ -15,11 +15,6 @@ # specific language governing permissions and limitations # under the License. -# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then -# kept as a fixture: the converter guide asks every converter to use the TPC-DS -# model as its baseline. Exercises a five-cube star, cross-cube calculated -# measures, a synthesized primary-key dimension, and meta.ossie parking. - cubes: - name: date_dim sql_table: tpcds.public.date_dim diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/item.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/item.yml index 79cf11de..7b6138ee 100644 --- a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/item.yml +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/item.yml @@ -15,11 +15,6 @@ # specific language governing permissions and limitations # under the License. -# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then -# kept as a fixture: the converter guide asks every converter to use the TPC-DS -# model as its baseline. Exercises a five-cube star, cross-cube calculated -# measures, a synthesized primary-key dimension, and meta.ossie parking. - cubes: - name: item sql_table: tpcds.public.item diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store.yml index c3d59832..8068a176 100644 --- a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store.yml +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store.yml @@ -15,11 +15,6 @@ # specific language governing permissions and limitations # under the License. -# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then -# kept as a fixture: the converter guide asks every converter to use the TPC-DS -# model as its baseline. Exercises a five-cube star, cross-cube calculated -# measures, a synthesized primary-key dimension, and meta.ossie parking. - cubes: - name: store sql_table: tpcds.public.store diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml index 45eb7c44..bdc2e649 100644 --- a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml @@ -15,11 +15,6 @@ # specific language governing permissions and limitations # under the License. -# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then -# kept as a fixture: the converter guide asks every converter to use the TPC-DS -# model as its baseline. Exercises a five-cube star, cross-cube calculated -# measures, a synthesized primary-key dimension, and meta.ossie parking. - cubes: - name: store_sales sql_table: tpcds.public.store_sales diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/views/tpcds_retail_model.yml b/converters/cube/tests/fixtures/tpcds_cube/model/views/tpcds_retail_model.yml index b4694965..fbb60ea2 100644 --- a/converters/cube/tests/fixtures/tpcds_cube/model/views/tpcds_retail_model.yml +++ b/converters/cube/tests/fixtures/tpcds_cube/model/views/tpcds_retail_model.yml @@ -15,13 +15,19 @@ # specific language governing permissions and limitations # under the License. -# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then -# kept as a fixture: the converter guide asks every converter to use the TPC-DS -# model as its baseline. Exercises a five-cube star, cross-cube calculated -# measures, a synthesized primary-key dimension, and meta.ossie parking. - views: - name: tpcds_retail_model + cubes: + - join_path: store_sales + includes: '*' + - join_path: store_sales.date_dim + includes: '*' + - join_path: store_sales.customer + includes: '*' + - join_path: store_sales.item + includes: '*' + - join_path: store_sales.store + includes: '*' description: TPC-DS retail semantic model for sales and customer analytics meta: ai_context: Use this semantic model for retail analytics. It provides comprehensive @@ -32,29 +38,18 @@ views: custom_extensions: - vendor_name: SALESFORCE data: | - { + \{ "tableau_workbook_id": "tpcds_retail_dashboard", "einstein_enabled": true, - "crm_sync": { + "crm_sync": \{ "enabled": true, "sync_frequency": "daily", "customer_mapping": "customer.c_customer_id -> Account.AccountNumber" - }, - "tableau_semantics": { + \}, + "tableau_semantics": \{ "published": true, "version": "0.1.1" - } - } + \} + \} - vendor_name: DBT - data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' - cubes: - - join_path: store_sales - includes: '*' - - join_path: store_sales.date_dim - includes: '*' - - join_path: store_sales.customer - includes: '*' - - join_path: store_sales.item - includes: '*' - - join_path: store_sales.store - includes: '*' + data: '\{"project_name": "tpcds_analytics", "models_path": "models/semantic"\}' diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index ebcb59fa..58a92738 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -1511,3 +1511,202 @@ def test_aggregate_spans_only_matches_real_calls(expr, expected): def test_has_top_level_operator_ignores_quoted_text(expr, expected): from ossie_cube.expressions import has_top_level_operator assert has_top_level_operator(expr) is expected + + +# --- review findings: model features that were silently mistranslated ------------ + +def test_a_case_dimension_becomes_a_real_case_expression(): + """A `case` dimension carries conditions instead of `sql`, so there is no column + to name. Emitting the dimension's own name claimed a physical column that does not + exist; Ossie expresses this natively.""" + files = _files(products=( + "cubes:\n - name: products\n sql_table: a.b.products\n dimensions:\n" + " - name: id\n sql: id\n type: number\n" + " primary_key: true\n" + " - name: size\n type: string\n case:\n when:\n" + " - sql: \"{CUBE}.size_value = 'xl'\"\n label: xl\n" + " - sql: \"{CUBE}.size_value = 'xxl'\"\n" + " label: \"it's big\"\n" + " else:\n label: Unknown\n")) + ossie, _ = convert_cube_to_ossie(files) + size = by_name(by_name(model_of(ossie)["datasets"])["products"]["fields"])["size"] + # A string label becomes a SQL literal, with quotes doubled as SQL requires. + assert expr_of(size) == ( + "CASE WHEN size_value = 'xl' THEN 'xl' " + "WHEN size_value = 'xxl' THEN 'it''s big' ELSE 'Unknown' END") + + +def test_a_case_dimension_restores_without_a_redundant_sql(): + """Cube rejects a dimension declaring both `case` and `sql` ("does not match any + of the allowed types"), so the generated sql is dropped when `case` comes back.""" + files = _files(products=( + "cubes:\n - name: products\n sql_table: a.b.products\n dimensions:\n" + " - name: size\n type: string\n case:\n when:\n" + " - sql: \"{CUBE}.v = 'xl'\"\n label: xl\n")) + _, back, _ = _roundtrip(files) + dim = by_name(parse(back["model/cubes/products.yml"])["cubes"][0]["dimensions"]) + assert "sql" not in dim["size"] + assert dim["size"]["case"]["when"][0]["label"] == "xl" + + +def test_a_case_label_may_be_an_expression(): + files = _files(products=( + "cubes:\n - name: products\n sql_table: a.b.products\n dimensions:\n" + " - name: size\n type: string\n case:\n when:\n" + " - sql: \"{CUBE}.v = 'xl'\"\n" + " label:\n sql: \"{CUBE}.english_size\"\n")) + ossie, _ = convert_cube_to_ossie(files) + size = by_name(by_name(model_of(ossie)["datasets"])["products"]["fields"])["size"] + assert expr_of(size) == "CASE WHEN v = 'xl' THEN english_size END" + + +def test_a_sub_query_dimension_is_reported(): + """`sub_query: true` means the sql references a *measure*, which an Ossie field + expression has no form for. It used to convert silently.""" + files = _files(products=( + "cubes:\n - name: products\n sql_table: a.b.products\n dimensions:\n" + " - name: users_count\n sql: \"{users.count}\"\n" + " type: number\n sub_query: true\n" + " - name: users\n sql_table: a.b.users\n dimensions:\n" + " - name: id\n sql: id\n type: number\n" + " primary_key: true\n" + " measures:\n - name: count\n type: count\n")) + _, issues = convert_cube_to_ossie(files) + assert any("sub_query" in i.detail + for i in issues.of_type(IssueType.APPROXIMATED)) + + +def test_duplicate_member_names_in_one_cube_are_rejected(): + """Cube refuses this too ("orders cube: d defined more than once"). Converting it + anyway emitted two Ossie fields of one name -- which the spec's own validator + rejects for a duplicate field name.""" + files = _files(o=( + "cubes:\n - name: o\n sql_table: a.b.t\n dimensions:\n" + " - name: d\n sql: a\n type: string\n" + " - name: d\n sql: b\n type: string\n")) + with pytest.raises(ConversionError, match="defined more than once"): + convert_cube_to_ossie(files) + + +def test_a_dimension_and_a_measure_sharing_a_name_are_rejected_on_import(): + files = _files(o=( + "cubes:\n - name: o\n sql_table: a.b.t\n dimensions:\n" + " - name: revenue\n sql: amount\n type: number\n" + " measures:\n - name: revenue\n sql: amount\n type: sum\n")) + with pytest.raises(ConversionError, match="defined more than once"): + convert_cube_to_ossie(files) + + +def test_an_empty_dimension_sql_is_reported(): + """Cube compiles `sql: ''` without complaint, so it is not refused -- but the Ossie + expression is empty and no consumer can evaluate it.""" + files = _files(o=( + "cubes:\n - name: o\n sql_table: a.b.t\n dimensions:\n" + " - name: d\n sql: ''\n type: string\n")) + _, issues = convert_cube_to_ossie(files) + assert any("empty" in i.detail for i in issues.of_type(IssueType.APPROXIMATED)) + + +def test_a_switch_dimension_keeps_its_type(): + """`switch` maps to String like an ordinary dimension and String maps back to + `string`, so the type has to be recorded or the dimension returns as a plain + string one carrying an orphaned `case` block.""" + files = _files(o=( + "cubes:\n - name: o\n sql_table: a.b.t\n dimensions:\n" + " - name: kind\n sql: kind\n type: switch\n")) + _, back, _ = _roundtrip(files) + dim = by_name(parse(back["model/cubes/o.yml"])["cubes"][0]["dimensions"]) + assert dim["kind"]["type"] == "switch" + # And the recording is not itself emitted as a Cube key. + assert "dim_type" not in dim["kind"] + + +def test_a_computed_primary_key_stays_on_its_own_dimension(): + """A Cube key can be an expression, and then the only name Ossie can carry is the + dimension's. Re-export used to synthesize a dimension reading a column of that + name -- which does not exist -- and move `primary_key: true` onto it, changing + what Cube counts.""" + files = _files(orders=( + "cubes:\n - name: orders\n sql_table: a.b.orders\n dimensions:\n" + " - name: order_key\n" + " sql: \"CONCAT({CUBE}.tenant_id, {CUBE}.id)\"\n" + " type: string\n primary_key: true\n" + " measures:\n - name: count\n type: count\n")) + ossie, back, _ = _roundtrip(files) + dims = parse(back["model/cubes/orders.yml"])["cubes"][0]["dimensions"] + assert len(dims) == 1 + assert dims[0]["name"] == "order_key" + assert dims[0]["primary_key"] is True + assert dims[0]["sql"] == "CONCAT(tenant_id, id)" + # Import records which entries are dimension names rather than columns, because + # the Ossie document alone cannot tell them apart afterwards. + assert stash_of(by_name(model_of(ossie)["datasets"])["orders"])[ + "computed_primary_key"] == ["order_key"] + + +# --- brace escaping ------------------------------------------------------------- +# +# Cube compiles every string in a model as a Python f-string, so an unescaped `{` +# anywhere -- a description, an AI context, a parked JSON blob -- makes the model fail +# to compile. `\{` is Cube's escape for a literal brace. + +def test_a_brace_in_free_text_is_escaped(): + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " description: 'sales in {region}'\n" + " datasets:\n" + " - name: orders\n" + " source: a.b.orders\n" + " description: 'holds {json} notes'\n" + " fields:\n" + " - name: id\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: id\n" + " datatype: Integer\n" + " description: 'the {id}'\n" + ) + files, _ = convert_ossie_to_cube(ossie) + cube = parse(files["model/cubes/orders.yml"])["cubes"][0] + assert cube["description"] == "holds \\{json\\} notes" + assert cube["dimensions"][0]["description"] == "the \\{id\\}" + view = parse(files["model/views/shop.yml"])["views"][0] + assert view["description"] == "sales in \\{region\\}" + # And reading it back returns the original text, not the escaped spelling. + ossie2, _ = convert_cube_to_ossie(files) + model = model_of(ossie2) + assert model["description"] == "sales in {region}" + assert by_name(model["datasets"])["orders"]["description"] == "holds {json} notes" + + +def test_a_parked_foreign_extension_is_escaped_and_restored(): + """The headline multi-vendor case: a foreign vendor's `data` is JSON, so it always + contains braces. Parking it unescaped made every such model fail to compile.""" + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: a.b.orders\n" + " fields:\n" + " - name: id\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: id\n" + " datatype: Integer\n" + " custom_extensions:\n" + " - vendor_name: DBT\n" + " data: '{\"project\": \"x\"}'\n" + ) + files, _ = convert_ossie_to_cube(ossie) + parked = parse(files["model/cubes/orders.yml"])["cubes"][0][ + "meta"]["ossie"]["custom_extensions"] + assert parked[0]["data"] == '\\{"project": "x"\\}' + ossie2, _ = convert_cube_to_ossie(files) + restored = by_name(model_of(ossie2)["datasets"])["orders"]["custom_extensions"] + assert {"vendor_name": "DBT", "data": '{"project": "x"}'} in restored diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index f685b029..34abfcd4 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -544,3 +544,81 @@ def test_synonyms_reach_cube_as_prose_and_are_parked_structurally(): meta = _cubes(files)["orders"]["meta"] assert meta["ai_context"] == "Order facts.\nAlso known as: purchases, sales." assert meta["ossie"]["ai_context"]["synonyms"] == ["purchases", "sales"] + + +# --- review findings: export side ----------------------------------------------- + +@pytest.mark.parametrize("key,path", [ + ("cube_files", "../../outside.yml"), + ("cube_files", "/etc/outside.yml"), + ("view_files", "../escaped.yml"), + ("extra_files", "../../notes.txt"), +]) +def test_a_stashed_path_may_not_escape_the_output_directory(key, path): + """The stash is part of the input document, so a path in it is untrusted. Export + used to join it onto `--output` unchecked, which wrote outside that directory.""" + import json + stash = {"_v": 1, "views": {}} + if key == "view_files": + # The path is only consulted for a view the stash actually carries. + stash["views"] = {"shop": {"name": "shop", + "cubes": [{"join_path": "orders", + "includes": "*"}]}} + stash["view_files"] = {"shop": path} + elif key == "extra_files": + stash["extra_files"] = {path: "x"} + else: + stash["cube_files"] = {"orders": path} + ossie = _ossie(_ORDERS) + ( + " custom_extensions:\n" + " - vendor_name: CUBE\n" + f" data: '{json.dumps(stash)}'\n") + with pytest.raises(ConversionError, match="absolute|escapes the output"): + convert_ossie_to_cube(ossie) + + +def test_a_field_and_a_metric_sharing_a_name_are_rejected(): + """Cube keeps one member namespace per cube ("orders cube: revenue defined more + than once"), so this produced a model Cube refuses to compile.""" + ossie = _ossie(_ORDERS, metrics=_metric("amount", "SUM(orders.amount)")) + with pytest.raises(ConversionError, match="share a name"): + convert_ossie_to_cube(ossie) + + +def test_a_metric_datatype_survives_the_round_trip(): + """Cube has no field for a measure's result type, and import can infer one only + for the count family -- so anything else has to be parked or it is lost.""" + ossie = _ossie(_ORDERS, metrics=( + " metrics:\n - name: total\n datatype: Decimal\n" + " expression:\n dialects:\n - dialect: ANSI_SQL\n" + " expression: SUM(orders.amount)\n")) + files, _ = convert_ossie_to_cube(ossie) + measure = _cubes(files)["orders"]["measures"][0] + assert measure["meta"]["ossie"]["datatype"] == "Decimal" + ossie2, _ = convert_cube_to_ossie(files) + assert model_of(ossie2)["metrics"][0]["datatype"] == "Decimal" + + +def test_a_count_metric_datatype_is_not_parked_because_import_infers_it(): + ossie = _ossie(_ORDERS, metrics=( + " metrics:\n - name: n\n datatype: Integer\n" + " expression:\n dialects:\n - dialect: ANSI_SQL\n" + " expression: COUNT(DISTINCT orders.id)\n")) + files, _ = convert_ossie_to_cube(ossie) + assert "meta" not in _cubes(files)["orders"]["measures"][0] + + +def test_relationship_extensions_are_parked_on_the_declaring_cube(): + """A Cube join entry takes only name/sql/relationship, so a relationship's foreign + extensions have nowhere to go on the join itself. They used to vanish silently.""" + rel = (" relationships:\n - name: r\n from: orders\n to: users\n" + " from_columns: [user_id]\n to_columns: [id]\n" + " custom_extensions:\n - vendor_name: DBT\n data: keep-me\n") + files, issues = convert_ossie_to_cube(_ossie(_TWO_DATASETS, rel)) + parked = _cubes(files)["orders"]["meta"]["ossie"]["join_extensions"] + assert parked["users"] == [{"vendor_name": "DBT", "data": "keep-me"}] + assert issues.of_type(IssueType.PARKED_IN_META) + # And they come back onto the relationship. + ossie2, _ = convert_cube_to_ossie(files) + restored = model_of(ossie2)["relationships"][0]["custom_extensions"] + assert {"vendor_name": "DBT", "data": "keep-me"} in restored diff --git a/converters/cube/tests/test_roundtrip_properties.py b/converters/cube/tests/test_roundtrip_properties.py index 3137245f..145f2aa1 100644 --- a/converters/cube/tests/test_roundtrip_properties.py +++ b/converters/cube/tests/test_roundtrip_properties.py @@ -76,12 +76,21 @@ def text(self): # containing them as templated and preserves it whole, exactly as # Cube's own CubeSchemaConverter does. That behavior has its own # targeted test. + # + # Braces are excluded for a different reason: an *unescaped* brace in a + # Cube string is not valid input at all. Cube compiles every string in a + # model as a Python f-string, so `{` there fails to compile -- the escaped + # `\{` is the only spelling that works, and that is what export emits. A + # generated model with a bare brace is therefore not a Cube model this + # converter should reproduce verbatim; normalizing it to the escaped form + # is the correct outcome, and `test_a_brace_in_free_text_is_escaped` + # pins it. return self.data.draw(st.text( alphabet=st.characters(min_codepoint=32, max_codepoint=126), min_size=1, max_size=24, ).map(str.strip).filter( lambda s: s and not s.startswith("#") - and not any(t in s for t in ("{{", "}}", "{%", "%}")))) + and "{" not in s and "}" not in s)) @settings(max_examples=150, deadline=None, suppress_health_check=[HealthCheck.too_slow]) From e721da5565f7a76642927a785ee334711efd1e9c Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Tue, 4 Aug 2026 15:36:48 +0500 Subject: [PATCH 27/46] Add a Cube-compiles-it gate, spec validation everywhere, and a feature matrix The suite could only check self-consistency: it compared YAML it had produced against YAML it had produced. Three gaps followed from that, and three more defects were sitting in them. Cube itself as a gate (tools/cube_compile.js + tests/_cube_gate.py). Cube compiles every string in a model as a Python f-string, resolves every member reference and enforces one member namespace per cube, so a model can round-trip byte-for-byte and still be one Cube refuses. Needs a built checkout in OSSIE_CUBE_REPO and skips without one, so CI is unaffected. It found, on first run, that exporting a hand-authored Ossie model produced a view Cube rejects: `Included member 'id' conflicts with existing member` -- two datasets both having an `id` is the normal case. Generated views now carry `prefix: true` on a cube whose members would collide, which is Cube's own remedy, and refuse rather than emit a model Cube would reject if a collision survives prefixing. The spec's own validator over every emitted document, not just the two committed fixtures -- including everything Hypothesis generates. Imported in-process from validation/validate.py, so it costs nothing per document. It checks what a field-level assertion structurally cannot: unique names, relationship references, and SQL parseability. A feature matrix in tests/fixtures/features/: one fixture per Cube data-model feature, each a valid Cube model (verified by compiling it), each asked the same four questions. Layout follows Cube's own suite. Writing it found two more: - A `rolling_window` measure without `multi_stage` converted to the bare aggregate, so `revenue_last_3_months` came out as `SUM(sales.amount)` -- the exact expression of the ordinary `revenue` measure beside it. Every windowing key (`rolling_window`, `time_shift`, the legacy multi-stage directives) is now parked with its position, like `multi_stage` already was. - A bare YAML date in an access policy aborted the conversion with a raw TypeError: PyYAML resolves `2022-01-01` to a date, which the JSON stash cannot hold. It normalizes to an ISO string, which is what Cube compares anyway. - A `switch` dimension enumerates `values` and has no `sql`, so an Ossie field -- which requires an expression -- cannot represent it. It used to invent a column and re-export a `sql` Cube rejects alongside `values`; it is parked whole now. 375 tests with both gates on, 362 with neither, 96% coverage. Interop matrix unchanged. --- .github/workflows/converter-cube-ci.yml | 2 + converters/cube/README.md | 48 +++- converters/cube/src/ossie_cube/_common.py | 21 +- converters/cube/src/ossie_cube/cube_to_osi.py | 51 ++++- converters/cube/src/ossie_cube/osi_to_cube.py | 53 ++++- converters/cube/tests/_cube_gate.py | 133 +++++++++++ converters/cube/tests/_roundtrip_helpers.py | 14 ++ .../tests/fixtures/features/access_policy.yml | 58 +++++ .../features/computed_primary_key.yml | 40 ++++ .../features/conditional_dimensions.yml | 61 +++++ .../fixtures/features/dimension_display.yml | 56 +++++ .../features/hierarchies_and_segments.yml | 45 ++++ .../fixtures/features/measure_variants.yml | 69 ++++++ .../fixtures/features/pre_aggregations.yml | 56 +++++ .../fixtures/features/sub_query_dimension.yml | 49 ++++ .../fixtures/features/time_granularities.yml | 42 ++++ .../tests/fixtures/features/view_curation.yml | 68 ++++++ converters/cube/tests/test_feature_matrix.py | 216 ++++++++++++++++++ converters/cube/tests/test_osi_to_cube.py | 5 +- converters/cube/tests/test_roundtrip.py | 40 +++- converters/cube/tools/cube_compile.js | 96 ++++++++ 21 files changed, 1201 insertions(+), 22 deletions(-) create mode 100644 converters/cube/tests/_cube_gate.py create mode 100644 converters/cube/tests/fixtures/features/access_policy.yml create mode 100644 converters/cube/tests/fixtures/features/computed_primary_key.yml create mode 100644 converters/cube/tests/fixtures/features/conditional_dimensions.yml create mode 100644 converters/cube/tests/fixtures/features/dimension_display.yml create mode 100644 converters/cube/tests/fixtures/features/hierarchies_and_segments.yml create mode 100644 converters/cube/tests/fixtures/features/measure_variants.yml create mode 100644 converters/cube/tests/fixtures/features/pre_aggregations.yml create mode 100644 converters/cube/tests/fixtures/features/sub_query_dimension.yml create mode 100644 converters/cube/tests/fixtures/features/time_granularities.yml create mode 100644 converters/cube/tests/fixtures/features/view_curation.yml create mode 100644 converters/cube/tests/test_feature_matrix.py create mode 100644 converters/cube/tools/cube_compile.js diff --git a/.github/workflows/converter-cube-ci.yml b/.github/workflows/converter-cube-ci.yml index a8cafb84..4db6fcc4 100644 --- a/.github/workflows/converter-cube-ci.yml +++ b/.github/workflows/converter-cube-ci.yml @@ -59,5 +59,7 @@ jobs: - name: Unit Tests working-directory: converters/cube + # The Cube-compiles-it gate needs a built Cube checkout (OSSIE_CUBE_REPO) and + # skips without one, so CI runs everything else. See converters/cube/README.md. run: | uv run pytest diff --git a/converters/cube/README.md b/converters/cube/README.md index 2232a2af..ec796a5f 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -410,13 +410,47 @@ uv sync uv run pytest ``` -Example-based unit tests per direction, CLI -behavior tests, fixture round-trip tests (including the -[TPC-DS model](../../examples/tpcds_semantic_model.yaml) the converter guide asks -for as a baseline), core-spec JSON Schema validation of every emitted Ossie -document, and Hypothesis property-based round-trip tests over generated Cube -models -- which fall back to a seeded sweep when `hypothesis` is unavailable, so -the properties still run. +Example-based unit tests per direction, CLI behavior tests, fixture round-trip tests +(including the [TPC-DS model](../../examples/tpcds_semantic_model.yaml) the converter +guide asks for as a baseline), a **feature matrix** of one fixture per Cube data-model +feature, core-spec validation of every emitted Ossie document, and Hypothesis +property-based round-trip tests over generated Cube models -- which fall back to a +seeded sweep when `hypothesis` is unavailable, so the properties still run. + +`tests/fixtures/features/` holds the feature matrix: `case`/`switch` dimensions, +custom granularities, presentation and masking metadata, measure variants +(`rolling_window`, `multi_stage`, `time_shift`, filters, `drill_members`), +hierarchies, segments, pre-aggregations, access policies, view curation, `sub_query` +dimensions and a computed primary key. Each fixture is a *valid Cube model* — verified +by compiling it — and each is asked the same four questions: does it convert, is the +Ossie spec-valid, does `Cube -> Ossie -> Cube` reproduce it, does Cube still compile +the result. Adding a feature means adding a fixture; the four assertions come for +free. The layout follows Cube's own suite, which keeps a fixture per feature. + +### Gates beyond the assertions + +Two checks the YAML assertions cannot replace, both wired into `pytest`: + +**The spec's own validator** runs over every Ossie document the suite produces — +including the ones Hypothesis generates — not just the committed fixtures. It checks +what a field-level assertion structurally cannot: unique names across the document, +relationship references that resolve, and every expression parseable as SQL. It is +imported in-process from `validation/validate.py`, so it costs nothing per document. + +**Cube itself** compiles every fixture and every converted model: + +```bash +OSSIE_CUBE_REPO=~/src/cube uv run pytest # runs the compile gate too +OSSIE_CUBE_REPO=~/src/cube node tools/cube_compile.js model/cubes/*.yml +``` + +This is the only check that can answer "would Cube load this?", and it earns its +keep: Cube compiles every string in a model as a Python f-string, resolves every +member reference, and enforces one member namespace per cube — so a model can +round-trip through Ossie byte-for-byte and still be one Cube refuses. Four defects +were found by asking, including an exported model that failed to compile at all and a +generated view whose `id` members collided. It needs a built Cube checkout and skips +without one, so it gates local and release runs rather than CI. `tools/interop_matrix.py` checks the other half of the job — whether the Ossie this converter emits is any use to the other spokes. It is not part of `pytest`, because diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index 649a0e8c..98f470f0 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -24,6 +24,7 @@ column references Ossie expressions use. """ +import datetime import json import re @@ -241,6 +242,24 @@ def read_stash(obj): return {} +def json_safe(value): + """Coerce YAML scalars JSON cannot hold into strings, recursively. + + The stash is a JSON blob, and PyYAML resolves an unquoted `2022-01-01` to a + `datetime.date` -- which `json.dumps` refuses, so a Cube model with a date in an + access policy used to abort the conversion with a raw TypeError. Dates become ISO + strings, which is what Cube compares against anyway (every value in a policy + filter reaches SQL as text). + """ + if isinstance(value, (datetime.date, datetime.datetime, datetime.time)): + return value.isoformat() + if isinstance(value, list): + return [json_safe(v) for v in value] + if isinstance(value, dict): + return {k: json_safe(v) for k, v in value.items()} + return value + + def write_stash(obj, data): """Attach a CUBE `custom_extensions` entry holding `data` (a dict). @@ -250,7 +269,7 @@ def write_stash(obj, data): if not data: return payload = {"_v": STASH_VERSION} - payload.update(data) + payload.update(json_safe(data)) blob = json.dumps(payload) exts = obj.setdefault("custom_extensions", []) for ext in exts: diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 804607e3..8498d727 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -540,11 +540,28 @@ def _convert_cube(cname, cube, plain, extra_joins, extra_measures, issues): ds["unique_keys"] = [list(k) for k in parked["unique_keys"]] fields = [] - for dim in _as_named_list(cube.get("dimensions"), f"{scope} dimensions"): + extra_dimensions = [] + for index, dim in enumerate( + _as_named_list(cube.get("dimensions"), f"{scope} dimensions")): dname = require_str(dim, "name", f"{scope}: dimension") + if snake(dim.get("type") or "") == "switch": + # A `switch` dimension enumerates `values` and has no `sql` at all -- it + # exists so `case` measures can pivot on it. An Ossie field *requires* an + # expression, and there is no column to name, so emitting one would invent + # a column (and re-export would give Cube a `sql` it rejects alongside + # `values`). It rides on the stash with its position instead, the same + # protocol multi-stage measures and unconvertible joins use. + issues.add(IssueType.PARKED_IN_META, f"{cname}.{dname}", + "switch dimension enumerates values rather than reading a " + "column, and an Ossie field requires an expression; preserved " + "in custom_extensions only") + extra_dimensions.append({"index": index, "dimension": dim}) + continue fields.extend(_convert_dimension(cname, dname, dim, plain, issues)) if fields: ds["fields"] = fields + if extra_dimensions: + stash["extra_dimensions"] = extra_dimensions primary_key = _primary_key_of(cube, cname) if primary_key: ds["primary_key"] = primary_key @@ -973,13 +990,18 @@ def expression(self, cname, mname, stack=()): if not mtype: raise ConversionError(f"measure '{scope}': missing required 'type'") - if measure.get("multi_stage"): - # group_by / reduce_by / time_shift / rank render as window functions - # over a grain other than the query's; Ossie has no form for that. + windowed = _windowing_key(measure) + if windowed: + # These all compute over a grain other than the query's -- a trailing + # range, a shifted period, an inner GROUP BY -- which renders as a window + # function. Ossie has no form for that, and emitting the bare aggregate + # would claim something else entirely: a `rolling_window` sum would read as + # a plain SUM, identical to an ordinary sum measure over the same column. self._issues.add( IssueType.MULTI_STAGE_MEASURE_PARKED, scope, - f"multi_stage measure (type '{mtype}'); preserved in " - f"custom_extensions only") + f"'{windowed}' measure (type '{mtype}') is computed over a grain other " + f"than the query's, which an Ossie expression has no form for; " + f"preserved in custom_extensions only") return self._remember(key, None) sql = measure.get("sql") filter_exprs = [ @@ -1077,6 +1099,23 @@ def _operand(self, cname, sql, stack): return translated +# Measure keys that make the value depend on a grain other than the query's. Cube +# renders each as a window function, so none has a static Ossie expression. +_WINDOWING_KEYS = ( + "multi_stage", "rolling_window", "time_shift", + # The legacy spelling of the multi-stage directives. + "group_by", "reduce_by", "add_group_by", +) + + +def _windowing_key(measure): + """The first windowing key present on a measure, or None.""" + for key in _WINDOWING_KEYS: + if measure.get(key): + return key + return None + + def _is_generated_part(measure): """True for a `public: false` measure a previous export created to hold one aggregate of a composite metric (marked `meta.ossie.part_of`).""" diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 7590b0ee..c81bc5c2 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -167,6 +167,7 @@ def _convert_model(model, dialect, base_cube, issues): # stashed original path, in which case they go back into the same file. stashed_paths = model_stash.get("cube_files") or {} files_content = {} + emitted_members = {} for ds_name, ds in datasets.items(): cname = cube_names[ds_name] cube = _build_cube(ds, cname, dim_names_by_cube[cname], @@ -177,9 +178,16 @@ def _convert_model(model, dialect, base_cube, issues): path = (safe_relative_path(stashed, f"cube '{cname}'") if stashed else cube_file(cname)) files_content.setdefault(path, {}).setdefault("cubes", []).append(cube) + # The members the cube really carries -- including a synthesized primary key, a + # merged geo dimension and measures restored from the stash -- which is what a + # generated view has to disambiguate against. + emitted_members[cname] = [ + m["name"] for key in ("dimensions", "measures", "segments") + for m in (cube.get(key) or []) if isinstance(m, dict) and m.get("name")] for vpath, views in _build_views(model, model_stash, cube_names, relationships, - datasets, base_cube).items(): + datasets, base_cube, + emitted_members).items(): files_content.setdefault(vpath, {}).setdefault("views", []).extend(views) files = {path: dump_yaml(content) for path, content in files_content.items()} @@ -337,6 +345,12 @@ def _build_cube(ds, cname, dim_names, inline_sql, ref_members, joins, measures, if dim["name"] in pk_names: dim["primary_key"] = True + # Dimensions a prior import could not express as an Ossie field (a `switch` one, + # which has no sql) go back at their original positions. + for item in sorted(stash.get("extra_dimensions") or [], + key=lambda x: x.get("index", 0)): + dimensions.insert(min(item.get("index", 0), len(dimensions)), + item["dimension"]) if dimensions: cube["dimensions"] = [_ordered(d, _DIM_KEY_ORDER) for d in dimensions] @@ -922,7 +936,7 @@ def _balanced(s): # --- views ---------------------------------------------------------------------- def _build_views(model, model_stash, cube_names, relationships, datasets, - base_cube): + base_cube, emitted_members): """Return {file path: [view dict, ...]}. A list per path, not a single view: several views can share one YAML file, and @@ -984,21 +998,33 @@ def _build_views(model, model_stash, cube_names, relationships, datasets, view["cubes"] = _view_cubes( cube_names, relationships, cube_names[_pick_base_cube(model.get("name", ""), datasets, - relationships, base_cube)]) + relationships, base_cube)], + emitted_members) out[view_file(vname)] = [view] return out -def _view_cubes(cube_names, relationships, base): +def _view_cubes(cube_names, relationships, base, emitted_members): """Build a generated view's `cubes:` list: the base cube plus every cube - reachable from it, each addressed by its full `join_path`.""" + reachable from it, each addressed by its full `join_path`. + + A view flattens every included member into one namespace, and Cube refuses one + where two members collide ("Included member 'id' conflicts with existing member"). + Two datasets both having an `id` is the normal case, not a corner one, so a cube + whose members would collide gets `prefix: true` -- Cube's own remedy, which renames + its members to `_` within the view only. + """ adjacency = {} for rel in relationships: a, b = cube_names[rel["from"]], cube_names[rel["to"]] adjacency.setdefault(a, []).append(b) adjacency.setdefault(b, []).append(a) + def members(cname): + return emitted_members.get(cname) or [] + entries = [{"join_path": base, "includes": "*"}] + claimed = {m.lower() for m in members(base)} paths = {base: base} queue = deque([base]) while queue: @@ -1007,7 +1033,22 @@ def _view_cubes(cube_names, relationships, base): if neighbor in paths: continue paths[neighbor] = f"{paths[current]}.{neighbor}" - entries.append({"join_path": paths[neighbor], "includes": "*"}) + own = members(neighbor) + entry = {"join_path": paths[neighbor], "includes": "*"} + if any(m.lower() in claimed for m in own): + entry["prefix"] = True + names = [f"{neighbor}_{m}" for m in own] + else: + names = list(own) + still_colliding = sorted(n for n in names if n.lower() in claimed) + if still_colliding: + raise ConversionError( + f"generated view: member(s) {', '.join(still_colliding)} from " + f"dataset '{neighbor}' collide with another dataset's even with a " + f"prefix; Cube views keep one namespace, so rename one in the " + f"Ossie model.") + claimed.update(n.lower() for n in names) + entries.append(entry) queue.append(neighbor) # A cube no relationship reaches cannot be addressed by a join path, so it is # simply not part of the generated view; it is still exported and joinable. diff --git a/converters/cube/tests/_cube_gate.py b/converters/cube/tests/_cube_gate.py new file mode 100644 index 00000000..5ed95b03 --- /dev/null +++ b/converters/cube/tests/_cube_gate.py @@ -0,0 +1,133 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Two gates the YAML assertions cannot replace. + +`assert_ossie_is_valid` runs the repo's own `validation/validate.py` over an emitted +Ossie document -- structure, unique names, relationship references, SQL parseability. +The converter's own tests assert field by field, which cannot notice a document that +is *shaped* wrong; a Cube cube with two dimensions of one name used to produce two +Ossie fields of one name, which this catches. + +`assert_cube_compiles` asks Cube itself whether an emitted model loads. Cube compiles +every string in a model as a Python f-string, resolves every member reference and +enforces one member namespace per cube, so a model can round-trip through Ossie +byte-for-byte and still be one Cube refuses. It needs a built Cube checkout, named by +`OSSIE_CUBE_REPO`, and skips when there is none -- so it gates local and release-time +runs rather than CI. +""" + +import json +import os +import pathlib +import shutil +import subprocess +import tempfile + +import pytest +import yaml + +_HERE = pathlib.Path(__file__).resolve().parent +_CONVERTER = _HERE.parent +_TOOL = _CONVERTER / "tools" / "cube_compile.js" + +# converters/cube -> converters -> repo root +_REPO_ROOT = _CONVERTER.parent.parent +_VALIDATOR = _REPO_ROOT / "validation" / "validate.py" + + +def _have(program): + return shutil.which(program) is not None + + +cube_gate = pytest.mark.skipif( + not (os.environ.get("OSSIE_CUBE_REPO") and _have("node")), + reason="needs a built Cube checkout in OSSIE_CUBE_REPO, and node", +) + +def assert_cube_compiles(files, label=""): + """Fail unless Cube itself compiles `files` ({relative name: YAML text}).""" + with tempfile.TemporaryDirectory(prefix="ossie-cube-compile-") as tmp: + paths = [] + for name, text in files.items(): + if not name.lower().endswith((".yml", ".yaml")): + # A `.js`/`.ts` model needs Cube's transpiler and a `.py` one is + # Jinja-driven; the converter preserves both without parsing them. + continue + dest = pathlib.Path(tmp) / pathlib.Path(name).name + dest.write_text(text) + paths.append(str(dest)) + assert paths, f"{label}: nothing to compile" + result = subprocess.run( + ["node", str(_TOOL), *paths], capture_output=True, text=True) + if result.returncode == 2: + pytest.skip(result.stdout.strip() or "cube_compile.js unavailable") + assert result.returncode == 0, ( + f"Cube refused the model {label}:\n{result.stdout}{result.stderr}") + + +def _load_validator(): + """Import `validation/validate.py` as a module. + + It is a standalone script, but its checks are plain functions over a parsed + document, so they can be called directly. In-process matters: a subprocess per + document is a second each, which rules out validating everything the property tests + generate -- and validating only the committed fixtures is how a badly *shaped* + document got through in the first place. + """ + import importlib.util + + spec = importlib.util.spec_from_file_location("ossie_validate", _VALIDATOR) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_VALIDATOR_MODULE = None +_VALIDATOR_ERROR = None +try: + if _VALIDATOR.exists(): + _VALIDATOR_MODULE = _load_validator() + _SCHEMA = json.loads( + (_REPO_ROOT / "core-spec" / "osi-schema.json").read_text()) +except Exception as exc: # missing jsonschema, a moved schema, a changed script + _VALIDATOR_ERROR = f"{type(exc).__name__}: {exc}" + + +validator_gate = pytest.mark.skipif( + _VALIDATOR_MODULE is None, + reason=f"validation/validate.py unavailable ({_VALIDATOR_ERROR})", +) + + +def assert_ossie_is_valid(ossie_yaml, label=""): + """Fail unless the repo's own validator accepts this Ossie document. + + Runs every check `validate.py` runs: JSON Schema, unique names, relationship + references, and SQL parseability of every expression. + """ + if _VALIDATOR_MODULE is None: + pytest.skip(f"validation/validate.py unavailable ({_VALIDATOR_ERROR})") + v = _VALIDATOR_MODULE + data = yaml.safe_load(ossie_yaml) + errors = (v.validate_schema(data, _SCHEMA) + + v.validate_unique_names(data) + + v.validate_references(data) + + v.validate_sql(data)) + assert not errors, ( + f"validation/validate.py rejected the Ossie for {label}:\n " + + "\n ".join(str(e) for e in errors)) diff --git a/converters/cube/tests/_roundtrip_helpers.py b/converters/cube/tests/_roundtrip_helpers.py index 2343f683..cb28e630 100644 --- a/converters/cube/tests/_roundtrip_helpers.py +++ b/converters/cube/tests/_roundtrip_helpers.py @@ -208,6 +208,20 @@ def assert_ossie_roundtrip_is_lossless(files): "Ossie -> Cube -> Ossie changed the model") +def assert_ossie_is_spec_valid(files): + """The Ossie a Cube model converts to satisfies the spec's own validator. + + Structural, so a field-level assertion cannot replace it: a Cube cube with two + dimensions of one name used to produce two Ossie fields of one name, which every + per-field assertion happily passed. + """ + from _cube_gate import assert_ossie_is_valid + + ossie, _ = convert_cube_to_ossie(files) + assert_ossie_is_valid(ossie, "generated model") + + def check_model(files): assert_cube_roundtrip_is_lossless(files) assert_ossie_roundtrip_is_lossless(files) + assert_ossie_is_spec_valid(files) diff --git a/converters/cube/tests/fixtures/features/access_policy.yml b/converters/cube/tests/fixtures/features/access_policy.yml new file mode 100644 index 00000000..97098708 --- /dev/null +++ b/converters/cube/tests/fixtures/features/access_policy.yml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# An access policy carries `securityContext` templates and bare YAML dates. Neither may +# be disturbed: the templates are Cube's own interpolation, so escaping them would break +# the policy. The date is quoted here so the fixture round-trips byte-for-byte; the +# *unquoted* form is normalized to a string (see +# test_a_bare_yaml_date_is_normalized_rather_than_crashing) because PyYAML resolves it +# to a date object, which the JSON stash cannot hold. +cubes: + - name: orders + sql_table: shop.public.orders + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: status + sql: status + type: string + - name: created_at + sql: created_at + type: time + measures: + - name: count + type: count + access_policy: + - role: viewer + row_level: + filters: + - member: status + operator: equals + values: + - completed + - member: created_at + operator: notInDateRange + values: + - '2022-01-01' + - "{ securityContext.currentDate }" + - role: '*' + member_level: + includes: '*' + excludes: + - status diff --git a/converters/cube/tests/fixtures/features/computed_primary_key.yml b/converters/cube/tests/fixtures/features/computed_primary_key.yml new file mode 100644 index 00000000..a9a8d395 --- /dev/null +++ b/converters/cube/tests/fixtures/features/computed_primary_key.yml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# A Cube primary key can be an expression. Ossie's `primary_key` names columns, so the +# only name there is to carry is the dimension's -- which export must put the key back +# on rather than synthesizing a dimension reading a column of that name. +cubes: + - name: order_lines + sql_table: shop.public.order_lines + dimensions: + - name: line_key + sql: "CONCAT({CUBE}.tenant_id, '-', {CUBE}.line_no)" + type: string + primary_key: true + - name: tenant_id + sql: tenant_id + type: number + - name: line_no + sql: line_no + type: number + measures: + - name: count + type: count + - name: quantity + sql: quantity + type: sum diff --git a/converters/cube/tests/fixtures/features/conditional_dimensions.yml b/converters/cube/tests/fixtures/features/conditional_dimensions.yml new file mode 100644 index 00000000..c7c4e10a --- /dev/null +++ b/converters/cube/tests/fixtures/features/conditional_dimensions.yml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# `switch` and `case` dimensions: the two kinds that carry no `sql` at all. Both used +# to convert to an Ossie expression naming a column that does not exist. +cubes: + - name: products + sql_table: shop.public.products + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: size_value + sql: size_value + type: string + - name: english_size + sql: english_size + type: string + # Conditions instead of sql; maps to a real Ossie CASE expression. + - name: size + type: string + case: + when: + - sql: "{CUBE}.size_value = 'xl-en'" + label: xl + - sql: "{CUBE}.size_value = 'xxl'" + label: "it's xxl" + else: + label: Unknown + # A dynamic label is an expression rather than a literal. + - name: localized_size + type: string + case: + when: + - sql: "{CUBE}.size_value = 'xl'" + label: + sql: "{CUBE}.english_size" + # Enumerated values, no sql: has no Ossie field form, so it is parked whole. + - name: currency + type: switch + values: + - USD + - EUR + measures: + - name: count + type: count diff --git a/converters/cube/tests/fixtures/features/dimension_display.yml b/converters/cube/tests/fixtures/features/dimension_display.yml new file mode 100644 index 00000000..d101b109 --- /dev/null +++ b/converters/cube/tests/fixtures/features/dimension_display.yml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Presentation and masking metadata Ossie has no field for. All of it is Cube-specific +# and rides in the stash, so what this fixture pins is that none of it is lost. +cubes: + - name: orders + sql_table: shop.public.orders + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: amount + sql: amount + type: number + format: currency + currency: USD + meta: + note: shown in the order list + - name: status + sql: status + type: string + title: Order Status + description: Current fulfilment state + order: asc + - name: avatar + sql: avatar_url + type: string + format: imageUrl + - name: secret_code + sql: secret_code + type: string + mask: + sql: "'****'" + - name: internal_note + sql: internal_note + type: string + public: false + measures: + - name: count + type: count diff --git a/converters/cube/tests/fixtures/features/hierarchies_and_segments.yml b/converters/cube/tests/fixtures/features/hierarchies_and_segments.yml new file mode 100644 index 00000000..0b9c73e4 --- /dev/null +++ b/converters/cube/tests/fixtures/features/hierarchies_and_segments.yml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Hierarchies and segments: both Cube-only, both stashed and restored verbatim. +cubes: + - name: users + sql_table: shop.public.users + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: country + sql: country + type: string + - name: city + sql: city + type: string + hierarchies: + - name: geography + title: Where they are + levels: + - country + - city + segments: + - name: active + sql: "{CUBE}.status = 'active'" + description: Not deleted + measures: + - name: count + type: count diff --git a/converters/cube/tests/fixtures/features/measure_variants.yml b/converters/cube/tests/fixtures/features/measure_variants.yml new file mode 100644 index 00000000..24ea2aba --- /dev/null +++ b/converters/cube/tests/fixtures/features/measure_variants.yml @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Measure shapes beyond a plain aggregate. `rolling_window` and `multi_stage` have no +# static Ossie expression at all, so they ride on the dataset stash with their original +# positions and come back interleaved with the measures rebuilt from metrics. +cubes: + - name: sales + sql_table: shop.public.sales + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: region + sql: region + type: string + - name: sold_at + sql: sold_at + type: time + measures: + - name: count + type: count + - name: revenue + sql: amount + type: sum + format: currency + currency: USD + drill_members: + - id + - region + - name: completed_revenue + sql: amount + type: sum + filters: + - sql: "{CUBE}.status = 'completed'" + - name: revenue_last_3_months + sql: amount + type: sum + rolling_window: + trailing: 3 month + - name: revenue_by_region + sql: "{revenue}" + type: number + multi_stage: true + group_by: + - region + - name: revenue_prior_year + sql: amount + type: sum + multi_stage: true + time_shift: + - time_dimension: sold_at + interval: 1 year + type: prior diff --git a/converters/cube/tests/fixtures/features/pre_aggregations.yml b/converters/cube/tests/fixtures/features/pre_aggregations.yml new file mode 100644 index 00000000..7e2a9361 --- /dev/null +++ b/converters/cube/tests/fixtures/features/pre_aggregations.yml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Pre-aggregations are a Cube-only performance construct with no Ossie counterpart -- +# legitimately so, which is why they are stashed rather than approximated. +cubes: + - name: orders + sql_table: shop.public.orders + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: status + sql: status + type: string + - name: created_at + sql: created_at + type: time + measures: + - name: count + type: count + - name: revenue + sql: amount + type: sum + pre_aggregations: + - name: main + measures: + - count + - revenue + dimensions: + - status + time_dimension: created_at + granularity: day + refresh_key: + every: 1 hour + - name: rollup_only + type: rollup + measures: + - revenue + dimensions: + - status diff --git a/converters/cube/tests/fixtures/features/sub_query_dimension.yml b/converters/cube/tests/fixtures/features/sub_query_dimension.yml new file mode 100644 index 00000000..58a6314f --- /dev/null +++ b/converters/cube/tests/fixtures/features/sub_query_dimension.yml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# A `sub_query` dimension reads a *measure* of another cube, which Cube resolves by +# aggregating in a subquery. An Ossie field expression is dataset-scoped SQL over +# columns, so the reference survives as text and is reported. +cubes: + - name: products + sql_table: shop.public.products + joins: + - name: orders + sql: "{CUBE}.id = {orders}.product_id" + relationship: one_to_many + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: order_count + sql: "{orders.count}" + type: number + sub_query: true + - name: orders + sql_table: shop.public.orders + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: product_id + sql: product_id + type: number + measures: + - name: count + type: count diff --git a/converters/cube/tests/fixtures/features/time_granularities.yml b/converters/cube/tests/fixtures/features/time_granularities.yml new file mode 100644 index 00000000..c6e49fe9 --- /dev/null +++ b/converters/cube/tests/fixtures/features/time_granularities.yml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Time dimensions: custom granularities, and the `type: time` <-> `is_time` mapping. +cubes: + - name: events + sql_table: shop.public.events + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: occurred_at + sql: occurred_at + type: time + granularities: + - name: fiscal_year + interval: 1 year + offset: 3 months + - name: sunday_week + interval: 1 week + origin: '2024-01-07' + - name: recorded_on + sql: recorded_on + type: time + measures: + - name: count + type: count diff --git a/converters/cube/tests/fixtures/features/view_curation.yml b/converters/cube/tests/fixtures/features/view_curation.yml new file mode 100644 index 00000000..cee35f99 --- /dev/null +++ b/converters/cube/tests/fixtures/features/view_curation.yml @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# A view is where the Ossie model boundary lives, and its curation -- prefixes, +# aliases, includes/excludes, folders -- has no Ossie form. It is stashed verbatim. +cubes: + - name: orders + sql_table: shop.public.orders + joins: + - name: users + sql: "{CUBE}.user_id = {users}.id" + relationship: many_to_one + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: user_id + sql: user_id + type: number + - name: status + sql: status + type: string + measures: + - name: count + type: count + - name: users + sql_table: shop.public.users + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: city + sql: city + type: string +views: + - name: sales + description: Curated sales view + meta: + ai_context: Prefer this view for sales questions. + cubes: + - join_path: orders + includes: + - status + - count + - join_path: orders.users + prefix: true + includes: + - city + folders: + - name: Attributes + includes: + - status diff --git a/converters/cube/tests/test_feature_matrix.py b/converters/cube/tests/test_feature_matrix.py new file mode 100644 index 00000000..01e42b2e --- /dev/null +++ b/converters/cube/tests/test_feature_matrix.py @@ -0,0 +1,216 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""One fixture per Cube data-model feature, each asserted four ways. + +The two whole-model fixtures cover the common shapes well but say nothing about the +long tail of the data model, which is where the silent defects were: a `case` dimension +converted to an expression naming a column that did not exist, a `switch` dimension +came back as a plain string one, a computed primary key moved onto a synthesized +dimension reading a nonexistent column, and a bare YAML date in an access policy +aborted the conversion. None of those were visible to a field-level assertion. + +Every fixture here is a *valid Cube model* -- verified by compiling it -- and each is +put through the same four questions: + +1. does it convert at all, and what does the converter say it could not carry; +2. is the Ossie it produces valid per the spec's own validator; +3. does `Cube -> Ossie -> Cube` reproduce it structurally; +4. does Cube itself still compile the result. + +Layout follows Cube's own test suite, which keeps a fixture per feature +(`hierarchies.yml`, `switch-dimension.yml`, `folders.yml`, `calendar_orders.yml`). +Adding a feature means adding a fixture; the four assertions come for free. +""" + +import pathlib + +import pytest +from _cube_gate import ( + assert_cube_compiles, + assert_ossie_is_valid, + cube_gate, + validator_gate, +) +from _util import by_name, expr_of, model_of, parse_files, stash_of + +from ossie_cube import IssueType, convert_cube_to_ossie, convert_ossie_to_cube + +_FEATURES = pathlib.Path(__file__).resolve().parent / "fixtures" / "features" +_FIXTURES = sorted(p.name for p in _FEATURES.glob("*.yml")) + + +def _load(name): + """One fixture, keyed the way a Cube model directory would key it.""" + return {f"model/cubes/{name}": (_FEATURES / name).read_text()} + + +def _roundtrip(name): + files = _load(name) + ossie, issues = convert_cube_to_ossie(files) + back, _ = convert_ossie_to_cube(ossie) + return files, ossie, back, issues + + +# --- the four questions, asked of every fixture ---------------------------------- + +@pytest.mark.parametrize("name", _FIXTURES) +def test_every_feature_converts(name): + _, ossie, _, _ = _roundtrip(name) + assert model_of(ossie)["datasets"] + + +@pytest.mark.parametrize("name", _FIXTURES) +def test_every_feature_roundtrips_structurally(name): + files, _, back, _ = _roundtrip(name) + assert parse_files(back) == parse_files(files) + + +@validator_gate +@pytest.mark.parametrize("name", _FIXTURES) +def test_every_feature_produces_valid_ossie(name): + _, ossie, _, _ = _roundtrip(name) + assert_ossie_is_valid(ossie, name) + + +@cube_gate +@pytest.mark.parametrize("name", _FIXTURES) +def test_every_feature_still_compiles_in_cube(name): + """The fixture compiles by construction; what this asks is whether the *converted* + model still does. A round trip can reproduce a model structurally and still emit + something Cube refuses -- a `sql` alongside `case`, an unescaped brace, two members + of one name.""" + files, _, back, _ = _roundtrip(name) + assert_cube_compiles(files, f"{name} (as committed)") + assert_cube_compiles(back, f"{name} (after a round trip)") + + +# --- what each feature is expected to do ----------------------------------------- + +def test_a_case_dimension_becomes_an_ossie_case_expression(): + _, ossie, _, _ = _roundtrip("conditional_dimensions.yml") + fields = by_name(by_name(model_of(ossie)["datasets"])["products"]["fields"]) + assert expr_of(fields["size"]) == ( + "CASE WHEN size_value = 'xl-en' THEN 'xl' " + "WHEN size_value = 'xxl' THEN 'it''s xxl' ELSE 'Unknown' END") + # A dynamic label is an expression, not a literal. + assert expr_of(fields["localized_size"]) == ( + "CASE WHEN size_value = 'xl' THEN english_size END") + + +def test_a_switch_dimension_has_no_ossie_field(): + _, ossie, _, issues = _roundtrip("conditional_dimensions.yml") + dataset = by_name(model_of(ossie)["datasets"])["products"] + assert "currency" not in by_name(dataset["fields"]) + # It rides on the stash with its position instead, and that is reported. + parked = stash_of(dataset)["extra_dimensions"] + assert [p["dimension"]["name"] for p in parked] == ["currency"] + assert any("switch" in i.detail + for i in issues.of_type(IssueType.PARKED_IN_META)) + + +def test_a_sub_query_dimension_is_reported_not_silently_converted(): + _, _, _, issues = _roundtrip("sub_query_dimension.yml") + assert any("sub_query" in i.detail + for i in issues.of_type(IssueType.APPROXIMATED)) + + +def test_a_computed_primary_key_returns_to_its_own_dimension(): + files, ossie, back, _ = _roundtrip("computed_primary_key.yml") + dataset = by_name(model_of(ossie)["datasets"])["order_lines"] + assert dataset["primary_key"] == ["line_key"] + # Recorded, because the Ossie document alone cannot tell a dimension name from a + # column name afterwards. + assert stash_of(dataset)["computed_primary_key"] == ["line_key"] + cube = parse_files(back)["model/cubes/computed_primary_key.yml"]["cubes"][0] + keys = [d for d in cube["dimensions"] if d.get("primary_key")] + assert [d["name"] for d in keys] == ["line_key"] + assert keys[0]["sql"].startswith("CONCAT(") + + +def test_a_multi_stage_measure_is_parked_with_its_position(): + _, ossie, _, issues = _roundtrip("measure_variants.yml") + dataset = by_name(model_of(ossie)["datasets"])["sales"] + parked = [p["measure"]["name"] for p in stash_of(dataset)["extra_measures"]] + # A rolling window, an inner GROUP BY and a time shift all compute over a grain + # other than the query's. Emitting the bare aggregate would have been worse than + # dropping it: `revenue_last_3_months` came out as `SUM(sales.amount)` -- the exact + # expression of the ordinary `revenue` measure beside it. + assert set(parked) == {"revenue_last_3_months", "revenue_by_region", + "revenue_prior_year"} + assert issues.of_type(IssueType.MULTI_STAGE_MEASURE_PARKED) + assert "revenue_last_3_months" not in by_name(model_of(ossie)["metrics"]) + + +def test_a_filtered_measure_folds_its_filter_into_the_expression(): + _, ossie, _, _ = _roundtrip("measure_variants.yml") + metric = by_name(model_of(ossie)["metrics"])["completed_revenue"] + assert expr_of(metric) == ( + "SUM(CASE WHEN (sales.status = 'completed') THEN sales.amount END)") + + +def test_an_access_policy_keeps_its_security_context_and_dates(): + """Two things must not be touched: a `securityContext` reference is Cube's own + interpolation, and a bare YAML date is not JSON-serializable -- it used to abort + the conversion with a raw TypeError.""" + files, ossie, back, _ = _roundtrip("access_policy.yml") + policy = stash_of(by_name(model_of(ossie)["datasets"])["orders"])[ + "cube_extras"]["access_policy"] + values = policy[0]["row_level"]["filters"][1]["values"] + assert "{ securityContext.currentDate }" in values + assert "2022-01-01" in values + + +def test_a_bare_yaml_date_is_normalized_rather_than_crashing(): + """PyYAML resolves an unquoted `2022-01-01` to a `datetime.date`, which the JSON + stash cannot hold -- it used to abort the conversion with a raw TypeError. It + becomes an ISO string, which is what Cube compares against anyway: every value in a + policy filter reaches SQL as text.""" + files = _load("access_policy.yml") + bare = {k: v.replace("- '2022-01-01'", "- 2022-01-01") + for k, v in files.items()} + ossie, _ = convert_cube_to_ossie(bare) + policy = stash_of(by_name(model_of(ossie)["datasets"])["orders"])[ + "cube_extras"]["access_policy"] + assert policy[0]["row_level"]["filters"][1]["values"][0] == "2022-01-01" + + +def test_view_curation_survives_untouched(): + _, ossie, back, _ = _roundtrip("view_curation.yml") + model = model_of(ossie) + # The view supplies the model's identity, and its curation has no Ossie form. + assert model["name"] == "sales" + view = stash_of(model)["views"]["sales"] + assert view["folders"][0]["name"] == "Attributes" + assert any(entry.get("prefix") for entry in view["cubes"]) + + +@pytest.mark.parametrize("name,keys", [ + ("dimension_display.yml", ["format", "currency", "order", "mask", "public"]), + ("time_granularities.yml", ["granularities"]), + ("hierarchies_and_segments.yml", ["hierarchies", "segments"]), + ("pre_aggregations.yml", ["pre_aggregations"]), +]) +def test_cube_only_keys_are_stashed_rather_than_dropped(name, keys): + """Everything here is legitimately Cube-specific -- presentation, physical + layout, access control -- so the right behaviour is to carry it in the stash and + leave the Ossie document clean, not to approximate it.""" + _, ossie, back, _ = _roundtrip(name) + emitted = parse_files(back)[f"model/cubes/{name}"]["cubes"][0] + flat = str(emitted) + for key in keys: + assert key in flat, f"{key} did not survive the round trip" diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 34abfcd4..782d726f 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -503,7 +503,10 @@ def test_generated_view_is_rooted_at_the_fk_sink(): view = parse(files["model/views/shop.yml"])["views"][0] assert view["cubes"] == [ {"join_path": "orders", "includes": "*"}, - {"join_path": "orders.users", "includes": "*"}, + # `prefix: true` because both cubes have an `id`: a view flattens every + # included member into one namespace and Cube refuses a collision, so this is + # Cube's own remedy rather than a stylistic choice. + {"join_path": "orders.users", "includes": "*", "prefix": True}, ] diff --git a/converters/cube/tests/test_roundtrip.py b/converters/cube/tests/test_roundtrip.py index 8416088c..b309cea5 100644 --- a/converters/cube/tests/test_roundtrip.py +++ b/converters/cube/tests/test_roundtrip.py @@ -26,6 +26,12 @@ import json import pytest +from _cube_gate import ( + assert_cube_compiles, + assert_ossie_is_valid, + cube_gate, + validator_gate, +) from _util import (REPO_ROOT, canon, load_fixture, load_fixture_dir, parse, parse_files) @@ -83,6 +89,37 @@ def test_imported_ossie_validates_against_core_spec_schema(fixture): jsonschema.validate(parse(ossie), schema) +@validator_gate +@pytest.mark.parametrize("fixture", FIXTURES) +def test_imported_ossie_passes_the_repo_validator(fixture): + """More than the schema: unique names across the document, relationship references + that resolve, and every expression parseable as SQL.""" + ossie, _ = convert_cube_to_ossie(load_fixture_dir(fixture)) + assert_ossie_is_valid(ossie, fixture) + + +@cube_gate +@pytest.mark.parametrize("fixture", FIXTURES) +def test_the_fixture_and_its_round_trip_both_compile_in_cube(fixture): + """The question a YAML comparison cannot ask. Both directions are checked, because + the committed fixture being valid Cube is itself an assertion worth holding: the + tpcds one was not, and nothing noticed until Cube was asked.""" + files = load_fixture_dir(fixture) + assert_cube_compiles(files, f"{fixture} (as committed)") + ossie, _ = convert_cube_to_ossie(files) + back, _ = convert_ossie_to_cube(ossie) + assert_cube_compiles(back, f"{fixture} (after a round trip)") + + +@cube_gate +def test_a_hand_authored_ossie_model_exports_to_a_model_cube_accepts(): + """Nothing here came from Cube, so nothing is restored from a stash -- every key is + one the exporter chose. That makes it the case most likely to produce something Cube + rejects.""" + files, _ = convert_ossie_to_cube(load_fixture("hand_authored_ossie.yaml")) + assert_cube_compiles(files, "hand_authored_ossie.yaml") + + @pytest.mark.parametrize("fixture", FIXTURES) def test_ossie_roundtrip_is_lossless(fixture): """Ossie -> Cube -> Ossie reproduces the model too. @@ -113,7 +150,8 @@ def test_hand_authored_ossie_gets_a_generated_view(): # Rooted at the FK sink, with the joined cube addressed by its join path. assert view["cubes"] == [ {"join_path": "orders", "includes": "*"}, - {"join_path": "orders.customers", "includes": "*"}, + # Both cubes carry an `id`, which a view cannot include twice. + {"join_path": "orders.customers", "includes": "*", "prefix": True}, ] diff --git a/converters/cube/tools/cube_compile.js b/converters/cube/tools/cube_compile.js new file mode 100644 index 00000000..3e97920b --- /dev/null +++ b/converters/cube/tools/cube_compile.js @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Does Cube accept this model? -- the one question a YAML round trip cannot answer. + * + * OSSIE_CUBE_REPO=~/src/cube node tools/cube_compile.js model/cubes/*.yml + * + * Prints `COMPILED OK`, or Cube's own errors, and exits 1 on failure. Wanted because + * Cube compiles every string in a model as a Python f-string, resolves every member + * reference, and enforces one member namespace per cube -- so a model can round-trip + * through Ossie byte-for-byte and still be one Cube refuses to load. Three defects of + * exactly that kind were found by running this. + * + * Needs a built Cube checkout (`yarn build` in the monorepo, or an installed + * node_modules with dist/). `tests/test_cube_compiles.py` skips when there isn't one, + * so this is a local and release-time gate rather than a CI one. + */ + +const fs = require('fs'); +const path = require('path'); + +const repo = process.env.OSSIE_CUBE_REPO; +if (!repo) { + console.log('SKIP OSSIE_CUBE_REPO is not set (point it at a built Cube checkout)'); + process.exit(2); +} + +const compilerDist = path.join( + repo, 'packages/cubejs-schema-compiler/dist/src'); +if (!fs.existsSync(compilerDist)) { + console.log(`SKIP no built schema compiler at ${compilerDist} (run yarn build)`); + process.exit(2); +} + +// The monorepo's packages are built independently, so a schema-compiler build can ask +// `getEnv` for a variable an older cubejs-backend-shared build does not know, which +// throws. Unknown keys fall back to undefined rather than taking the run down: this +// script is asking about *model* validity, not about environment configuration. +try { + const shared = require( + path.join(repo, 'packages/cubejs-backend-shared/dist/src/env')); + const realGetEnv = shared.getEnv; + shared.getEnv = (key, ...rest) => { + try { + return realGetEnv(key, ...rest); + } catch (e) { + return undefined; + } + }; +} catch (e) { + // Older or differently-laid-out checkout: carry on and let compile() report. +} + +const { prepareCompiler } = require(path.join(compilerDist, 'compiler/PrepareCompiler')); + +const files = process.argv.slice(2); +if (!files.length) { + console.log('usage: cube_compile.js [...]'); + process.exit(2); +} + +// Cube keys models by file name, and its loader does not care about directories, so a +// flat list is enough -- `cubes/orders.yml` and `views/sales.yml` compile together. +const dataSchemaFiles = files.map((p) => ({ + fileName: path.basename(p), + content: fs.readFileSync(p, 'utf8'), +})); + +const { compiler } = prepareCompiler( + { localPath: () => path.dirname(files[0]), dataSchemaFiles: () => Promise.resolve(dataSchemaFiles) }, + { adapter: 'postgres' }); + +compiler.compile() + .then(() => console.log('COMPILED OK')) + .catch((e) => { + // Cube's compile errors are the useful part; the stack is noise here. + console.log(`COMPILE FAILED\n${String((e && e.message) || e)}`); + process.exit(1); + }); From b35cc882df93e7704c9ed12fc0cd19eef6313383 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Tue, 4 Aug 2026 16:33:33 +0500 Subject: [PATCH 28/46] Fix five more review findings; the sixth was not a defect Five confirmed and fixed. The P1 was not reproducible -- see below. - A metric over a field with no expression in a usable dialect emitted `{CUBE.field}` for a dimension that was never created. Cube's verdict: "orders.legacy_amount cannot be resolved. There's no such member or cube." The field is no longer claimed as a member, and the metric is dropped with it. - Two relationships between the same pair of datasets became two joins of one name. This does not fail: Cube's transpiler keys joins by target, keeps the last and silently discards the first -- verified by generating SQL, where a `buyer`/`seller` pair joined on `seller_id` and `buyer` was simply gone. Wrong numbers are worse than no output, so this is refused with the remedy in the message. - A generated part name was checked against the *reference* members rather than every dimension, so a plain field named `ratio_part_1` failed the conversion instead of pushing the parts to `_part_2`/`_part_3`. - A stashed measure title was escaped again on the way out, turning a valid `Revenue \{USD\}` into `Revenue \\{USD\\}`. Stash content is byte-identical by design; only Ossie-sourced strings are escaped. - A `case` label kept Cube's brace escaping when it became a SQL literal in the Ossie CASE expression, so a consumer would compare against `large \{special\}`. Not a defect: wrapped single aggregates do NOT lose fan-out protection. Asked directly, `SUM({CUBE}.amount) / 100` as one calculated measure and the same thing split into a hidden `type: sum` plus a ratio produce byte-identical SQL under fan-out -- both go through `SELECT DISTINCT ` and the `keys` subquery, differing only in whether Cube renders the aggregate as `SUM` or `sum`. Cube's correction is keyed on the cube a measure belongs to, not on whether it is structured. Decomposing would add a hidden member for nothing, so the behaviour is now pinned by a test that records the evidence. 381 tests with both gates, 368 with neither, 96% coverage. --- converters/cube/README.md | 2 + converters/cube/src/ossie_cube/cube_to_osi.py | 2 +- converters/cube/src/ossie_cube/osi_to_cube.py | 89 +++++++++++++++--- converters/cube/tests/test_edge_cases.py | 18 ++++ converters/cube/tests/test_osi_to_cube.py | 91 +++++++++++++++++++ 5 files changed, 190 insertions(+), 12 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index ec796a5f..1aab2d39 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -135,6 +135,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | — | `type: switch` dimension | Maps to `String` like an ordinary dimension, and `String` maps back to `string` -- so the Cube type is recorded in the stash, or the dimension would return as a plain string one carrying an orphaned `case` block. | | — | `sub_query: true` dimension | The sql references a *measure*, which an Ossie field expression has no form for. The reference is emitted as text with an `APPROXIMATED` issue, and the flag rides in the stash so export restores the working Cube form. | | `metric.datatype` | `meta.ossie.datatype` | Cube has no field for a measure's result type. Import infers one for the count family (whose result type does not depend on the operand) and reads a parked one otherwise, so a `Decimal` sum survives. | +| several relationships between the same two datasets | — | **Refused.** A cube's `joins` are keyed by target, so Cube holds one join per target; emitting two does not fail but silently keeps the last, and every query through the lost relationship then joins on the surviving predicate. Model the second path as its own dataset. | | relationship `custom_extensions` | cube `meta.ossie.join_extensions` | A Cube join entry takes only name/sql/relationship, so a relationship's foreign-vendor extensions ride on the declaring cube keyed by join target. | | — | `type: geo` dimension | An Ossie field holds one expression and a geo dimension has two, so it **splits** into `_latitude` / `_longitude` (`Float`). Reconstruction data rides on the latitude half. See [Geo dimensions](#geo-dimensions). | | relationship | `joins[]` on a cube | `many_to_one` on cube A -> `from: A`(many), `to: B`(one). `one_to_many` is flipped so Ossie's `from` is the many side; the declared side and type are stashed so export restores the original. | @@ -144,6 +145,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | `COUNT(DISTINCT x)` | `type: count_distinct` | | | `APPROX_COUNT_DISTINCT(x)` | `type: count_distinct_approx` | Cube resolves the warehouse-specific function itself. | | `COUNT(DISTINCT )` | bare `type: count` | See [Fan-out](#fan-out) -- the primary key is load-bearing here. | +| one aggregate inside a larger expression | `type: number` (calculated) | Deliberately not decomposed: Cube applies its row-multiplication correction to a calculated measure just as it does to a structured one. `SUM({CUBE}.amount) / 100` and the same split into a hidden `type: sum` plus a ratio generate *identical* SQL under fan-out. Splitting would add a hidden member and buy nothing. | | several aggregates in one expression | one `public: false` measure per aggregate + a `type: number` measure referencing them | Each part is declared on the cube its own operand reads, so Cube corrects row multiplication per aggregate rather than once for the whole expression. The parts carry `meta.ossie.part_of`, and import skips them and inlines their SQL back through the references -- recovering the original expression exactly. | | anything else | `type: number` (calculated) | A `{other_measure}` reference is **inlined**, because that is what Cube itself does; Ossie has no metric-to-metric reference. | | — | measure `filters` | Folded into `CASE WHEN … THEN … END` inside the aggregate, exactly as Cube's own `applyMeasureFilters` renders it. | diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 8498d727..608daed9 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -741,7 +741,7 @@ def _case_label(cname, dname, holder): f"`sql`") translated, _ = cube_sql_to_ossie(label["sql"], cname) return translated - text = str(label if label is not None else "") + text = unescape_braces_from_cube(str(label if label is not None else "")) return "'" + text.replace("'", "''") + "'" diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index c81bc5c2..c0e216cc 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -37,6 +37,7 @@ from ._common import ( AGG_TO_RESULT_DATATYPE, DATATYPE_TO_DIM_TYPE, + DOTTED_REF_RE, DEFAULT_DATATYPE_FOR_CUBE_TYPE, OSSIE_FUNC_TO_AGG, OSSIE_VERSION, @@ -55,6 +56,7 @@ primary_key_operand, read_stash, referenced_datasets, + quoted_runs, require_str, safe_relative_path, sanitize_name, @@ -146,6 +148,8 @@ def _convert_model(model, dialect, base_cube, issues): # lands. dim_names_by_cube = {} members_by_cube = {} + all_members_by_cube = {} + dropped_by_cube = {} inline_sql_by_cube = {} pk_by_cube = {} for ds_name, ds in datasets.items(): @@ -155,13 +159,19 @@ def _convert_model(model, dialect, base_cube, issues): # Not every member: only those the `{CUBE.member}` form is required for. members_by_cube[cname] = _reference_members( ds, dim_names_by_cube[cname], dialect) + # Every dimension name the cube will carry, and the fields that will not + # become one. A generated measure name has to avoid the first, and a metric + # over the second cannot be rendered at all. + all_members_by_cube[cname] = set(dim_names_by_cube[cname].values()) + dropped_by_cube[cname] = _undialected_fields(ds, dialect) pk_by_cube[cname] = [str(c) for c in (ds.get("primary_key") or [])] joins_by_cube, join_parked_by_cube = _build_joins( relationships, cube_names, issues) measures_by_cube = _build_measures( - model, cube_names, members_by_cube, inline_sql_by_cube, pk_by_cube, - datasets, relationships, base_cube, dialect, issues) + model, cube_names, members_by_cube, all_members_by_cube, dropped_by_cube, + inline_sql_by_cube, pk_by_cube, datasets, relationships, base_cube, dialect, + issues) # Cubes, grouped by the file they belong in: several datasets can share one # stashed original path, in which case they go back into the same file. @@ -414,11 +424,23 @@ def _reference_members(ds, dim_names, dialect): if not dname: continue expr = pick_expression(field.get("expression"), dialect) - if expr is None or not is_simple_identifier(expr) or expr.strip() != dname: + if expr is None: + # No usable dialect: this field becomes no dimension at all, so claiming + # it as a member would make a metric over it emit `{CUBE.name}` -- + # "orders.legacy_amount cannot be resolved", in Cube's words. + continue + if not is_simple_identifier(expr) or expr.strip() != dname: needed.add(dname) return needed +def _undialected_fields(ds, dialect): + """Field names with no expression in a usable dialect, so no Cube dimension.""" + return {field.get("name") for field in (ds.get("fields") or []) + if field.get("name") + and pick_expression(field.get("expression"), dialect) is None} + + def _resolve_dimension_names(ds, scope): """Map each of a dataset's fields to the Cube dimension name it becomes. @@ -660,6 +682,7 @@ def _build_joins(relationships, cube_names, issues): """ joins_by_cube = {} parked_by_cube = {} + declared_targets = {} for rel in relationships: rname = rel.get("name", "") from_cols = rel.get("from_columns") or [] @@ -709,6 +732,21 @@ def _build_joins(relationships, cube_names, issues): foreign = foreign_vendor_extensions(rel) if foreign: parked_by_cube.setdefault(own, {})[other] = foreign + # A Cube cube's `joins` are keyed by target cube name, so it can hold exactly + # one join per target. Emitting two does not fail: the transpiler keeps the + # last and silently discards the first, and every query through the lost + # relationship then joins on the surviving predicate instead -- so a `buyer` + # query returns seller-joined numbers. Wrong numbers are worse than no output, + # which is the same reasoning the fan-out mapping follows. + existing = declared_targets.setdefault(own, {}) + if other in existing: + raise ConversionError( + f"Model: relationships '{existing[other]}' and '{rname}' both join " + f"dataset '{own}' to '{other}'. A Cube cube can declare one join per " + f"target, and emitting both would silently keep only the second. " + f"Model the second path as its own dataset (a view over the same " + f"table) so each join has a distinct target.") + existing[other] = rname joins_by_cube.setdefault(own, []).append( _ordered(join, ["name", "sql", "relationship"])) return joins_by_cube, parked_by_cube @@ -716,9 +754,9 @@ def _build_joins(relationships, cube_names, issues): # --- measures ------------------------------------------------------------------- -def _build_measures(model, cube_names, members_by_cube, inline_sql_by_cube, - pk_by_cube, datasets, relationships, base_cube, dialect, - issues): +def _build_measures(model, cube_names, members_by_cube, all_members_by_cube, + dropped_by_cube, inline_sql_by_cube, pk_by_cube, datasets, + relationships, base_cube, dialect, issues): """Group Ossie metrics into per-cube `measures` lists.""" name = model.get("name", "") sanitized = set(cube_names.values()) @@ -757,6 +795,17 @@ def resolve_base(): "no ANSI_SQL or preferred-dialect expression; metric dropped") continue + missing = _references_a_dropped_field(expr, sanitized, cube_names, + dropped_by_cube) + if missing: + # The field has no expression in a usable dialect, so Cube gets no + # dimension for it -- and a measure referencing one it did not get is a + # model Cube refuses to compile. The metric goes with the field. + issues.add(IssueType.NO_USABLE_DIALECT, scope, + f"references {', '.join(sorted(missing))}, which has no " + f"expression in a usable dialect and so becomes no Cube " + f"dimension; the metric is dropped with it") + continue referenced = referenced_datasets(expr, sanitized) target = stash.get("cube") or ( next(iter(referenced)) if len(referenced) == 1 else resolve_base()) @@ -781,7 +830,8 @@ def resolve_base(): # seeing one opaque expression -- see _decompose_measure. public_sql = _decompose_measure( expr, spans, mname, target, measures_by_cube, members_by_cube, - inline_sql_by_cube, pk_by_cube, sanitized, name) + all_members_by_cube, inline_sql_by_cube, pk_by_cube, sanitized, + name) measure = {"name": mname, "sql": public_sql, "type": "number"} else: measure = _measure_from_expression( @@ -792,9 +842,23 @@ def resolve_base(): return measures_by_cube +def _references_a_dropped_field(expr, sanitized, cube_names, dropped_by_cube): + """`dataset.field` references in `expr` naming a field that becomes no dimension.""" + by_cube_name = {cname: ds_name for ds_name, cname in cube_names.items()} + missing = set() + for text, quoted in quoted_runs(expr): + if quoted: + continue + for match in DOTTED_REF_RE.finditer(text): + cname, fname = match.group(1), match.group(2) + if cname in sanitized and fname in (dropped_by_cube.get(cname) or ()): + missing.add(f"'{by_cube_name.get(cname, cname)}.{fname}'") + return missing + + def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, - members_by_cube, inline_sql_by_cube, pk_by_cube, sanitized, - model_name): + members_by_cube, all_members_by_cube, inline_sql_by_cube, + pk_by_cube, sanitized, model_name): """Emit one `public: false` measure per aggregate; return the sql referencing them. Cube corrects for row multiplication per measure, keyed on the cube that measure @@ -810,7 +874,7 @@ def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, # is unique across dimensions and measures alike -- so the check is over both, and # over every cube rather than the one part it happens to land on. taken = {m["name"].lower() for ms in measures_by_cube.values() for m in ms} - taken |= {n.lower() for ns in members_by_cube.values() for n in ns} + taken |= {n.lower() for ns in all_members_by_cube.values() for n in ns} out, cursor, index = [], 0, 0 for start, end in spans: piece = expr[start:end] @@ -903,7 +967,10 @@ def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_ def _apply_measure_metadata(metric, measure, stash): if stash.get("title"): - measure["title"] = escape_braces_for_cube(stash["title"]) + # Not escaped: this came out of the stash, so it is already whatever Cube + # needs it to be. Escaping it again turned a valid `Revenue \{USD\}` into + # `Revenue \\{USD\\}`. Only Ossie-sourced strings are escaped. + measure["title"] = stash["title"] if metric.get("description"): measure["description"] = escape_braces_for_cube(metric["description"]) parked = {} diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 58a92738..cb91311c 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -1710,3 +1710,21 @@ def test_a_parked_foreign_extension_is_escaped_and_restored(): ossie2, _ = convert_cube_to_ossie(files) restored = by_name(model_of(ossie2)["datasets"])["orders"]["custom_extensions"] assert {"vendor_name": "DBT", "data": '{"project": "x"}'} in restored + + +def test_a_case_label_is_unescaped_on_the_way_into_an_expression(): + """A Cube label is escaped text; the Ossie CASE expression wants a plain SQL + literal. Leaving the backslashes in put them inside the literal, so a consumer + would compare against `large \\{special\\}` rather than `large {special}`.""" + files = _files(products=( + "cubes:\n - name: products\n sql_table: a.b.products\n dimensions:\n" + " - name: size\n type: string\n case:\n when:\n" + " - sql: \"{CUBE}.v = 'x'\"\n" + " label: 'large \\{special\\}'\n")) + ossie, _ = convert_cube_to_ossie(files) + size = by_name(by_name(model_of(ossie)["datasets"])["products"]["fields"])["size"] + assert expr_of(size) == "CASE WHEN v = 'x' THEN 'large {special}' END" + # The stashed `case` block still restores the Cube spelling exactly. + _, back, _ = _roundtrip(files) + dim = by_name(parse(back["model/cubes/products.yml"])["cubes"][0]["dimensions"]) + assert dim["size"]["case"]["when"][0]["label"] == "large \\{special\\}" diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 782d726f..0b470aca 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -625,3 +625,94 @@ def test_relationship_extensions_are_parked_on_the_declaring_cube(): ossie2, _ = convert_cube_to_ossie(files) restored = model_of(ossie2)["relationships"][0]["custom_extensions"] assert {"vendor_name": "DBT", "data": "keep-me"} in restored + + +# --- review round three ---------------------------------------------------------- + +def test_a_wrapped_single_aggregate_stays_one_calculated_measure(): + """Deliberately *not* decomposed, and the reason is worth recording: Cube applies + its row-multiplication correction to a calculated measure exactly as it does to a + structured one. Asked directly, `SUM({CUBE}.amount) / 100` as `type: number` and the + same thing split into a hidden `type: sum` plus a ratio produce identical SQL under + fan-out -- both go through `SELECT DISTINCT ` and the `keys` subquery, differing + only in whether Cube renders the aggregate as `SUM` or `sum`. Splitting it would add + a hidden measure and buy nothing.""" + files, _ = convert_ossie_to_cube( + _ossie(_ORDERS, metrics=_metric("pct", "SUM(orders.amount) / 100"))) + measures = _cubes(files)["orders"]["measures"] + assert measures == [ + {"name": "pct", "sql": "SUM({CUBE}.amount) / 100", "type": "number"}] + + +def test_a_metric_over_a_field_with_no_usable_dialect_is_dropped_too(): + """The field becomes no dimension, so a measure referencing it is a model Cube + refuses: "orders.legacy_amount cannot be resolved. There's no such member or cube." + """ + ds = ( + " - name: orders\n" + " source: sales.public.orders\n" + " primary_key:\n - id\n" + " fields:\n" + " - name: id\n" + " expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: id\n" + " datatype: Integer\n" + " - name: legacy_amount\n" + " expression:\n dialects:\n" + " - dialect: TABLEAU\n expression: amount\n" + " datatype: Decimal\n" + ) + files, issues = convert_ossie_to_cube( + _ossie(ds, metrics=_metric("total", "SUM(orders.legacy_amount)"))) + assert "measures" not in _cubes(files)["orders"] + assert any("dropped with it" in i.detail + for i in issues.of_type(IssueType.NO_USABLE_DIALECT)) + + +def test_a_generated_part_name_avoids_an_existing_dimension(): + """The suffix loop only sees names it is told about. It used to be given the + *reference* members rather than every dimension, so a plain field named + `ratio_part_1` collided and the conversion failed instead of picking the next + free name.""" + ds = _ORDERS + ( + " - name: ratio_part_1\n" + " expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: ratio_part_1\n" + " datatype: Decimal\n" + ) + files, _ = convert_ossie_to_cube(_ossie(ds, metrics=_metric( + "ratio", "SUM(orders.amount) / COUNT(DISTINCT orders.id)"))) + names = [m["name"] for m in _cubes(files)["orders"]["measures"]] + assert names == ["ratio_part_2", "ratio_part_3", "ratio"] + # And the dimension of that name is untouched. + assert "ratio_part_1" in by_name(_cubes(files)["orders"]["dimensions"]) + + +def test_two_relationships_to_one_dataset_are_refused(): + """A cube's `joins` are keyed by target, so Cube can hold one join per target. + Emitting two does not fail -- the transpiler keeps the last and silently discards + the first, so every query through the lost relationship joins on the surviving + predicate. Verified against Cube: with `buyer` and `seller` both declared, the SQL + joins on `seller_id` and `buyer` is simply gone.""" + rel = (" relationships:\n" + " - name: buyer\n from: orders\n to: users\n" + " from_columns: [id]\n to_columns: [id]\n" + " - name: seller\n from: orders\n to: users\n" + " from_columns: [amount]\n to_columns: [id]\n") + with pytest.raises(ConversionError, match="one join per target"): + convert_ossie_to_cube(_ossie(_TWO_DATASETS, rel)) + + +def test_a_stashed_measure_title_is_not_escaped_twice(): + """It came out of the stash, so it is already whatever Cube needs. Escaping it + again turned a valid `Revenue \\{USD\\}` into `Revenue \\\\{USD\\\\}`.""" + src = {"model/cubes/orders.yml": ( + "cubes:\n - name: orders\n sql_table: a.b.orders\n dimensions:\n" + " - name: id\n sql: id\n type: number\n" + " primary_key: true\n" + " measures:\n - name: revenue\n sql: amount\n type: sum\n" + " title: 'Revenue \\{USD\\}'\n")} + ossie, _ = convert_cube_to_ossie(src) + back, _ = convert_ossie_to_cube(ossie) + measure = parse(back["model/cubes/orders.yml"])["cubes"][0]["measures"][0] + assert measure["title"] == "Revenue \\{USD\\}" From 454b676b5be39ee75ce29cd26748786f4dad798d Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Tue, 4 Aug 2026 17:01:11 +0500 Subject: [PATCH 29/46] Fix five more review findings, two of them silently wrong SQL All five reproduced first. - Ossie regular identifiers are case-insensitive -- core-spec/expression_language.md gives the rule outright ("Regular identifiers are upper cased; quoted identifiers have their quotes stripped") -- and the converter compared them exactly. So `SUM(orders.AMOUNT)` over a computed field `amount` emitted `{CUBE}.AMOUNT`, a raw column that bypasses the member's own expression and sums the wrong thing, and `SUM(ORDERS.amount)` was left as raw SQL naming a table that does not exist. Lookups now use the spec's normalized form and emit the canonical Cube spelling, because Cube's own resolution is case-sensitive. Quoted identifiers keep exact semantics. - A join written as `{CUBE.user_key}` became `from_columns: [user_key]`. Ossie relationship columns are *columns*, and `user_key` is a dimension -- the column is `user_id`. A member reference now resolves to the column it reads, and a member reading an expression (`CONCAT(...)`) parks the whole join, since there is no column to name. The stashed sql only ever repaired the Cube round trip, not the Ossie document other spokes read. - Generated part names were allocated against only the measures built so far, so conversion was order-dependent: a composite `ratio` ahead of a metric named `ratio_part_1` took that name and the later metric collided, while the reverse order worked. Every name the metrics will claim is reserved up front. - A stashed `extra_files` entry for a generated path silently replaced a converted cube with arbitrary text. Refused. - A field with `is_time` and no datatype came back asserting `datatype: DateTime`. The absence is recorded, since the spec says not to infer a scalar type from `is_time` alone. 394 tests with both gates, 381 with neither, 96% coverage. Interop unchanged. --- converters/cube/README.md | 15 +- converters/cube/src/ossie_cube/_common.py | 61 ++++-- converters/cube/src/ossie_cube/cube_to_osi.py | 103 +++++++-- converters/cube/src/ossie_cube/osi_to_cube.py | 52 ++++- converters/cube/tests/test_edge_cases.py | 197 +++++++++++++++++- 5 files changed, 388 insertions(+), 40 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index 1aab2d39..72d9628a 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -128,7 +128,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | field | `dimensions[]` entry | Export: a name that is not a valid Cube identifier is sanitized; a case-insensitive collision is an error, never a silent merge. | | `field.expression` | dimension `sql` | Dataset-scoped, so `{CUBE}.col` <-> `col`. Export emits `{CUBE}.column` for a raw column and `{CUBE.member}` for a declared member, and never spells the cube's own name (which would break under `extends`). | | `field.datatype` | dimension `type` (**required**) | `String`->`string`, `Boolean`->`boolean`, `Date`/`Time`/`DateTime`/`DateTimeTz`->`time`, `Integer`/`Decimal`/`Float`->`number`, `Opaque`->`string`. Import maps back, choosing `Decimal` for `number` -- Cube collapses three Ossie types into one, so any single answer is a guess, and a stated datatype is what another converter can act on. Export parks the exact one in `meta.ossie.datatype`, which import prefers when present, so `Integer` and `Float` still survive a round trip. | -| `field.dimension.is_time` | `type: time` | Import sets `is_time: true` for a time dimension. | +| `field.dimension.is_time` | `type: time` | Import sets `is_time: true` for a time dimension. A field carrying `is_time` but *no* `datatype` records the absence (`meta.ossie.untyped`), since the spec says not to infer a scalar type from `is_time` alone — otherwise `type: time` would come back asserting `DateTime`. | | `field.label` / `description` | dimension `title` / `description` | | | `field.ai_context.instructions` | dimension `meta.ai_context` | Cube's documented AI-only context field. | | `field.expression` (`CASE WHEN …`) | dimension `case` | A Cube `case` dimension carries conditions instead of `sql` (Cube rejects both together), so it has no column to name. It maps to a real Ossie `CASE WHEN … THEN … ELSE … END`: a string `label` becomes a SQL literal, the `{sql: …}` form becomes that expression. The `case` block still rides in the stash, so export restores the Cube form exactly and drops the generated `sql`. | @@ -139,7 +139,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | relationship `custom_extensions` | cube `meta.ossie.join_extensions` | A Cube join entry takes only name/sql/relationship, so a relationship's foreign-vendor extensions ride on the declaring cube keyed by join target. | | — | `type: geo` dimension | An Ossie field holds one expression and a geo dimension has two, so it **splits** into `_latitude` / `_longitude` (`Float`). Reconstruction data rides on the latitude half. See [Geo dimensions](#geo-dimensions). | | relationship | `joins[]` on a cube | `many_to_one` on cube A -> `from: A`(many), `to: B`(one). `one_to_many` is flipped so Ossie's `from` is the many side; the declared side and type are stashed so export restores the original. | -| `from_columns` / `to_columns` | join `sql` | Only an AND-chain of equalities between two member references maps. Anything else (non-equi, range, literal, third cube) is preserved verbatim in the stash. | +| `from_columns` / `to_columns` | join `sql` | Only an AND-chain of equalities mapping to **physical columns** converts. `{CUBE}.user_id` is already one; `{CUBE.user_key}` names a *member*, so it resolves to the column that member reads (`user_id`). A member reading an expression (`CONCAT(...)`) has no column for Ossie to name, so the whole join is preserved verbatim in the stash rather than described wrongly — as is anything else (non-equi, range, literal, third cube). | | metric | `measures[]` on the cube its expression references | Import hoists cube-scoped measures to the model level, qualifying a colliding name as `__` and stashing the original name and owning cube. | | `SUM`/`AVG`/`MIN`/`MAX(x)` | `type: sum`/`avg`/`min`/`max` + `sql` | | | `COUNT(DISTINCT x)` | `type: count_distinct` | | @@ -163,6 +163,14 @@ dimension extras (`format`, `currency`, `granularities`, `case`, `sub_query`, measure, joins with no Ossie form, Jinja-templated members, and files with no Ossie form (`.js`/`.ts` models, non-model YAML). +**Identifier case**: Ossie regular (unquoted) identifiers are case-insensitive — the +core spec's *normalized* form upper-cases them and strips quotes from quoted ones — so +`orders.AMOUNT` addresses the field `amount`. Lookups use that form, and what is +emitted is the canonical **Cube** spelling, because Cube's own member resolution *is* +case-sensitive. Matching exactly, as this converter first did, emitted `{CUBE}.AMOUNT`: +a raw column that bypasses the member's expression, so a metric silently aggregated the +wrong thing. + **Expression dialects**: Cube SQL is the SQL of the model's data source, and the Ossie dialect enum has no `CUBE` entry -- so import emits `ANSI_SQL`, and export prefers `ANSI_SQL` with `--dialect` prepending a warehouse dialect (e.g. @@ -374,6 +382,9 @@ invalid) when an input breaks one of these: validator rejects for a duplicate field name; - a stashed file path is absolute or escapes the output directory: the stash is part of the input document, so a path in it is untrusted; +- a stashed extra file would overwrite a generated cube or view file: those restore + verbatim, so one landing on a generated path would replace a converted model with + arbitrary text; - a cube uses `extends` -- resolving it means reproducing Cube's definition-merge semantics exactly, so it is refused rather than half-applied; - a bare `type: count` measure's cube declares no primary key; diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index 98f470f0..6b54e800 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -500,6 +500,29 @@ def sub_outside_quotes(sql, transform): for text, quoted in quoted_runs(sql)) +def normalize_identifier(name): + """An Ossie identifier in the spec's *normalized* form, for matching. + + From core-spec/expression_language.md: "Regular identifiers (unquoted) should be + case insensitive [...] Regular identifiers are upper cased; quoted identifiers have + their quotes stripped". So `orders.AMOUNT` addresses the field `amount`, and + matching them exactly -- as this converter used to -- emitted `{CUBE}.AMOUNT`, a raw + column that bypasses the member's own expression entirely. + + Cube identifiers, by contrast, *are* case-sensitive, so the canonical Cube spelling + is what gets emitted; this form is only used to find it. + """ + text = str(name).strip() + if len(text) >= 2 and text.startswith('"') and text.endswith('"'): + return text[1:-1].replace('""', '"') + return text.upper() + + +def canonical_map(names): + """{normalized identifier: the name as spelled}, for case-insensitive lookup.""" + return {normalize_identifier(n): n for n in names} + + def referenced_datasets(expr, known): """The dataset names an Ossie expression references, ignoring quoted text. @@ -508,12 +531,15 @@ def referenced_datasets(expr, known): `SUM(orders.amount) || ' per users.id unit'` reads as a two-dataset metric and gets attributed to the base cube rather than to `orders`. """ + canonical = canonical_map(known) found = set() for text, quoted in quoted_runs(expr): if quoted: continue - found |= {m.group(1) for m in DOTTED_REF_RE.finditer(text) - if m.group(1) in known} + for match in DOTTED_REF_RE.finditer(text): + name = canonical.get(normalize_identifier(match.group(1))) + if name is not None: + found.add(name) return found @@ -638,22 +664,33 @@ def ossie_expr_to_cube_sql(expr, own_cube, own_members=(), cube_names=(), own text with a column reference. """ escaped = str(expr).replace("{", "\\{").replace("}", "\\}") - known = set(cube_names) - members = set(own_members) - inline = inline_sql or {} + known = canonical_map(cube_names) + members = canonical_map(own_members) + own_norm = normalize_identifier(own_cube) if own_cube else None + inline_sql_by_norm = { + normalize_identifier(cube): {normalize_identifier(f): sql + for f, sql in fields.items()} + for cube, fields in (inline_sql or {}).items() + } def repl(m): head, name = m.group(1), m.group(2) - substitute = (inline.get(head) or {}).get(name) + # Ossie regular identifiers are case-insensitive, so the reference is matched + # in normalized form; what is *emitted* is the canonical Cube spelling, since + # Cube's own member lookup is case-sensitive. + head_n, name_n = normalize_identifier(head), normalize_identifier(name) + substitute = (inline_sql_by_norm.get(head_n) or {}).get(name_n) if substitute is not None: # Already-Cube SQL, so it bypasses the escaping above; `{CUBE}` inside # it means `head`, which only stays true while head is the own cube. - return (str(substitute) if head == own_cube - else requalify_self_refs(substitute, head)) - if head == own_cube: - return "{CUBE." + name + "}" if name in members else "{CUBE}." + name - if head in known: - return "{" + head + "." + name + "}" + cube = known.get(head_n, head) + return (str(substitute) if head_n == own_norm + else requalify_self_refs(substitute, cube)) + if head_n == own_norm: + return ("{CUBE." + members[name_n] + "}" if name_n in members + else "{CUBE}." + name) + if head_n in known: + return "{" + known[head_n] + "." + name + "}" # Not a dataset in this model -- a genuine schema-qualified table # reference or an unrelated dotted token. Leave it alone. return m.group(0) diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 608daed9..deb6c3c4 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -671,13 +671,18 @@ def _finish_dimension_field(cname, dname, dim, field, stash, issues): f"cube '{cname}': dimension '{dname}' has unknown type '{dtype}'") # A precise datatype parked by a previous export wins over the default the Cube # type maps to, since Cube itself cannot hold the distinction. - parked_dt = parked_of(dim.get("meta")).get("datatype") - field["datatype"] = parked_dt or datatype + parked = parked_of(dim.get("meta")) + # A field that carried no datatype keeps carrying none: Ossie says not to infer a + # scalar type from `is_time` alone, so emitting DateTime for a `type: time` + # dimension would assert something the model never said. + if not parked.get("untyped"): + field["datatype"] = parked.get("datatype") or datatype # `type` is normally regenerated from the datatype, so it costs no stash entry. # A `switch` dimension is the exception: it maps to String like an ordinary one, # and String maps back to `string`, so the type has to be recorded or the # dimension comes back as a plain string one carrying an orphaned `case` block. - if DATATYPE_TO_DIM_TYPE.get(field["datatype"]) != dtype: + # With no datatype there is nothing to regenerate from, so the type is recorded. + if DATATYPE_TO_DIM_TYPE.get(field.get("datatype")) != dtype: stash["dim_type"] = dtype if dtype == "time": field["dimension"] = {"is_time": True} @@ -818,7 +823,7 @@ def _convert_joins(cubes, skipped_files, issues): f"{what}: unknown relationship '{join['relationship']}'") sql = require_str(join, "sql", what) - pairs = _decompose_join_sql(sql, cname, target, what, issues) + pairs = _decompose_join_sql(sql, cname, target, what, cubes, issues) if pairs is None: extra_joins.setdefault(cname, []).append( {"index": index, "join": join}) @@ -875,7 +880,7 @@ def _convert_joins(cubes, skipped_files, issues): return relationships, extra_joins -def _decompose_join_sql(sql, own_cube, target, what, issues): +def _decompose_join_sql(sql, own_cube, target, what, cubes, issues): """Split a Cube join `sql` into (own_column, target_column) pairs. Only an AND-chain of equalities between one own-cube reference and one @@ -891,12 +896,14 @@ def _decompose_join_sql(sql, own_cube, target, what, issues): f"join clause '{clause.strip()}' is not a single equality; " f"preserved in custom_extensions only") return None - left = _ref_target(sides[0], own_cube, target) - right = _ref_target(sides[1], own_cube, target) + left = _ref_target(sides[0], own_cube, target, cubes) + right = _ref_target(sides[1], own_cube, target, cubes) if left is None or right is None: issues.add(IssueType.PARKED_IN_META, what, - f"join clause '{clause.strip()}' is not between two member " - f"references; preserved in custom_extensions only") + f"join clause '{clause.strip()}' does not resolve to two " + f"physical columns -- Ossie relationship columns are columns, so " + f"a member reading an expression has none to name; preserved in " + f"custom_extensions only") return None (lcube, lcol), (rcube, rcol) = left, right if lcube == own_cube and rcube == target: @@ -911,20 +918,76 @@ def _decompose_join_sql(sql, own_cube, target, what, issues): return pairs or None -def _ref_target(side, own_cube, target): - """Resolve one side of a join equality to (cube_name, column), or None.""" - translated, _ = cube_sql_to_ossie(side, own_cube) - translated = translated.strip() - if is_simple_identifier(translated): - # A bare name came from `{CUBE}.col`, `{CUBE.col}`, or `{col}` -- all of - # which address the cube the join is declared on. - return (own_cube, translated) - m = DOTTED_REF_RE.fullmatch(translated) - if m and m.group(1) in (own_cube, target): - return (m.group(1), m.group(2)) +_JOIN_SIDE_RE = re.compile( + r"^\s*\$?\{\s*([^{}]*?)\s*\}\s*(?:\.\s*([A-Za-z_][A-Za-z0-9_]*))?\s*$") + + +def _column_of(cubes, cname, member): + """The physical column a dimension reads, or None when it reads more than one. + + Cube's "no `sql` means the same-named column" rule applies, and a dimension whose + sql is a single column resolves to that column -- so `user_key` with `sql: user_id` + resolves to `user_id`. A computed dimension (`CONCAT(...)`), a geo one, or an + unknown name has no single column and returns None. + """ + for dim in _as_named_list((cubes.get(cname) or {}).get("dimensions"), + f"cube '{cname}' dimensions"): + if dim.get("name") != member: + continue + if snake(dim.get("type") or "") == "geo": + return None + sql = dim.get("sql") + if sql is None: + return member + translated, _ = cube_sql_to_ossie(sql, cname) + translated = translated.strip() + return translated if is_simple_identifier(translated) else None return None +def _ref_target(side, own_cube, target, cubes): + """Resolve one side of a join equality to (cube_name, physical column), or None. + + Ossie's `from_columns`/`to_columns` name *columns*, so the two Cube reference forms + cannot be treated alike. `{CUBE}.user_id` is a raw column and passes straight + through; `{CUBE.user_key}` names a *member*, whose own sql is what Cube joins on -- + so it has to be resolved to the column that member reads. A member that reads an + expression rather than a column has no Ossie column to name at all, and returning + None here parks the whole join instead of inventing one. + """ + text = str(side).strip() + if is_simple_identifier(text): + # Bare SQL, no reference: a column of the cube the join is declared on. + return (own_cube, text) + m = _JOIN_SIDE_RE.match(text) + if not m: + return None + body, suffix = m.group(1).strip(), m.group(2) + head, _, rest = body.partition(".") + aliases = {"CUBE", "TABLE", own_cube} + + if suffix: + # `{X}.column` -- an alias plus a raw column. + if rest or head not in aliases | {target}: + return None + cube = own_cube if head in aliases else target + return (cube, suffix) + + if rest: + # `{X.member}` -- a member reference. + cube = own_cube if head in aliases else head if head == target else None + if cube is None: + return None + column = _column_of(cubes, cube, rest) + return (cube, column) if column else None + + # `{member}` -- an unqualified member of the declaring cube. + if body in aliases: + return None # a bare alias with no column means nothing here + column = _column_of(cubes, own_cube, body) + return (own_cube, column) if column else None + + def _rebuild_join_sql(target, pairs): """The canonical form export emits, used to decide whether the original has to be stashed. The own side is always `{CUBE}` so the join keeps working when the diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index c0e216cc..9959346f 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -56,6 +56,8 @@ primary_key_operand, read_stash, referenced_datasets, + canonical_map, + normalize_identifier, quoted_runs, require_str, safe_relative_path, @@ -205,7 +207,14 @@ def _convert_model(model, dialect, base_cube, issues): # Files a prior import could not convert (`.js` models, Jinja-templated YAML, # non-model YAML) restore verbatim. for fname, text in (model_stash.get("extra_files") or {}).items(): - files[safe_relative_path(fname, "stashed extra file")] = text + path = safe_relative_path(fname, "stashed extra file") + if path in files: + # These restore verbatim, so letting one land on a generated path would + # replace a converted cube or view with arbitrary text and report nothing. + raise ConversionError( + f"stashed extra file '{path}' would overwrite the generated model " + f"file of the same name; rename the dataset or the stashed file.") + files[path] = text return files, issues @@ -577,6 +586,11 @@ def _build_dimensions(ds, cname, dim_names, inline_sql, ref_members, dialect, dt = field.get("datatype") if dt and DEFAULT_DATATYPE_FOR_CUBE_TYPE.get(dim["type"]) != dt: parked["datatype"] = dt + elif not dt: + # Ossie says not to infer a scalar type from `is_time` alone, so the + # absence is recorded: Cube's `type: time` would otherwise come back as + # `datatype: DateTime`, asserting something the model never said. + parked["untyped"] = True # Keys the exporter consumes itself rather than writing onto the dimension: # `sql`/`type` are an older stash shape, `dim_type` supplies the Cube type, # and `geo` was used to merge the halves back together. @@ -768,6 +782,23 @@ def resolve_base(): name, datasets, relationships, base_cube)]) return base_cache[0] + # Every measure name the metrics will produce, reserved up front. Allocating part + # names against only the measures built *so far* made the conversion order- + # dependent: a composite `ratio` ahead of a metric named `ratio_part_1` took that + # name first and the later metric then collided, while the reverse order worked. + reserved = set() + for metric in (model.get("metrics") or []): + mstash = read_stash(metric) + raw = metric.get("name") + if not isinstance(raw, str): + continue + reserved.add((mstash.get("name") + or sanitize_name(raw, "metric", set())).lower()) + if isinstance(mstash.get("measure"), dict): + stashed_name = mstash["measure"].get("name") + if stashed_name: + reserved.add(str(stashed_name).lower()) + measures_by_cube = {} for metric in (model.get("metrics") or []): mname_raw = require_str(metric, "name", "metric") @@ -831,7 +862,7 @@ def resolve_base(): public_sql = _decompose_measure( expr, spans, mname, target, measures_by_cube, members_by_cube, all_members_by_cube, inline_sql_by_cube, pk_by_cube, sanitized, - name) + name, reserved) measure = {"name": mname, "sql": public_sql, "type": "number"} else: measure = _measure_from_expression( @@ -845,20 +876,29 @@ def resolve_base(): def _references_a_dropped_field(expr, sanitized, cube_names, dropped_by_cube): """`dataset.field` references in `expr` naming a field that becomes no dimension.""" by_cube_name = {cname: ds_name for ds_name, cname in cube_names.items()} + canonical = canonical_map(sanitized) + dropped_norm = { + cname: canonical_map(fields) + for cname, fields in dropped_by_cube.items() + } missing = set() for text, quoted in quoted_runs(expr): if quoted: continue for match in DOTTED_REF_RE.finditer(text): - cname, fname = match.group(1), match.group(2) - if cname in sanitized and fname in (dropped_by_cube.get(cname) or ()): + cname = canonical.get(normalize_identifier(match.group(1))) + if cname is None: + continue + fname = (dropped_norm.get(cname) or {}).get( + normalize_identifier(match.group(2))) + if fname is not None: missing.add(f"'{by_cube_name.get(cname, cname)}.{fname}'") return missing def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, members_by_cube, all_members_by_cube, inline_sql_by_cube, - pk_by_cube, sanitized, model_name): + pk_by_cube, sanitized, model_name, reserved): """Emit one `public: false` measure per aggregate; return the sql referencing them. Cube corrects for row multiplication per measure, keyed on the cube that measure @@ -875,6 +915,8 @@ def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, # over every cube rather than the one part it happens to land on. taken = {m["name"].lower() for ms in measures_by_cube.values() for m in ms} taken |= {n.lower() for ns in all_members_by_cube.values() for n in ns} + # Names later metrics will claim, so allocation does not depend on metric order. + taken |= {n for n in reserved if n != mname.lower()} out, cursor, index = [], 0, 0 for start, end in spans: piece = expr[start:end] diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index cb91311c..9f981509 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -568,7 +568,8 @@ def test_join_clause_reaching_an_unrelated_cube_is_preserved(): )) ossie, back, issues = _roundtrip(files) assert "relationships" not in model_of(ossie) - assert any("not between two member references" in i.detail for i in issues) + assert any("does not resolve to two physical columns" in i.detail + for i in issues) assert parse_files(back) == parse_files(files) @@ -1728,3 +1729,197 @@ def test_a_case_label_is_unescaped_on_the_way_into_an_expression(): _, back, _ = _roundtrip(files) dim = by_name(parse(back["model/cubes/products.yml"])["cubes"][0]["dimensions"]) assert dim["size"]["case"]["when"][0]["label"] == "large \\{special\\}" + + +# --- review round four ----------------------------------------------------------- + +_JOIN_MEMBERS = ( + "cubes:\n" + " - name: orders\n" + " sql_table: a.b.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{JOINSQL}\"\n" + " relationship: many_to_one\n" + " dimensions:\n" + " - name: id\n sql: id\n type: number\n primary_key: true\n" + " - name: user_key\n sql: user_id\n type: number\n" + " - name: tenant_user_id\n" + " sql: \"CONCAT({CUBE}.tenant, {CUBE}.user_id)\"\n type: string\n" + " - name: users\n" + " sql_table: a.b.users\n" + " dimensions:\n" + " - name: id\n sql: id\n type: number\n primary_key: true\n" +) + + +def _join_model(join_sql): + return _files(m=_JOIN_MEMBERS.replace("{JOINSQL}", join_sql)) + + +@pytest.mark.parametrize("join_sql,expected", [ + # A raw column passes straight through. + ("{CUBE}.user_id = {users}.id", ["user_id"]), + # A *member* reference names a dimension, not a column -- so it resolves to the + # column that dimension reads. `user_key` reads `user_id`. + ("{CUBE.user_key} = {users.id}", ["user_id"]), + ("{user_key} = {users.id}", ["user_id"]), +]) +def test_a_join_member_resolves_to_the_column_it_reads(join_sql, expected): + """Ossie's from_columns/to_columns are physical columns. Emitting the *member* name + gave downstream converters a column that need not exist -- `user_key` is a dimension, + the column is `user_id`.""" + ossie, back, _ = _roundtrip(_join_model(join_sql)) + rel = model_of(ossie)["relationships"][0] + assert rel["from_columns"] == expected + assert rel["to_columns"] == ["id"] + # The original spelling is stashed, so Cube gets its own form back. + assert parse_files(back) == parse_files(_join_model(join_sql)) + + +def test_a_join_on_a_computed_member_is_parked_whole(): + """`tenant_user_id` is `CONCAT(...)`, so there is no column for Ossie to name. The + join has no Ossie form and is preserved rather than described wrongly.""" + ossie, back, issues = _roundtrip( + _join_model("{CUBE.tenant_user_id} = {users.id}")) + assert "relationships" not in model_of(ossie) + assert any("does not resolve to two physical columns" in i.detail + for i in issues) + assert parse_files(back) == parse_files( + _join_model("{CUBE.tenant_user_id} = {users.id}")) + + +@pytest.mark.parametrize("reference", [ + # Ossie regular identifiers are case-insensitive (core-spec: "Regular identifiers + # are upper cased"), so all of these address the same computed field. + "orders.amount", + "orders.AMOUNT", + "ORDERS.amount", + "Orders.Amount", +]) +def test_identifiers_match_case_insensitively(reference): + """Matching exactly emitted `{CUBE}.AMOUNT` -- a raw column that bypasses the + member's own expression, so the metric silently summed the wrong thing.""" + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: a.b.orders\n" + " fields:\n" + " - name: amount\n" + " expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: amount * 2\n" + " datatype: Decimal\n" + " metrics:\n" + " - name: total\n" + " expression:\n dialects:\n" + f" - dialect: ANSI_SQL\n expression: SUM({reference})\n" + ) + files, _ = convert_ossie_to_cube(ossie) + measure = parse(files["model/cubes/orders.yml"])["cubes"][0]["measures"][0] + # The member reference, which inlines `amount * 2` -- not `{CUBE}.AMOUNT`, a raw + # column that would bypass the member's expression and sum the wrong thing. + assert measure == {"name": "total", "sql": "{CUBE.amount}", "type": "sum"} + + +def test_a_quoted_identifier_keeps_its_exact_case(): + """The spec's normalization strips quotes without upper-casing, so a quoted + identifier stays an exact match -- `"Amount"` is not the field `amount`.""" + from ossie_cube._common import normalize_identifier + + assert normalize_identifier("amount") == "AMOUNT" + assert normalize_identifier('"Amount"') == "Amount" + assert normalize_identifier('"a""b"') == 'a"b' + + +@pytest.mark.parametrize("order", [ + ["ratio", "ratio_part_1"], + ["ratio_part_1", "ratio"], +]) +def test_generated_part_names_do_not_depend_on_metric_order(order): + """Allocating against only the measures built *so far* made this order-dependent: + the composite metric first took `ratio_part_1` and the later metric of that name + then collided, while the reverse order worked.""" + def metric(name): + expr = ("SUM(orders.amount) / COUNT(DISTINCT orders.id)" + if name == "ratio" else "SUM(orders.amount)") + return (f" - name: {name}\n expression:\n dialects:\n" + f" - dialect: ANSI_SQL\n expression: {expr}\n") + + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: a.b.orders\n" + " primary_key:\n - id\n" + " fields:\n" + " - name: id\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: id\n" + " datatype: Integer\n" + " - name: amount\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: amount\n" + " datatype: Decimal\n" + " metrics:\n" + "".join(metric(n) for n in order) + ) + files, _ = convert_ossie_to_cube(ossie) + names = [m["name"] for m in + parse(files["model/cubes/orders.yml"])["cubes"][0]["measures"]] + # Same generated names either way, and the user's own metric keeps its name. + assert sorted(names) == ["ratio", "ratio_part_1", "ratio_part_2", "ratio_part_3"] + + +def test_a_stashed_extra_file_may_not_overwrite_generated_output(): + """`extra_files` restore verbatim, so one landing on a generated path replaced a + converted cube with arbitrary text and reported nothing.""" + import json + + stash = {"_v": 1, "views": {}, + "extra_files": {"model/cubes/orders.yml": "# hijacked\n"}} + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: a.b.orders\n" + " fields:\n" + " - name: id\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: id\n" + " datatype: Integer\n" + " custom_extensions:\n" + " - vendor_name: CUBE\n" + f" data: '{json.dumps(stash)}'\n" + ) + with pytest.raises(ConversionError, match="would overwrite the generated"): + convert_ossie_to_cube(ossie) + + +def test_is_time_without_a_datatype_does_not_acquire_one(): + """Ossie says not to infer a scalar type from `is_time` alone, so a field that + carried no datatype must not come back asserting DateTime.""" + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: events\n" + " source: a.b.events\n" + " fields:\n" + " - name: occurred_at\n" + " expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: occurred_at\n" + " dimension:\n is_time: true\n" + ) + files, _ = convert_ossie_to_cube(ossie) + dim = parse(files["model/cubes/events.yml"])["cubes"][0]["dimensions"][0] + assert dim["type"] == "time" + assert dim["meta"]["ossie"]["untyped"] is True + ossie2, _ = convert_cube_to_ossie(files) + field = by_name(by_name(model_of(ossie2)["datasets"])["events"]["fields"])[ + "occurred_at"] + assert "datatype" not in field + assert field["dimension"]["is_time"] is True From 9e9c8c7b863b5efa013df661e0104ef3ccc47dcf Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Tue, 4 Aug 2026 17:32:17 +0500 Subject: [PATCH 30/46] Fix five more review findings, one of them a real fan-out hole All five reproduced first. [P1] A calculated measure was judged fan-out-safe by its outer Cube type, which says nothing about the aggregates inside it: `SUM({CUBE}.ltv) / 100` is a `type: number` measure whose value is still a sum. On a fanned-out cube it produced an unsafe static Ossie expression with no FANOUT_UNSAFE_METRIC, and `--strict-fanout` accepted it. Safety is now judged on the resolved expression via sqlglot, treating SUM/AVG and non-DISTINCT COUNT as non-idempotent -- and an unparseable expression as unsafe rather than assumed safe. Note this is the *opposite* direction from the wrapped-aggregate finding refuted last round: that one was about Cube's runtime correction on export, which does apply; this one is about a static Ossie expression on import, which cannot. - Cross-cube member names were not canonicalized, so `SUM(orders.amount + USERS.ID)` emitted `{users.ID}` where the member is declared `id`. Cube's lookup is case-sensitive, so the reference did not resolve. The target cube's own spelling is used now, for every cube rather than just the own one. - A quoted identifier was treated like a string literal -- opaque -- so a valid `SUM("Orders"."Amount")` was never rewritten and bypassed the member it named. A double-quoted run is a *name*, so it is parsed, and the spec's table decides the match: `orders."AMOUNT"` is the field `amount`, `orders."Amount"` is not. Literal handling is unchanged and still pinned by a test, since a `{...}` emitted into a literal would be interpolated by Cube's f-string compilation. - A join member pointing at another member resolved one level, so a chain ending in a computed dimension yielded that dimension's name as if it were a column. The chain is followed to its end, with cycle detection; only a chain ending in a real column converts, and the rest park. A dimension whose sql is its own name (`id` with `sql: id`) is the plain case, not a chain -- getting that wrong first made every such join unresolvable. - Part-name allocation ignored stashed members, so a composite metric could generate `ratio_part_1` and then fail when a stashed segment of that name was restored. 413 tests with both gates, 400 with neither, 96% coverage. Interop unchanged. --- converters/cube/README.md | 14 +- converters/cube/src/ossie_cube/_common.py | 66 +++++- converters/cube/src/ossie_cube/cube_to_osi.py | 35 +++- converters/cube/src/ossie_cube/expressions.py | 38 ++++ converters/cube/src/ossie_cube/osi_to_cube.py | 34 ++- converters/cube/tests/test_edge_cases.py | 195 +++++++++++++++++- 6 files changed, 355 insertions(+), 27 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index 72d9628a..c2fc84c2 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -166,10 +166,15 @@ Ossie form (`.js`/`.ts` models, non-model YAML). **Identifier case**: Ossie regular (unquoted) identifiers are case-insensitive — the core spec's *normalized* form upper-cases them and strips quotes from quoted ones — so `orders.AMOUNT` addresses the field `amount`. Lookups use that form, and what is -emitted is the canonical **Cube** spelling, because Cube's own member resolution *is* -case-sensitive. Matching exactly, as this converter first did, emitted `{CUBE}.AMOUNT`: -a raw column that bypasses the member's expression, so a metric silently aggregated the -wrong thing. +emitted is the canonical **Cube** spelling (for the target cube's members too, not just +its own), because Cube's own member resolution *is* case-sensitive. Matching exactly, as +this converter first did, emitted `{CUBE}.AMOUNT`: a raw column that bypasses the +member's expression, so a metric silently aggregated the wrong thing. + +A **quoted** identifier is a name, not a string literal, so it is parsed rather than +skipped — and the spec's table decides what it matches: `orders."AMOUNT"` is the field +`amount` (force-matched to the normalized case), while `orders."Amount"` is not, and +stays a raw quoted column. **Expression dialects**: Cube SQL is the SQL of the model's data source, and the Ossie dialect enum has no `CUBE` entry -- so import emits `ANSI_SQL`, and export @@ -215,6 +220,7 @@ emit a silently-wrong one: | `count_distinct_approx` | `APPROX_COUNT_DISTINCT(x)` | Yes, inherently | | `min` / `max` | `MIN(x)` / `MAX(x)` | Yes -- idempotent under duplication | | `sum`, `avg`, `count` + `sql` | `SUM(x)`, `AVG(x)`, `COUNT(x)` | **No** | +| `number` (calculated) containing one of those | the expression verbatim | **No** — and judged on the *resolved expression*, not the measure type: `SUM({CUBE}.ltv) / 100` is a `number` measure whose value is still a sum. | Only the last row is at risk, and only when its own cube is the `to` (one) side of a relationship in the model. The converter computes that from the Ossie graph and diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index 6b54e800..619c85a7 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -61,6 +61,16 @@ r"(?)` is Cube's bare `type: count` -- @@ -995,14 +1011,16 @@ def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_ if agg is not None: measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( inner, target, members, sanitized, - inline_sql=inline_sql_by_cube) + inline_sql=inline_sql_by_cube, + members_by_cube=all_members_by_cube) measure["type"] = agg return measure # A ratio, a window expression, or a multi-dataset aggregate: Cube expresses # these as a calculated measure whose sql carries the aggregation. measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( - expr, target, members, sanitized, inline_sql=inline_sql_by_cube) + expr, target, members, sanitized, inline_sql=inline_sql_by_cube, + members_by_cube=all_members_by_cube) measure["type"] = "number" return measure diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 9f981509..74483737 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -1428,7 +1428,9 @@ def test_several_semantic_models_convert_the_first_with_an_issue(): ("a = 'x'", [("a = ", False), ("'x'", True)]), ("'x' = a", [("'x'", True), (" = a", False)]), ("'it''s'", [("'it'", True), ("'s'", True)]), - ('"col" = `c`', [('"col"', True), (" = ", False), ("`c`", True)]), + # A double-quoted run is an *identifier*, not a literal, so it stays parseable -- + # `QUOTED_DOTTED_REF_RE` matches it as one identifier part. + ('"col" = `c`', [('"col" = ', False), ("`c`", True)]), ("a = 'unterminated", [("a = ", False), ("'unterminated", True)]), ("plain", [("plain", False)]), ]) @@ -1923,3 +1925,194 @@ def test_is_time_without_a_datatype_does_not_acquire_one(): "occurred_at"] assert "datatype" not in field assert field["dimension"]["is_time"] is True + + +# --- review round five ----------------------------------------------------------- + +_FANOUT_CALC = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: a.b.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {users}.id\"\n" + " relationship: many_to_one\n" + " dimensions:\n" + " - name: user_id\n sql: user_id\n type: number\n" + " - name: users\n" + " sql_table: a.b.users\n" + " dimensions:\n" + " - name: id\n sql: id\n type: number\n primary_key: true\n" + " measures:\n" + " - name: ltv_pct\n" + " sql: \"SUM({CUBE}.ltv) / 100\"\n" + " type: number\n" +)) + + +def test_a_calculated_measure_is_judged_on_its_aggregates_not_its_type(): + """A Cube calculated measure is classified by its outer type, which says nothing + about the aggregates inside: `SUM({CUBE}.ltv) / 100` is a `number` measure whose + value is still a sum. Judging it by `type` alone let an unsafe expression through + unreported -- even under strict mode.""" + _, issues = convert_cube_to_ossie(_FANOUT_CALC) + assert issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) + with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): + convert_cube_to_ossie(_FANOUT_CALC, strict_fanout=True) + + +@pytest.mark.parametrize("expr,unsafe", [ + ("SUM(users.ltv) / 100", True), + ("AVG(users.ltv)", True), + ("COUNT(users.id)", True), # no DISTINCT: duplication counts twice + ("COUNT(DISTINCT users.id)", False), # idempotent under duplication + ("MIN(users.x) + MAX(users.y)", False), + ("COUNT(DISTINCT users.id) / MAX(users.x)", False), + ("users.a + users.b", False), # no aggregate at all + ("not valid sql (((", True), # unparseable: assume the worse +]) +def test_non_idempotent_aggregate_detection(expr, unsafe): + from ossie_cube.expressions import has_non_idempotent_aggregate + + assert has_non_idempotent_aggregate(expr) is unsafe + + +def test_a_cross_cube_member_gets_the_target_cubes_own_spelling(): + """`{users.ID}` does not resolve when the member is declared `id` -- Cube's member + lookup is case-sensitive even though Ossie's identifiers are not.""" + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: a.b.orders\n" + " fields:\n" + " - name: amount\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: amount\n" + " datatype: Decimal\n" + " - name: users\n" + " source: a.b.users\n" + " primary_key:\n - id\n" + " fields:\n" + " - name: id\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: id\n" + " datatype: Integer\n" + " relationships:\n" + " - name: r\n from: orders\n to: users\n" + " from_columns: [amount]\n to_columns: [id]\n" + " metrics:\n" + " - name: m\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: SUM(orders.amount + USERS.ID)\n" + ) + files, _ = convert_ossie_to_cube(ossie) + assert parse(files["model/cubes/orders.yml"])["cubes"][0][ + "measures"][0]["sql"] == "{CUBE}.amount + {users.id}" + + +@pytest.mark.parametrize("reference,expected", [ + # An ANSI double-quoted identifier is a *name*, not a string literal, so it is + # parsed. Per core-spec, a quoted identifier is force-matched to the normalized + # (upper) case: `"AMOUNT"` is the field `amount`, `"Amount"` is not. + ('orders."AMOUNT"', "SUM({CUBE.amount})"), + ('"ORDERS"."AMOUNT"', "SUM({CUBE.amount})"), + ('orders."Amount"', 'SUM({CUBE}."Amount")'), + ('orders."amount"', 'SUM({CUBE}."amount")'), +]) +def test_quoted_identifiers_are_parsed_not_skipped(reference, expected): + """The whole double-quoted region used to be treated as opaque -- the same handling + string literals get -- so a quoted reference was never rewritten and bypassed the + member it named.""" + from ossie_cube._common import ossie_expr_to_cube_sql + + assert ossie_expr_to_cube_sql( + f"SUM({reference})", "orders", {"amount"}, {"orders"}) == expected + + +def test_a_single_quoted_literal_is_still_opaque(): + """The change above must not weaken literal handling: Cube compiles every string as + an f-string, so a `{...}` emitted into a literal would be interpolated.""" + from ossie_cube._common import ossie_expr_to_cube_sql + + assert ossie_expr_to_cube_sql( + "SUM(orders.amount) || ' orders.amount '", "orders", {"amount"}, + {"orders"}) == "SUM({CUBE.amount}) || ' orders.amount '" + + +_CHAIN = ( + "cubes:\n" + " - name: orders\n" + " sql_table: a.b.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE.user_key} = {users.id}\"\n" + " relationship: many_to_one\n" + " dimensions:\n" + " - name: user_key\n sql: \"{CUBE.mid}\"\n type: string\n" + " - name: mid\n sql: \"{MID_SQL}\"\n type: string\n" + " - name: users\n" + " sql_table: a.b.users\n" + " dimensions:\n" + " - name: id\n sql: id\n type: number\n primary_key: true\n" +) + + +@pytest.mark.parametrize("mid_sql,expected", [ + # The chain ends in a real column, so the relationship names that column. + ("user_id", ["user_id"]), + # It ends in an expression, so there is no column to name: the join is parked. + ("CONCAT({CUBE}.a, {CUBE}.b)", None), + # A cycle Cube would reject must not hang the walk either. + ("{CUBE.user_key}", None), +]) +def test_a_join_member_chain_is_followed_to_its_end(mid_sql, expected): + """`{CUBE.x}` flattens to the bare name `x`, which *looks* like a column but is only + one if `x` itself reads one. Resolving a single level treated a computed dimension at + the end of the chain as a physical column.""" + files = _files(m=_CHAIN.replace("{MID_SQL}", mid_sql)) + ossie, back, _ = _roundtrip(files) + rels = model_of(ossie).get("relationships") + if expected is None: + assert rels is None + else: + assert rels[0]["from_columns"] == expected + # Either way Cube gets its own model back. + assert parse_files(back) == parse_files(files) + + +def test_a_generated_part_name_avoids_a_stashed_member(): + """A stashed segment, `switch` dimension or multi-stage measure is restored verbatim + on export, so a part name colliding with one failed the conversion at the very end + rather than picking the next free name.""" + import json + + stash = {"_v": 1, + "cube_extras": {"segments": [{"name": "ratio_part_1", "sql": "x"}]}} + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: a.b.orders\n" + " primary_key:\n - id\n" + " fields:\n" + " - name: id\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: id\n" + " datatype: Integer\n" + " - name: amount\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: amount\n" + " datatype: Decimal\n" + " custom_extensions:\n" + " - vendor_name: CUBE\n" + f" data: '{json.dumps(stash)}'\n" + " metrics:\n" + " - name: ratio\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: SUM(orders.amount) / COUNT(DISTINCT orders.id)\n" + ) + files, _ = convert_ossie_to_cube(ossie) + cube = parse(files["model/cubes/orders.yml"])["cubes"][0] + assert [m["name"] for m in cube["measures"]] == [ + "ratio_part_2", "ratio_part_3", "ratio"] + assert [s["name"] for s in cube["segments"]] == ["ratio_part_1"] From e169d09de6bcb5a5d24007611438c907540e6808 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Tue, 4 Aug 2026 18:01:49 +0500 Subject: [PATCH 31/46] Generate from both ends; the new direction found a defect immediately Five review rounds found 27 defects, and every one of the last ten came from an input shape the generator could not produce. This widens it. The larger gap was not the shapes but the *direction*: every property so far started from a Cube model, so it only ever asserted things about documents that came out of a Cube file and therefore carry a stash. A hand-authored Ossie model has none -- every key the exporter writes is one it chose rather than restored -- which is the harder direction and where the last two rounds' findings all landed. There is now a generator for it, asserting the document is spec-valid, that Ossie -> Cube -> Ossie preserves every metric and field expression, and (gated) that Cube compiles the result. It failed on its first run, on the most ordinary star schema there is: the fact carries `dim_0_id` as a foreign key, and prefixing `dim_0`'s own `id` produces that same name -- so the prefix remedy added two rounds ago collided in its own right, and the code *refused the model*. Refusing was wrong. The clashing member is now excluded from the generated view and reported; it stays queryable on the cube itself. It also caught a bug in the generator: unquoted `rnd.text()` emitted `description: 61`, an integer, which the spec validator on the *input* rejected -- the reason that check is there. Also added to the Cube-side generator, all mapping to past findings: - join keys in all three reference forms (raw column, member, member chain), where only the first is a column and the others must resolve or park; - calculated `type: number` measures, whose value is still an aggregate; - mixed-case Cube member names, which must come back spelled exactly as written. Metric and field comparison is on the spec's normalized identifier form, since canonicalizing reference case is intended behaviour -- `MAX(FACT.value)` and `MAX(fact.value)` are the same expression. The normalization rules themselves stay pinned by targeted tests. 487 tests with both gates, 461 with neither, 96% coverage. --- converters/cube/README.md | 16 +- converters/cube/src/ossie_cube/osi_to_cube.py | 46 ++-- converters/cube/tests/_roundtrip_helpers.py | 251 +++++++++++++++++- converters/cube/tests/test_osi_to_cube.py | 40 +++ .../cube/tests/test_roundtrip_properties.py | 32 ++- 5 files changed, 359 insertions(+), 26 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index c2fc84c2..93fb65bc 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -433,8 +433,20 @@ Example-based unit tests per direction, CLI behavior tests, fixture round-trip t (including the [TPC-DS model](../../examples/tpcds_semantic_model.yaml) the converter guide asks for as a baseline), a **feature matrix** of one fixture per Cube data-model feature, core-spec validation of every emitted Ossie document, and Hypothesis -property-based round-trip tests over generated Cube models -- which fall back to a -seeded sweep when `hypothesis` is unavailable, so the properties still run. +property-based round-trip tests **from both ends** -- which fall back to a seeded +sweep when `hypothesis` is unavailable, so the properties still run. + +Generating from Cube only proves things about models that came *out* of a Cube file, and +therefore carry a stash. A hand-authored Ossie model has none, so every key the exporter +writes is one it chose rather than restored -- which is the harder direction, and where +review findings kept landing. So there is a second generator for that direction, drawing +composite metrics (the decomposition path), mixed-case and quoted references, computed +fields, and join keys in all three reference forms. It asserts the generated document is +spec-valid, that `Ossie -> Cube -> Ossie` preserves every metric and field expression +modulo the identifier-case canonicalization, and -- when a Cube checkout is available -- +that Cube compiles the export. It found a defect on its first run: a generated view over +an ordinary star schema, where the fact's `users_id` foreign key collides with `users.id` +once prefixed. `tests/fixtures/features/` holds the feature matrix: `case`/`switch` dimensions, custom granularities, presentation and masking metadata, measure variants diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index a3c0b944..6fd05ab5 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -212,8 +212,8 @@ def _convert_model(model, dialect, base_cube, issues): for m in (cube.get(key) or []) if isinstance(m, dict) and m.get("name")] for vpath, views in _build_views(model, model_stash, cube_names, relationships, - datasets, base_cube, - emitted_members).items(): + datasets, base_cube, emitted_members, + issues).items(): files_content.setdefault(vpath, {}).setdefault("views", []).extend(views) files = {path: dump_yaml(content) for path, content in files_content.items()} @@ -1063,7 +1063,7 @@ def _balanced(s): # --- views ---------------------------------------------------------------------- def _build_views(model, model_stash, cube_names, relationships, datasets, - base_cube, emitted_members): + base_cube, emitted_members, issues): """Return {file path: [view dict, ...]}. A list per path, not a single view: several views can share one YAML file, and @@ -1126,12 +1126,13 @@ def _build_views(model, model_stash, cube_names, relationships, datasets, cube_names, relationships, cube_names[_pick_base_cube(model.get("name", ""), datasets, relationships, base_cube)], - emitted_members) + emitted_members, vname, issues) out[view_file(vname)] = [view] return out -def _view_cubes(cube_names, relationships, base, emitted_members): +def _view_cubes(cube_names, relationships, base, emitted_members, view_name, + issues): """Build a generated view's `cubes:` list: the base cube plus every cube reachable from it, each addressed by its full `join_path`. @@ -1162,19 +1163,30 @@ def members(cname): paths[neighbor] = f"{paths[current]}.{neighbor}" own = members(neighbor) entry = {"join_path": paths[neighbor], "includes": "*"} - if any(m.lower() in claimed for m in own): + prefixed = any(m.lower() in claimed for m in own) + if prefixed: entry["prefix"] = True - names = [f"{neighbor}_{m}" for m in own] - else: - names = list(own) - still_colliding = sorted(n for n in names if n.lower() in claimed) - if still_colliding: - raise ConversionError( - f"generated view: member(s) {', '.join(still_colliding)} from " - f"dataset '{neighbor}' collide with another dataset's even with a " - f"prefix; Cube views keep one namespace, so rename one in the " - f"Ossie model.") - claimed.update(n.lower() for n in names) + # A prefix can collide in its own right: in a star schema the fact carries + # `dim_0_id` as its foreign key, and prefixing `dim_0`'s `id` produces that + # same name. Cube holds one member per name, so the ones that still clash are + # excluded rather than refused -- the model is ordinary, and the excluded + # member is reachable on the cube itself. + kept, dropped = [], [] + for member in own: + emitted = f"{neighbor}_{member}" if prefixed else member + if emitted.lower() in claimed: + dropped.append(member) + else: + kept.append(emitted) + if dropped: + entry["excludes"] = sorted(dropped) + issues.add( + IssueType.APPROXIMATED, f"view '{view_name}'", + f"member(s) {', '.join(sorted(dropped))} of dataset '{neighbor}' " + f"are excluded from the generated view: their names collide with " + f"another dataset's and a Cube view keeps one member namespace. " + f"They remain queryable on the cube itself.") + claimed.update(n.lower() for n in kept) entries.append(entry) queue.append(neighbor) # A cube no relationship reaches cannot be addressed by a join path, so it is diff --git a/converters/cube/tests/_roundtrip_helpers.py b/converters/cube/tests/_roundtrip_helpers.py index cb28e630..3afefdd0 100644 --- a/converters/cube/tests/_roundtrip_helpers.py +++ b/converters/cube/tests/_roundtrip_helpers.py @@ -124,19 +124,49 @@ def _build_cube(rnd, name, is_fact, dim_names): if rnd.chance(0.5): cube["description"] = rnd.text() + join_keys = {} if is_fact and dim_names: - cube["joins"] = [ - {"name": d, "sql": "{CUBE}." + f"{d}_id" + " = {" + f"{d}.id" + "}", - "relationship": "many_to_one"} - for d in dim_names - ] + joins = [] + for d in dim_names: + # The three reference forms a join key can take. Only the raw column is a + # column; the others name *members*, and Ossie relationship columns are + # columns -- so the converter has to resolve or park them. Generating only + # the raw form left both paths to the targeted tests. + form = rnd.pick(("raw", "member", "chained")) + join_keys[d] = form + if form == "raw": + left = "{CUBE}." + f"{d}_id" + elif form == "member": + left = "{CUBE." + f"{d}_key" + "}" + else: + left = "{CUBE." + f"{d}_via" + "}" + joins.append({"name": d, "sql": left + " = {" + f"{d}.id" + "}", + "relationship": "many_to_one"}) + cube["joins"] = joins dimensions = [{"name": "id", "sql": "id", "type": "number", "primary_key": True}] for d in dim_names: dimensions.append({"name": f"{d}_id", "sql": f"{d}_id", "type": "number"}) + form = join_keys.get(d) + if form == "member": + # A renamed dimension: the join names the member, the column is `_id`. + dimensions.append({"name": f"{d}_key", "sql": f"{d}_id", + "type": "number"}) + elif form == "chained": + # A member pointing at another member, which is where single-level + # resolution used to stop and hand back a member name as a column. + dimensions.append({"name": f"{d}_via", + "sql": "{CUBE." + f"{d}_key" + "}", + "type": "number"}) + dimensions.append({"name": f"{d}_key", "sql": f"{d}_id", + "type": "number"}) for i in range(rnd.count(0, 3)): - dimensions.append(_build_dimension(rnd, f"attr_{i}")) + # Cube identifiers *are* case-sensitive, so a mixed-case member name has to come + # back spelled exactly as written -- the converter normalizes case only when + # matching Ossie references, never when emitting a Cube name. + name = f"Attr{i}" if rnd.chance(0.3) else f"attr_{i}" + dimensions.append(_build_dimension(rnd, name)) if rnd.chance(0.25): dimensions.append({ "name": "place", "type": "geo", @@ -158,6 +188,16 @@ def _build_cube(rnd, name, is_fact, dim_names): if rnd.chance(0.25): measure["format"] = "currency" measures.append(measure) + # A calculated measure: classified by its outer type, which says nothing about the + # aggregates inside it. Only idempotent ones here -- a `SUM` inside a calculated + # measure on a fanned-out cube is reported rather than converted silently, and that + # refusal has its own targeted test. + if rnd.chance(0.3): + measures.append({ + "name": "calc", + "sql": "MAX({CUBE}.value) - MIN({CUBE}.value)", + "type": "number", + }) cube["measures"] = measures return cube @@ -225,3 +265,202 @@ def check_model(files): assert_cube_roundtrip_is_lossless(files) assert_ossie_roundtrip_is_lossless(files) assert_ossie_is_spec_valid(files) + + +# --- hand-authored Ossie --------------------------------------------------------- +# +# The builders above all start from Cube, so every property they assert is about a +# model that came *out* of a Cube file -- and therefore carries a stash. A +# hand-authored Ossie model has none, so every key the exporter writes is one it +# chose rather than restored, which is the harder direction and the one review +# findings kept landing in: cross-cube member spelling, quoted identifiers, generated +# view members, part-name allocation. None of it had generated coverage. + +# Aggregates safe on any dataset, so a generated metric never depends on fan-out. +OSSIE_AGGS = ["MIN", "MAX", "COUNT_DISTINCT"] + + +def _ossie_agg(func, reference): + return (f"COUNT(DISTINCT {reference})" if func == "COUNT_DISTINCT" + else f"{func}({reference})") + + +def _yaml_text(value): + """A single-quoted YAML scalar, so a generated `61` stays the string "61". + + Emitting text unquoted made the generator produce documents the spec rejects + (`description: 61` is an integer), which the validity check on the *input* caught -- + the reason that check is there. + """ + return "'" + str(value).replace("'", "''") + "'" + + +def _cased(rnd, name): + """The same identifier, sometimes spelled in another case or quoted. + + Ossie regular identifiers are case-insensitive and the spec's normalized form + upper-cases them, so all of these address the same field -- and a quoted upper-case + one does too ("force-matched to normalized case"). Generating only lowercase left + the whole matching path to targeted tests. + """ + if rnd.chance(0.15): + return name.upper() + if rnd.chance(0.1): + return '"' + name.upper() + '"' + return name + + +def build_ossie_model(rnd): + """Generate a hand-authored Ossie model (no stash) as a YAML string.""" + dim_names = [f"dim_{i}" for i in range(rnd.count(1, 2))] + fact = "fact" + + lines = ["version: 0.2.0.dev0", "semantic_model:", "- name: shop"] + if rnd.chance(0.5): + lines.append(f" description: {_yaml_text(rnd.text())}") + lines.append(" datasets:") + + fields_by_dataset = {} + for name in [fact] + dim_names: + fields = _ossie_fields(rnd, name, dim_names if name == fact else ()) + fields_by_dataset[name] = fields + lines.append(f" - name: {name}") + lines.append(f" source: shop.public.{name}") + lines.append(" primary_key:") + lines.append(" - id") + if rnd.chance(0.4): + lines.append(f" description: {_yaml_text(rnd.text())}") + lines.append(" fields:") + for fname, expr, datatype in fields: + lines.append(f" - name: {fname}") + lines.append(" expression:") + lines.append(" dialects:") + lines.append(" - dialect: ANSI_SQL") + lines.append(f" expression: {expr}") + lines.append(f" datatype: {datatype}") + + # Every dimension dataset is reachable from the fact, so a generated view has an + # unambiguous root and cross-dataset metrics have a join path. + lines.append(" relationships:") + for d in dim_names: + lines.append(f" - name: {fact}_to_{d}") + lines.append(f" from: {fact}") + lines.append(f" to: {d}") + lines.append(f" from_columns: [{d}_id]") + lines.append(" to_columns: [id]") + + lines.append(" metrics:") + for text in _ossie_metrics(rnd, fact, dim_names, fields_by_dataset): + lines.extend(text) + return "\n".join(lines) + "\n" + + +def _ossie_fields(rnd, dataset, dim_names): + """(name, ANSI expression, datatype) for one dataset's fields.""" + fields = [("id", "id", "Integer")] + for d in dim_names: + fields.append((f"{d}_id", f"{d}_id", "Integer")) + for i in range(rnd.count(1, 2)): + name = f"attr_{i}" if rnd.chance(0.7) else f"Attr{i}" + if rnd.chance(0.3): + # A computed field, which must be referenced as `{CUBE.member}` so Cube + # inlines its expression rather than reading a column of that name. + fields.append((name, f"LOWER({name}_raw)", "String")) + else: + fields.append((name, name, "String")) + fields.append(("value", "value", "Decimal")) + return fields + + +def _ossie_metrics(rnd, fact, dim_names, fields_by_dataset): + """Metric blocks: single aggregates, and composites that decomposition splits.""" + out = [] + + def block(name, expression): + entry = [f" - name: {name}", + " expression:", + " dialects:", + " - dialect: ANSI_SQL", + f" expression: {expression}"] + if rnd.chance(0.3): + entry.insert(1, f" description: {_yaml_text(rnd.text())}") + out.append(entry) + + # One aggregate over a field of the fact, sometimes referenced in another case. + field = rnd.pick([f[0] for f in fields_by_dataset[fact]]) + block("single", _ossie_agg(rnd.pick(OSSIE_AGGS), + f"{_cased(rnd, fact)}.{_cased(rnd, field)}")) + + # A composite over one dataset: two aggregates, so export splits it into a measure + # per aggregate plus a ratio referencing them, and import inlines it back. + if rnd.chance(0.6): + block("composite", "{} / {}".format( + _ossie_agg("MAX", f"{fact}.value"), + _ossie_agg("COUNT_DISTINCT", f"{fact}.id"))) + + # A composite spanning two datasets, which puts each part on its own cube -- the + # case cross-cube member spelling has to get right. + if dim_names and rnd.chance(0.6): + other = rnd.pick(dim_names) + block("crossing", "{} / {}".format( + _ossie_agg("MAX", f"{_cased(rnd, fact)}.value"), + _ossie_agg("COUNT_DISTINCT", f"{_cased(rnd, other)}.{_cased(rnd, 'id')}"))) + return out + + +def assert_ossie_first_roundtrip(ossie_yaml): + """A hand-authored Ossie model exports and re-imports unchanged.""" + from _cube_gate import assert_ossie_is_valid + + # The generator itself has to produce a valid document, or the rest proves nothing. + assert_ossie_is_valid(ossie_yaml, "generated Ossie model") + files, _ = convert_ossie_to_cube(ossie_yaml) + back, _ = convert_cube_to_ossie(files) + assert_ossie_is_valid(back, "re-imported Ossie model") + return files, back + + +def _normalize_refs(expression): + """An expression with every `dataset.field` reference in the spec's normalized form. + + Identifier case is *deliberately* canonicalized by the converter -- `MAX(FACT.value)` + and `MAX(fact.value)` are the same expression, and what comes back is the canonical + spelling -- so comparing raw text would report intended behaviour as a change. The + normalization rules themselves are pinned by targeted tests, not by this one. + """ + from ossie_cube._common import (QUOTED_DOTTED_REF_RE, normalize_identifier, + split_dotted_ref) + + def repl(match): + head, name = split_dotted_ref(match.group(0)) + return f"{normalize_identifier(head)}.{normalize_identifier(name)}" + + return QUOTED_DOTTED_REF_RE.sub(repl, expression) + + +def check_ossie_model(ossie_yaml): + files, back = assert_ossie_first_roundtrip(ossie_yaml) + original = load_yaml(ossie_yaml)["semantic_model"][0] + returned = load_yaml(back)["semantic_model"][0] + + # Metrics must come back with the same names and the same expressions: a composite + # one is split into hidden measures on the way out and inlined back on the way in, + # which is the most intricate path in the converter and had no generated coverage. + def metrics(model): + return {m["name"]: _normalize_refs(m["expression"]["dialects"][0]["expression"]) + for m in (model.get("metrics") or [])} + + assert metrics(returned) == metrics(original), ( + "Ossie -> Cube -> Ossie changed the metrics") + + # And the fields, which is where a computed expression has to survive as a member + # reference rather than being flattened to a column of the same name. + def fields(model): + return {ds["name"]: {f["name"]: _normalize_refs( + f["expression"]["dialects"][0]["expression"]) + for f in (ds.get("fields") or [])} + for ds in model["datasets"]} + + assert fields(returned) == fields(original), ( + "Ossie -> Cube -> Ossie changed the fields") + return files diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 0b470aca..bc5daa8c 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -716,3 +716,43 @@ def test_a_stashed_measure_title_is_not_escaped_twice(): back, _ = convert_ossie_to_cube(ossie) measure = parse(back["model/cubes/orders.yml"])["cubes"][0]["measures"][0] assert measure["title"] == "Revenue \\{USD\\}" + + +def test_a_generated_view_excludes_members_a_prefix_cannot_disambiguate(): + """The ordinary star schema: the fact carries `users_id` as its foreign key, and + prefixing `users`' own `id` produces that same name -- so the prefix remedy collides + in its own right. Refusing was wrong; the model is as standard as they come. The + clashing member is excluded from the view and reported, and stays queryable on the + cube itself.""" + datasets = ( + " - name: orders\n" + " source: shop.public.orders\n" + " primary_key:\n - id\n" + " fields:\n" + " - name: id\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: id\n" + " datatype: Integer\n" + " - name: users_id\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: users_id\n" + " datatype: Integer\n" + " - name: users\n" + " source: shop.public.users\n" + " primary_key:\n - id\n" + " fields:\n" + " - name: id\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: id\n" + " datatype: Integer\n" + ) + rel = (" relationships:\n" + " - name: r\n from: orders\n to: users\n" + " from_columns: [users_id]\n to_columns: [id]\n") + files, issues = convert_ossie_to_cube(_ossie(datasets, rel)) + view = parse(files["model/views/shop.yml"])["views"][0] + assert view["cubes"] == [ + {"join_path": "orders", "includes": "*"}, + # `users.id` would become `users_id`, which `orders` already has. + {"join_path": "orders.users", "includes": "*", "prefix": True, + "excludes": ["id"]}, + ] + assert any("excluded from the generated view" in i.detail + for i in issues.of_type(IssueType.APPROXIMATED)) diff --git a/converters/cube/tests/test_roundtrip_properties.py b/converters/cube/tests/test_roundtrip_properties.py index 145f2aa1..1311df0f 100644 --- a/converters/cube/tests/test_roundtrip_properties.py +++ b/converters/cube/tests/test_roundtrip_properties.py @@ -24,7 +24,14 @@ """ import pytest -from _roundtrip_helpers import RandomRnd, build_cube_model, check_model +from _cube_gate import assert_cube_compiles, cube_gate +from _roundtrip_helpers import ( + RandomRnd, + build_cube_model, + build_ossie_model, + check_model, + check_ossie_model, +) try: from hypothesis import HealthCheck, given, settings @@ -44,6 +51,23 @@ def test_seeded_models_roundtrip(seed): check_model(build_cube_model(RandomRnd(seed))) +@pytest.mark.parametrize("seed", SEEDS) +def test_seeded_ossie_models_roundtrip(seed): + """The same sweep from the other end: a hand-authored Ossie model, which carries no + stash, so every key the exporter writes is one it chose rather than restored.""" + check_ossie_model(build_ossie_model(RandomRnd(seed))) + + +@cube_gate +@pytest.mark.parametrize("seed", SEEDS[:12]) +def test_seeded_ossie_models_compile_in_cube(seed): + """The export path is where Cube's own verdict matters most: nothing is restored + from a stash, so every member reference, view entry and measure name was chosen by + this converter. A slice of the sweep, since each case spawns a Cube compile.""" + files = check_ossie_model(build_ossie_model(RandomRnd(seed))) + assert_cube_compiles(files, f"generated Ossie model (seed {seed})") + + if HAVE_HYPOTHESIS: class _HypothesisRnd: """The `Rnd` interface backed by a Hypothesis data strategy.""" @@ -97,3 +121,9 @@ def text(self): @given(st.data()) def test_generated_models_roundtrip(data): check_model(build_cube_model(_HypothesisRnd(data))) + + @settings(max_examples=150, deadline=None, + suppress_health_check=[HealthCheck.too_slow]) + @given(st.data()) + def test_generated_ossie_models_roundtrip(data): + check_ossie_model(build_ossie_model(_HypothesisRnd(data))) From 4e71673e419b009c7fb5f14ab19f5cecd3807829 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Tue, 4 Aug 2026 19:59:05 +0500 Subject: [PATCH 32/46] Fix seven more review findings, plus two found while verifying them All seven reproduced first. Two of the fixes exposed further bugs, both mine. [P1] Fan-out was judged on the cube a measure is *declared* on, so `SUM(users.ltv) / SUM(orders.amount)` -- which sits on `orders` while `users` is the fanned-out side -- reported nothing and passed strict mode. It is now judged on the resolved expression, per aggregate, with the datasets each aggregate reads. [P1] The non-idempotent set was a blocklist of SUM/AVG/COUNT, which silently declared every unlisted aggregate safe: STDDEV, VARIANCE, MEDIAN, ARRAY_AGG all passed strict mode on a fanned-out cube. Inverted to an allowlist of the provably idempotent ones (MIN, MAX, COUNT(DISTINCT), APPROX_COUNT_DISTINCT); anything else is unsafe. This immediately flagged a real one in the TPC-DS fixture: `store_productivity` is `SUM(...) / NULLIF(SUM(store.s_number_employees), 0)`, and summing a `store` column across a fanning join inflates it. Unreported until now. - Expressions are authored against *Ossie* names, but lookup received only the sanitized Cube ones -- so `SUM("ORDER ITEMS"."GROSS AMOUNT")` stayed raw SQL naming a table that does not exist. Both spellings resolve now, for datasets and members, and the Cube spelling is what gets emitted. - The dropped-field check matched only unquoted references, so a metric over a field that became no dimension survived with a dangling one. - Aggregate *discovery* was looking inside quoted identifiers after I narrowed the shared quote mask last round, so `orders."SUM(X)"` produced a bogus hidden measure. Reference rewriting needs to see inside a quoted identifier (it is a name); aggregate discovery must not. They now use separate scanners. - A join key written `{CUBE}.tenant_user_id` was resolved as a member when a computed dimension of that name existed, parking a join over a column that was right there. The reference *form* decides now, before translation flattens both to one name. - Stashed segments in the mapping form (`segments: {name: {...}}`) iterated as bare strings, so they were skipped by both name reservation and the collision check -- emitting a cube with a duplicate member. Found while verifying the above, neither reported: - `{users}.ltv` in a measure resolved to `orders.users.ltv`: the own-cube prefix was applied to another cube's alias. A three-part name matches no reference, which is also how it escaped the fan-out analysis. - Making dataset lookup accept both spellings made `referenced_datasets` return the *written* spelling, so a measure was filed under a cube that does not exist and vanished with no issue. It returns the canonical name now. 510 tests with both gates, 485 with neither, 96% coverage. --- converters/cube/README.md | 20 ++- converters/cube/src/ossie_cube/_common.py | 80 ++++++++---- converters/cube/src/ossie_cube/cube_to_osi.py | 76 +++++++++--- converters/cube/src/ossie_cube/expressions.py | 44 +++++-- converters/cube/src/ossie_cube/osi_to_cube.py | 96 +++++++++----- converters/cube/tests/test_edge_cases.py | 117 ++++++++++++++++++ converters/cube/tests/test_osi_to_cube.py | 72 +++++++++++ 7 files changed, 424 insertions(+), 81 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index 93fb65bc..58bb930c 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -222,8 +222,24 @@ emit a silently-wrong one: | `sum`, `avg`, `count` + `sql` | `SUM(x)`, `AVG(x)`, `COUNT(x)` | **No** | | `number` (calculated) containing one of those | the expression verbatim | **No** — and judged on the *resolved expression*, not the measure type: `SUM({CUBE}.ltv) / 100` is a `number` measure whose value is still a sum. | -Only the last row is at risk, and only when its own cube is the `to` (one) side of -a relationship in the model. The converter computes that from the Ossie graph and +Safety is judged on the **resolved expression, per aggregate, per dataset** — not on +the measure's Cube type, and not on the cube it is declared on. Both shortcuts were +wrong: a calculated `type: number` measure's type says nothing about the aggregates +inside it, and the cube a measure is declared on is not necessarily the one an aggregate +inside it *reads*. `SUM(users.ltv) / SUM(orders.amount)` sits on `orders` while `users` +is the fanned-out side. The idempotent set is an **allowlist** (`MIN`, `MAX`, +`COUNT(DISTINCT …)`, `APPROX_COUNT_DISTINCT`) because the set of aggregate functions is +open-ended — listing the unsafe ones silently declared `STDDEV`, `MEDIAN` and +`ARRAY_AGG` safe. + +The TPC-DS fixture carries a real example: `store_productivity` is +`SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), 0)`, and +summing a `store` column across a fanning join inflates it. Cube corrects that at query +time; a static expression cannot, so it is reported. + +Only the last row is at risk, and only when the dataset the aggregate reads is the `to` +(one) side of a relationship in the model. The converter computes that from the Ossie +graph and **records a `FANOUT_UNSAFE_METRIC` issue** naming the metric, the dataset and the relationship responsible -- refusing a whole model over one such metric would leave the spoke on the other side with nothing. Pass `--strict-fanout` to refuse instead, diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index 619c85a7..449088e0 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -353,7 +353,8 @@ def instructions_of(ai_context): return None -def cube_sql_to_ossie(sql, own_cube, resolve_ref=None, self_prefix=None): +def cube_sql_to_ossie(sql, own_cube, resolve_ref=None, self_prefix=None, + cube_names=()): """Translate Cube member references in a SQL string to the plain references Ossie expressions use. Returns (translated, changed). @@ -381,6 +382,7 @@ def cube_sql_to_ossie(sql, own_cube, resolve_ref=None, self_prefix=None): if not isinstance(sql, str): sql = str(sql) changed = False + known_cubes = set(cube_names) protected = sql.replace("\\{", _ESC_OPEN).replace("\\}", _ESC_CLOSE) def repl(m): @@ -401,6 +403,12 @@ def repl(m): # reference. if body in _SELF_REFS or (own_cube and body == own_cube): return _SELF_MARK + if body in known_cubes: + # Another cube's alias (`{users}.ltv` -- a raw column of the joined + # cube), so the trailing `.column` hangs off *that* cube. Prefixing it + # with the own cube produced `orders.users.ltv`, a three-part name no + # reference matches -- which also hid it from the fan-out analysis. + return body return f"{self_prefix}.{body}" if self_prefix else body if head in _SELF_REFS or (own_cube and head == own_cube): return f"{self_prefix}.{rest}" if self_prefix else rest @@ -533,11 +541,6 @@ def normalize_identifier(name): return text.upper() -def canonical_map(names): - """{normalized identifier: the name as spelled}, for case-insensitive lookup.""" - return {normalize_identifier(n): n for n in names} - - def referenced_datasets(expr, known): """The dataset names an Ossie expression references, ignoring quoted text. @@ -546,7 +549,11 @@ def referenced_datasets(expr, known): `SUM(orders.amount) || ' per users.id unit'` reads as a two-dataset metric and gets attributed to the base cube rather than to `orders`. """ - canonical = canonical_map(known) + # `lookup_map`, not `canonical_map`: `known` may map an accepted spelling to the + # canonical name, and what callers need back is the canonical one -- a name they can + # place a measure under. Returning the spelling as written filed it under a cube that + # does not exist, and the measure vanished with no issue reported. + canonical = lookup_map(known) found = set() for text, quoted in quoted_runs(expr): if quoted: @@ -578,16 +585,40 @@ def split_dotted_ref(text): def quoted_char_mask(sql): - """One flag per character of `sql`: True where it sits inside a quoted run. + """One flag per character: True where it sits inside *any* quoted region. - For a caller that needs offsets into the original string rather than a rewrite. + Deliberately wider than `quoted_runs`, and the two are not interchangeable. Rewriting + a reference must look inside a double-quoted identifier, because that is a name. + *Finding an aggregate call* must not: `orders."SUM(X)"` is a column whose name happens + to contain `SUM(`, and treating it as a call produced a bogus hidden measure and + malformed SQL. So single quotes, double quotes and backticks are all opaque here. """ - mask = [] - for text, quoted in quoted_runs(sql): - mask.extend([quoted] * len(text)) + mask, quote = [], None + for ch in str(sql): + if quote: + mask.append(True) + if ch == quote: + quote = None + elif ch in "'\"`": + mask.append(True) + quote = ch + else: + mask.append(False) return mask +def lookup_map(names): + """{normalized identifier: the name to emit}. + + Accepts a set of names (each maps to itself) or a mapping of *any* accepted spelling + to the canonical one -- which is what lets an Ossie name and the Cube name it + sanitizes to both resolve to the Cube spelling. + """ + if isinstance(names, dict): + return {normalize_identifier(k): v for k, v in names.items()} + return {normalize_identifier(n): n for n in names} + + def source_part_count(source): """How many identifier parts a dotted dataset `source` has, or None for a query. @@ -698,8 +729,8 @@ def ossie_expr_to_cube_sql(expr, own_cube, own_members=(), cube_names=(), own text with a column reference. """ escaped = str(expr).replace("{", "\\{").replace("}", "\\}") - known = canonical_map(cube_names) - members = canonical_map(own_members) + known = lookup_map(cube_names) + members = lookup_map(own_members) own_norm = normalize_identifier(own_cube) if own_cube else None inline_sql_by_norm = { normalize_identifier(cube): {normalize_identifier(f): sql @@ -708,7 +739,7 @@ def ossie_expr_to_cube_sql(expr, own_cube, own_members=(), cube_names=(), } foreign_members = { - normalize_identifier(cube): canonical_map(names) + normalize_identifier(cube): lookup_map(names) for cube, names in (members_by_cube or {}).items() } @@ -718,21 +749,26 @@ def repl(m): # in normalized form; what is *emitted* is the canonical Cube spelling, since # Cube's own member lookup is case-sensitive. head_n, name_n = normalize_identifier(head), normalize_identifier(name) + # Resolve the dataset first, then decide which branch applies. Comparing the + # written spelling against `own_cube` directly was wrong once the two could + # differ: dataset `Order Items` becomes cube `order_items`, so a reference to + # `"ORDER ITEMS"` took the cross-cube branch on its own cube. + target = known.get(head_n) + is_own = target == own_cube or (target is None and head_n == own_norm) substitute = (inline_sql_by_norm.get(head_n) or {}).get(name_n) if substitute is not None: # Already-Cube SQL, so it bypasses the escaping above; `{CUBE}` inside # it means `head`, which only stays true while head is the own cube. - cube = known.get(head_n, head) - return (str(substitute) if head_n == own_norm - else requalify_self_refs(substitute, cube)) - if head_n == own_norm: + return (str(substitute) if is_own + else requalify_self_refs(substitute, target or head)) + if is_own: return ("{CUBE." + members[name_n] + "}" if name_n in members else "{CUBE}." + name) - if head_n in known: + if target is not None: # A cross-cube member needs the target cube's own spelling for the same # reason: `{users.ID}` does not resolve when the member is declared `id`. - target = known[head_n] - member = (foreign_members.get(head_n) or {}).get(name_n, name) + member = (foreign_members.get(normalize_identifier(target)) + or {}).get(name_n, name) return "{" + target + "." + member + "}" # Not a dataset in this model -- a genuine schema-qualified table # reference or an unrelated dotted token. Leave it alone. diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index a10185f7..96ed2f82 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -51,6 +51,7 @@ dump_yaml, filtered_operand, is_simple_identifier, + referenced_datasets, join_source, load_yaml, primary_key_count_expression, @@ -65,7 +66,11 @@ write_stash, ) from .converter_issues import IssueLog, IssueType -from .expressions import has_non_idempotent_aggregate, has_top_level_operator +from .expressions import ( + has_non_idempotent_aggregate, + has_top_level_operator, + unsafe_aggregate_spans, +) # Cube keys the converter maps natively at the cube level; everything else is # stashed verbatim in the dataset's `cube_extras` and restored on export. @@ -918,6 +923,11 @@ def _decompose_join_sql(sql, own_cube, target, what, cubes, issues): return pairs or None +# The alias-dot form: `{CUBE}.column` / `{TABLE}.column` / `{cube}.column`. Whatever +# follows the dot is a raw physical column, not a member. +_ALIAS_COLUMN_RE = re.compile( + r"^\$?\{\s*[A-Za-z_][A-Za-z0-9_]*\s*\}\s*\.\s*([A-Za-z_][A-Za-z0-9_]*)\s*$") + _JOIN_SIDE_RE = re.compile( r"^\s*\$?\{\s*([^{}]*?)\s*\}\s*(?:\.\s*([A-Za-z_][A-Za-z0-9_]*))?\s*$") @@ -947,6 +957,12 @@ def _column_of(cubes, cname, member, seen=()): sql = dim.get("sql") if sql is None: return member + if _ALIAS_COLUMN_RE.match(str(sql).strip()): + # The explicit raw-column form (`{CUBE}.tenant_user_id`) names a column, full + # stop -- even if a dimension of that name also exists. Deciding on the + # *translated* text lost that distinction, since both forms flatten to the + # same bare name, and the join was parked over a column that was right there. + return _ALIAS_COLUMN_RE.match(str(sql).strip()).group(1) translated, _ = cube_sql_to_ossie(sql, cname) translated = translated.strip() if not is_simple_identifier(translated): @@ -1048,6 +1064,7 @@ def __init__(self, cubes, pk_by_cube, issues): self._issues = issues self._raw = {} self._cache = {} + self._cube_names = set(cubes) for cname, cube in cubes.items(): for m in _as_named_list(cube.get("measures"), f"cube '{cname}' measures"): self._raw[(cname, require_str(m, "name", f"cube '{cname}': measure"))] = m @@ -1143,7 +1160,7 @@ def _translate(self, sql, cname, stack): """ out, _ = cube_sql_to_ossie( sql, cname, resolve_ref=lambda body: self._inline(body, cname, stack), - self_prefix=cname) + self_prefix=cname, cube_names=self._cube_names) return out def _inline(self, body, cname, stack): @@ -1195,6 +1212,27 @@ def _operand(self, cname, sql, stack): ) +def _fanout_unsafe_datasets(expr, own_cube, dataset_names): + """Datasets read by an aggregate in `expr` that duplicate rows would inflate. + + Per aggregate, because a single expression can mix safe and unsafe ones over + different datasets. An aggregate naming no dataset is read as being over the cube the + measure is declared on. + """ + spans = unsafe_aggregate_spans(expr) + if spans: + found = set() + for start, end in spans: + found |= (referenced_datasets(expr[start:end], dataset_names) + or {own_cube}) + return found + if has_non_idempotent_aggregate(expr): + # An aggregate the span scanner does not know (STDDEV, MEDIAN, ARRAY_AGG...): + # there is no span to attribute, so every dataset the expression reads counts. + return referenced_datasets(expr, dataset_names) or {own_cube} + return set() + + def _windowing_key(measure): """The first windowing key present on a measure, or None.""" for key in _WINDOWING_KEYS: @@ -1251,7 +1289,7 @@ def _convert_measures(cubes, pk_by_cube, plain_by_cube, fanned_out, issues): f"colliding measures in Cube") seen.add(metric_name) metric = _convert_measure(cname, mname, metric_name, measure, resolver, - fanned_out, plain, issues) + fanned_out, plain, set(cubes), issues) if metric is not None: metrics.append(metric) else: @@ -1261,7 +1299,7 @@ def _convert_measures(cubes, pk_by_cube, plain_by_cube, fanned_out, issues): def _convert_measure(cname, mname, metric_name, measure, resolver, fanned_out, - plain, issues): + plain, dataset_names, issues): scope = f"{cname}.{mname}" expr = resolver.expression(cname, mname) if expr is None: @@ -1281,22 +1319,24 @@ def _convert_measure(cname, mname, metric_name, measure, resolver, fanned_out, and not measure.get("filters") ) - # Fan-out: a non-idempotent aggregate on a dataset the graph can multiply. - # Cube fixes this at query time by deduplicating on the primary key; a static - # expression cannot, so the caller has to be told. - unsafe = mtype in FANOUT_UNSAFE_AGGS or (mtype == "count" and sql is not None) - if not unsafe and mtype in CALCULATED_MEASURE_TYPES: - # A calculated measure is classified by its outer type, which says nothing - # about the aggregates inside it: `SUM({CUBE}.ltv) / 100` is a `number` measure - # whose value is still a sum. Judged on the resolved expression instead. - unsafe = has_non_idempotent_aggregate(expr) - if unsafe and cname in fanned_out: + # Fan-out: a non-idempotent aggregate over a dataset the graph can multiply. Cube + # fixes this at query time by deduplicating on the primary key; a static expression + # cannot, so the caller has to be told. + # + # Judged on the resolved expression and per aggregate, not on the measure's Cube + # type and its own cube. Both shortcuts were wrong: a calculated measure's type says + # nothing about the aggregates inside it, and the cube a measure is *declared* on is + # not necessarily the one an aggregate inside it *reads* -- `SUM(users.ltv) / + # SUM(orders.amount)` sits on `orders` while `users` is the fanned-out side. + for dataset in sorted(_fanout_unsafe_datasets(expr, cname, dataset_names)): + if dataset not in fanned_out: + continue issues.add( IssueType.FANOUT_UNSAFE_METRIC, scope, - f"'{mtype}' over dataset '{cname}', which relationship " - f"'{fanned_out[cname]}' fans out; Cube deduplicates on the primary key " - f"at query time but a static Ossie expression cannot, so a consumer " - f"joining through that relationship may over-count") + f"a non-idempotent aggregate reads dataset '{dataset}', which " + f"relationship '{fanned_out[dataset]}' fans out; Cube deduplicates on " + f"the primary key at query time but a static Ossie expression cannot, so " + f"a consumer joining through that relationship may over-count") metric = { "name": metric_name, diff --git a/converters/cube/src/ossie_cube/expressions.py b/converters/cube/src/ossie_cube/expressions.py index 3fdbe491..4307bac9 100644 --- a/converters/cube/src/ossie_cube/expressions.py +++ b/converters/cube/src/ossie_cube/expressions.py @@ -101,7 +101,13 @@ def aggregate_spans(expr): text = str(expr) if parse(text) is None or is_single_aggregate(text): return [] + return _scan_aggregates(text) + +def _scan_aggregates(text): + """Every outermost aggregate call in `text`, as (start, end) offsets.""" + if parse(text) is None: + return [] candidates = [] upper = text.upper() quoted = quoted_char_mask(text) @@ -162,9 +168,18 @@ def _match_paren(text, open_at): return None -# Aggregates whose value changes when input rows are duplicated. `Count` is here only -# in its non-DISTINCT form -- COUNT(DISTINCT x) is idempotent under duplication. -_NON_IDEMPOTENT_NODES = (exp.Sum, exp.Avg) +# The only aggregates whose value survives duplicate input rows. Everything else is +# treated as unsafe -- an allowlist rather than a blocklist, because the set of +# aggregate functions is open-ended (STDDEV, VARIANCE, MEDIAN, ARRAY_AGG, PERCENTILE...) +# and listing the unsafe ones meant every unlisted one was silently declared safe. +_IDEMPOTENT_NODES = (exp.Min, exp.Max, exp.ApproxDistinct) + + +def is_idempotent_aggregate(node): + """True if duplicating input rows cannot change this aggregate's value.""" + if isinstance(node, _IDEMPOTENT_NODES): + return True + return isinstance(node, exp.Count) and _counts_distinct(node) def has_non_idempotent_aggregate(expr): @@ -182,12 +197,23 @@ def has_non_idempotent_aggregate(expr): tree = parse(expr) if tree is None: return True - for node in tree.walk(): - if isinstance(node, _NON_IDEMPOTENT_NODES): - return True - if isinstance(node, exp.Count) and not _counts_distinct(node): - return True - return False + return any(isinstance(node, exp.AggFunc) and not is_idempotent_aggregate(node) + for node in tree.walk()) + + +def unsafe_aggregate_spans(expr): + """(start, end) of every aggregate in `expr` that duplication would inflate. + + Offsets index the original string, so a caller can ask which datasets each unsafe + aggregate reads -- the measure's own cube is not necessarily the fanned-out one. + """ + return [(start, end) for start, end in _scan_aggregates(str(expr)) + if not _parses_to_idempotent(str(expr)[start:end])] + + +def _parses_to_idempotent(text): + node = parse(text) + return node is not None and is_idempotent_aggregate(node) def _counts_distinct(node): diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 6fd05ab5..2ae058e7 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -56,9 +56,11 @@ primary_key_operand, read_stash, referenced_datasets, - canonical_map, + QUOTED_DOTTED_REF_RE, + lookup_map, normalize_identifier, quoted_runs, + split_dotted_ref, require_str, safe_relative_path, sanitize_name, @@ -151,6 +153,7 @@ def _convert_model(model, dialect, base_cube, issues): dim_names_by_cube = {} members_by_cube = {} all_members_by_cube = {} + member_lookup_by_cube = {} dropped_by_cube = {} inline_sql_by_cube = {} pk_by_cube = {} @@ -168,6 +171,9 @@ def _convert_model(model, dialect, base_cube, issues): # or a multi-stage measure is restored verbatim on export, and a generated part # name colliding with one fails the conversion at the very end. ds_stash = read_stash(ds) + member_lookup_by_cube[cname] = dict(dim_names_by_cube[cname]) + member_lookup_by_cube[cname].update( + {dname: dname for dname in dim_names_by_cube[cname].values()}) all_members_by_cube[cname] = ( set(dim_names_by_cube[cname].values()) | {str(item["dimension"]["name"]) @@ -176,18 +182,16 @@ def _convert_model(model, dialect, base_cube, issues): | {str(item["measure"]["name"]) for item in (ds_stash.get("extra_measures") or []) if (item.get("measure") or {}).get("name")} - | {str(seg["name"]) - for seg in ((ds_stash.get("cube_extras") or {}).get("segments") or []) - if isinstance(seg, dict) and seg.get("name")}) + | _stashed_segment_names(ds_stash)) dropped_by_cube[cname] = _undialected_fields(ds, dialect) pk_by_cube[cname] = [str(c) for c in (ds.get("primary_key") or [])] joins_by_cube, join_parked_by_cube = _build_joins( relationships, cube_names, issues) measures_by_cube = _build_measures( - model, cube_names, members_by_cube, all_members_by_cube, dropped_by_cube, - inline_sql_by_cube, pk_by_cube, datasets, relationships, base_cube, dialect, - issues) + model, cube_names, members_by_cube, all_members_by_cube, + member_lookup_by_cube, dropped_by_cube, inline_sql_by_cube, pk_by_cube, + datasets, relationships, base_cube, dialect, issues) # Cubes, grouped by the file they belong in: several datasets can share one # stashed original path, in which case they go back into the same file. @@ -416,8 +420,12 @@ def _build_cube(ds, cname, dim_names, inline_sql, ref_members, joins, measures, def _reject_member_collisions(cname, dimensions, measures, cube_extras, issues): seen = {} + segments = cube_extras.get("segments") + if isinstance(segments, dict): + # The mapping form: keyed by name, with the body as the value. + segments = [{"name": name} for name in segments] groups = [("dimension", dimensions), ("measure", measures), - ("segment", cube_extras.get("segments") or [])] + ("segment", segments or [])] for kind, members in groups: for member in members: if not isinstance(member, dict) or not member.get("name"): @@ -440,7 +448,7 @@ def _reference_members(ds, dim_names, dialect): plain member is identical either way, and the raw-column form is what survives a round trip without stashing the spelling. """ - needed = set() + needed = {} for field in (ds.get("fields") or []): fname = field.get("name") dname = dim_names.get(fname) @@ -453,7 +461,11 @@ def _reference_members(ds, dim_names, dialect): # "orders.legacy_amount cannot be resolved", in Cube's words. continue if not is_simple_identifier(expr) or expr.strip() != dname: - needed.add(dname) + # Both spellings map to the Cube name: an expression is authored against the + # Ossie field name, which for a sanitized name is not the Cube one -- + # `Gross Amount` becomes the dimension `gross_amount`. + needed[dname] = dname + needed[fname] = dname return needed @@ -696,6 +708,21 @@ def _dimension_type(field, stash, scope, issues): # --- joins ---------------------------------------------------------------------- +def _stashed_segment_names(stash): + """Names of the segments a stash carries, in either Cube spelling. + + Cube accepts `segments:` as a list of entries carrying `name` *or* as a mapping keyed + by name. Handling only the list form meant a mapping iterated as bare strings and was + skipped -- so a generated measure could take a name a restored segment also uses, and + the collision check missed it for the same reason, emitting a model Cube rejects. + """ + segments = (stash.get("cube_extras") or {}).get("segments") + if isinstance(segments, dict): + return {str(name) for name in segments} + return {str(seg["name"]) for seg in (segments or []) + if isinstance(seg, dict) and seg.get("name")} + + def _build_joins(relationships, cube_names, issues): """Group Ossie relationships into per-cube `joins` lists. @@ -783,11 +810,16 @@ def _build_joins(relationships, cube_names, issues): # --- measures ------------------------------------------------------------------- def _build_measures(model, cube_names, members_by_cube, all_members_by_cube, - dropped_by_cube, inline_sql_by_cube, pk_by_cube, datasets, - relationships, base_cube, dialect, issues): + member_lookup_by_cube, dropped_by_cube, inline_sql_by_cube, + pk_by_cube, datasets, relationships, base_cube, dialect, issues): """Group Ossie metrics into per-cube `measures` lists.""" name = model.get("name", "") - sanitized = set(cube_names.values()) + # A metric is authored against Ossie names, and a name needing sanitization has a + # different Cube name -- dataset `Order Items` becomes cube `order_items`. Accept + # either spelling and emit the Cube one. Passing only the Cube names left + # `SUM("ORDER ITEMS".amount)` as raw SQL naming a table that does not exist. + sanitized = dict(cube_names) + sanitized.update({cname: cname for cname in cube_names.values()}) base_cache = [] def resolve_base(): @@ -875,14 +907,14 @@ def resolve_base(): # seeing one opaque expression -- see _decompose_measure. public_sql = _decompose_measure( expr, spans, mname, target, measures_by_cube, members_by_cube, - all_members_by_cube, inline_sql_by_cube, pk_by_cube, sanitized, - name, reserved) + all_members_by_cube, member_lookup_by_cube, inline_sql_by_cube, + pk_by_cube, sanitized, name, reserved) measure = {"name": mname, "sql": public_sql, "type": "number"} else: measure = _measure_from_expression( expr, target, mname, stash, members_by_cube.get(target, set()), inline_sql_by_cube, pk_by_cube.get(target, []), sanitized, - all_members_by_cube) + member_lookup_by_cube) _apply_measure_metadata(metric, measure, stash) _place(measures_by_cube, target, measure, name) return measures_by_cube @@ -891,29 +923,33 @@ def resolve_base(): def _references_a_dropped_field(expr, sanitized, cube_names, dropped_by_cube): """`dataset.field` references in `expr` naming a field that becomes no dimension.""" by_cube_name = {cname: ds_name for ds_name, cname in cube_names.items()} - canonical = canonical_map(sanitized) + canonical = lookup_map(sanitized) dropped_norm = { - cname: canonical_map(fields) + cname: lookup_map(fields) for cname, fields in dropped_by_cube.items() } missing = set() for text, quoted in quoted_runs(expr): if quoted: continue - for match in DOTTED_REF_RE.finditer(text): - cname = canonical.get(normalize_identifier(match.group(1))) + # The quoted-reference parser, so `orders."LEGACY_AMOUNT"` is seen as well: + # matching only unquoted references let a metric over a dropped field survive + # with a reference to a dimension that was never created. + for match in QUOTED_DOTTED_REF_RE.finditer(text): + head, field = split_dotted_ref(match.group(0)) + cname = canonical.get(normalize_identifier(head)) if cname is None: continue - fname = (dropped_norm.get(cname) or {}).get( - normalize_identifier(match.group(2))) + fname = (dropped_norm.get(cname) or {}).get(normalize_identifier(field)) if fname is not None: missing.add(f"'{by_cube_name.get(cname, cname)}.{fname}'") return missing def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, - members_by_cube, all_members_by_cube, inline_sql_by_cube, - pk_by_cube, sanitized, model_name, reserved): + members_by_cube, all_members_by_cube, member_lookup, + inline_sql_by_cube, pk_by_cube, sanitized, model_name, + reserved): """Emit one `public: false` measure per aggregate; return the sql referencing them. Cube corrects for row multiplication per measure, keyed on the cube that measure @@ -949,7 +985,7 @@ def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, part = _measure_from_expression( piece, part_target, part_name, {}, members_by_cube.get(part_target, set()), inline_sql_by_cube, - pk_by_cube.get(part_target, []), sanitized, all_members_by_cube) + pk_by_cube.get(part_target, []), sanitized, member_lookup) part["public"] = False part["meta"] = {"ossie": {"part_of": mname}} _place(measures_by_cube, part_target, part, model_name) @@ -957,7 +993,7 @@ def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, out.append(ossie_expr_to_cube_sql( expr[cursor:start], fallback, members_by_cube.get(fallback, set()), sanitized, inline_sql=inline_sql_by_cube, - members_by_cube=all_members_by_cube)) + members_by_cube=member_lookup)) # `{CUBE.x}` for a part on the same cube as the public measure: an explicit # name pins the reference to this cube and breaks if it is extended. qualifier = "CUBE" if part_target == fallback else part_target @@ -965,7 +1001,7 @@ def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, cursor = end out.append(ossie_expr_to_cube_sql( expr[cursor:], fallback, members_by_cube.get(fallback, set()), sanitized, - inline_sql=inline_sql_by_cube, members_by_cube=all_members_by_cube)) + inline_sql=inline_sql_by_cube, members_by_cube=member_lookup)) return "".join(out) @@ -979,7 +1015,7 @@ def _place(measures_by_cube, target, measure, model_name): def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_cube, - primary_key, sanitized, all_members_by_cube=None): + primary_key, sanitized, member_lookup=None): """Turn an Ossie metric expression back into a structured Cube measure. `COUNT(DISTINCT )` is Cube's bare `type: count` -- @@ -1012,7 +1048,7 @@ def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_ measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( inner, target, members, sanitized, inline_sql=inline_sql_by_cube, - members_by_cube=all_members_by_cube) + members_by_cube=member_lookup) measure["type"] = agg return measure @@ -1020,7 +1056,7 @@ def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_ # these as a calculated measure whose sql carries the aggregation. measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( expr, target, members, sanitized, inline_sql=inline_sql_by_cube, - members_by_cube=all_members_by_cube) + members_by_cube=member_lookup) measure["type"] = "number" return measure diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 74483737..e710e387 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -2116,3 +2116,120 @@ def test_a_generated_part_name_avoids_a_stashed_member(): assert [m["name"] for m in cube["measures"]] == [ "ratio_part_2", "ratio_part_3", "ratio"] assert [s["name"] for s in cube["segments"]] == ["ratio_part_1"] + + +# --- review round six ------------------------------------------------------------ + +_TWO_CUBE_CALC = ( + "cubes:\n" + " - name: orders\n" + " sql_table: a.b.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {users}.id\"\n" + " relationship: many_to_one\n" + " dimensions:\n" + " - name: user_id\n sql: user_id\n type: number\n" + " - name: id\n sql: id\n type: number\n primary_key: true\n" + " measures:\n" + " - name: m\n sql: \"{SQL}\"\n type: number\n" + " - name: users\n" + " sql_table: a.b.users\n" + " dimensions:\n" + " - name: id\n sql: id\n type: number\n primary_key: true\n" +) + + +@pytest.mark.parametrize("sql,flagged", [ + # The measure sits on `orders`; `users` is the fanned-out side. Checking only the + # cube a measure is *declared* on reported nothing at all. + ("SUM({users}.ltv) / SUM({CUBE}.amount)", True), + # An aggregate the span scanner does not know still has to be caught. + ("STDDEV({users}.ltv)", True), + ("VARIANCE({users}.ltv) + 1", True), + # Unsafe, but only over the cube that is not fanned out. + ("SUM({CUBE}.amount) / 100", False), + # Idempotent under duplication, so safe on any cube. + ("MAX({users}.ltv) - MIN({users}.ltv)", False), + ("COUNT(DISTINCT {users}.id) / 2", False), +]) +def test_fanout_is_judged_per_aggregate_and_per_dataset(sql, flagged): + files = _files(m=_TWO_CUBE_CALC.replace("{SQL}", sql)) + _, issues = convert_cube_to_ossie(files) + assert bool(issues.of_type(IssueType.FANOUT_UNSAFE_METRIC)) is flagged + if flagged: + with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): + convert_cube_to_ossie(files, strict_fanout=True) + + +@pytest.mark.parametrize("expr,unsafe", [ + # An allowlist, because the set of aggregate functions is open-ended: listing the + # unsafe ones declared every unlisted one safe. + ("STDDEV(users.x)", True), + ("VARIANCE(users.x)", True), + ("MEDIAN(users.x)", True), + ("ARRAY_AGG(users.x)", True), + ("SUM(users.x)", True), + ("COUNT(users.id)", True), + ("MIN(users.x)", False), + ("MAX(users.x)", False), + ("COUNT(DISTINCT users.id)", False), + ("APPROX_COUNT_DISTINCT(users.x)", False), +]) +def test_only_provably_idempotent_aggregates_are_treated_as_safe(expr, unsafe): + from ossie_cube.expressions import has_non_idempotent_aggregate + + assert has_non_idempotent_aggregate(expr) is unsafe + + +def test_a_cross_cube_alias_is_not_prefixed_with_the_own_cube(): + """`{users}.ltv` is a raw column of the *joined* cube, so the trailing column hangs + off `users`. Prefixing it with the declaring cube produced `orders.users.ltv` -- a + three-part name no reference matches, which also hid it from the fan-out analysis.""" + files = _files(m=_TWO_CUBE_CALC.replace( + "{SQL}", "MAX({users}.ltv) - MIN({CUBE}.amount)")) + ossie, _ = convert_cube_to_ossie(files) + assert expr_of(by_name(model_of(ossie)["metrics"])["m"]) == ( + "MAX(users.ltv) - MIN(orders.amount)") + + +def test_an_aggregate_name_inside_a_quoted_identifier_is_not_a_call(): + """`orders."SUM(X)"` is a column whose name happens to contain `SUM(`. Reference + rewriting has to look inside a quoted identifier -- it is a name -- but aggregate + *discovery* must not, or it splits out a hidden measure and emits malformed SQL.""" + from ossie_cube.expressions import aggregate_spans + + expr = 'MAX(orders.value) + orders."SUM(X)"' + assert [expr[s:e] for s, e in aggregate_spans(expr)] == ["MAX(orders.value)"] + + +def test_an_explicit_raw_column_wins_over_a_dimension_of_that_name(): + """`{CUBE}.tenant_user_id` names a column, full stop -- even where a computed + dimension of that name also exists. Deciding on the translated text lost the + distinction, because both reference forms flatten to the same bare name.""" + model = ( + "cubes:\n" + " - name: orders\n" + " sql_table: a.b.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE.user_key} = {users.id}\"\n" + " relationship: many_to_one\n" + " dimensions:\n" + " - name: user_key\n sql: \"{KEY}\"\n type: string\n" + " - name: tenant_user_id\n" + " sql: \"CONCAT({CUBE}.a, {CUBE}.b)\"\n type: string\n" + " - name: users\n" + " sql_table: a.b.users\n" + " dimensions:\n" + " - name: id\n sql: id\n type: number\n" + " primary_key: true\n" + ) + raw = _files(m=model.replace("{KEY}", "{CUBE}.tenant_user_id")) + ossie, back, _ = _roundtrip(raw) + assert model_of(ossie)["relationships"][0]["from_columns"] == ["tenant_user_id"] + assert parse_files(back) == parse_files(raw) + # The member form still parks: that one really does read an expression. + member = _files(m=model.replace("{KEY}", "{CUBE.tenant_user_id}")) + ossie2, _, _ = _roundtrip(member) + assert "relationships" not in model_of(ossie2) diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index bc5daa8c..935f168e 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -756,3 +756,75 @@ def test_a_generated_view_excludes_members_a_prefix_cannot_disambiguate(): ] assert any("excluded from the generated view" in i.detail for i in issues.of_type(IssueType.APPROXIMATED)) + + +@pytest.mark.parametrize("reference,expected", [ + # A metric is authored against *Ossie* names, which for a name needing sanitization + # is not the Cube name: dataset `Order Items` becomes cube `order_items`. + ('"ORDER ITEMS"."GROSS AMOUNT"', "{CUBE.gross_amount}"), + ("order_items.gross_amount", "{CUBE.gross_amount}"), +]) +def test_a_reference_may_use_either_the_ossie_or_the_cube_name(reference, expected): + ossie = ( + f"version: {OSSIE_VERSION}\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: Order Items\n" + " source: shop.public.oi\n" + " fields:\n" + " - name: Gross Amount\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: gross_raw * 2\n" + " datatype: Decimal\n" + " metrics:\n" + " - name: m\n expression:\n dialects:\n" + f" - dialect: ANSI_SQL\n expression: SUM({reference})\n" + ) + files, _ = convert_ossie_to_cube(ossie) + cube = parse(files["model/cubes/order_items.yml"])["cubes"][0] + assert cube["measures"] == [{"name": "m", "sql": expected, "type": "sum"}] + + +def test_a_quoted_reference_to_a_dropped_field_drops_its_metric_too(): + """The dropped-field check matched only unquoted references, so a metric over a + field that became no dimension survived with a dangling reference.""" + ds = ( + " - name: orders\n" + " source: shop.public.orders\n" + " primary_key:\n - id\n" + " fields:\n" + " - name: id\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: id\n" + " datatype: Integer\n" + " - name: legacy_amount\n expression:\n dialects:\n" + " - dialect: TABLEAU\n expression: amount\n" + " datatype: Decimal\n" + ) + files, issues = convert_ossie_to_cube( + _ossie(ds, metrics=_metric("t", 'SUM(orders."LEGACY_AMOUNT")'))) + assert "measures" not in _cubes(files)["orders"] + assert any("dropped with it" in i.detail + for i in issues.of_type(IssueType.NO_USABLE_DIALECT)) + + +def test_mapping_form_stashed_segments_are_reserved_and_checked(): + """Cube accepts `segments:` as a list *or* as a mapping keyed by name. Handling only + the list form meant a mapping iterated as bare strings and was skipped -- so a + generated part could take a restored segment's name, and the collision check missed + it too, emitting a model Cube rejects.""" + import json + + stash = {"_v": 1, + "cube_extras": {"segments": {"ratio_part_1": {"sql": "x"}}}} + ds = _ORDERS.replace( + " datatype: Decimal\n", + " datatype: Decimal\n" + " custom_extensions:\n" + " - vendor_name: CUBE\n" + f" data: '{json.dumps(stash)}'\n", 1) + files, _ = convert_ossie_to_cube(_ossie(ds, metrics=_metric( + "ratio", "SUM(orders.amount) / COUNT(DISTINCT orders.id)"))) + cube = _cubes(files)["orders"] + assert [m["name"] for m in cube["measures"]] == [ + "ratio_part_2", "ratio_part_3", "ratio"] + assert set(cube["segments"]) == {"ratio_part_1"} From 404c12a794ea577a893ddf3c678fef81debb90d9 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Tue, 4 Aug 2026 21:21:04 +0500 Subject: [PATCH 33/46] Fix six more review findings; two were regressions in last round's fixes All six reproduced first, and two of them were introduced by the previous round. [P1] Fan-out attribution stopped at the first *recognized* aggregate, so `SUM(orders.amount) + STDDEV(users.ltv)` reported only `orders` and strict mode accepted it. Attribution now walks the sqlglot tree instead of matching aggregate names in text, reading the table qualifiers inside each non-idempotent node -- so an aggregate with no Cube mapping is attributed like any other, and the text-scanning helper it replaced is gone. [P1] Join keys could still name columns that do not exist, two ways, both from last round's `_ALIAS_COLUMN_RE` shortcut: - a `case`/`switch` dimension has no `sql`, and "no sql means the same-named column" was applied to it -- there is no column of that name; - the alias in `{users}.region_id` was not checked against the *owning* cube, so another cube's column was reported as this dataset's. Only `CUBE`/`TABLE`/the owning cube's alias resolves now; the rest park. - Identifier resolution was incomplete in three ways. A name needing quotes could not be referenced at all (`"Order Items"` did not match the dataset `Order Items`), so lookups now key on the exact spelling as well as the normalized form -- a deliberate superset of the spec's column-matching table, which is about database columns rather than declared member names. A plain field referenced as `orders."AMOUNT"` emitted `{CUBE}."AMOUNT"`, forcing an exact uppercase match against a column named `amount`. And inline SQL (a split geo half) was looked up by the written token rather than the resolved cube, so it was never substituted through a sanitized dataset name. - A calculated measure referencing a rolling/multi-stage one aborted the entire import. The dependent has no static form either; it is parked alongside its dependency. - Collecting the members a generated view must disambiguate assumed every collection was a list, so mapping-form segments were skipped -- a segment named `users_id` and a prefixed `users.id` both reached the view under that name. - `SUM(DISTINCT x)` and `AVG(DISTINCT x)` were reported unsafe. DISTINCT collapses duplicates before the aggregate sees them, so any aggregate over a distinct set is duplication-invariant, not just COUNT. One test changed meaning: `orders."amount"` now resolves to the field `amount` rather than staying raw. That follows from the exact-spelling key, and the reasoning is recorded in the test. 524 tests with both gates, 499 with neither, 96% coverage. --- converters/cube/README.md | 14 ++- converters/cube/src/ossie_cube/_common.py | 68 ++++++++--- converters/cube/src/ossie_cube/cube_to_osi.py | 74 ++++++++---- converters/cube/src/ossie_cube/expressions.py | 61 ++++++---- converters/cube/src/ossie_cube/osi_to_cube.py | 22 +++- converters/cube/tests/test_edge_cases.py | 113 +++++++++++++++++- converters/cube/tests/test_osi_to_cube.py | 78 ++++++++++++ 7 files changed, 363 insertions(+), 67 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index 58bb930c..e1a57a6f 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -139,7 +139,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | relationship `custom_extensions` | cube `meta.ossie.join_extensions` | A Cube join entry takes only name/sql/relationship, so a relationship's foreign-vendor extensions ride on the declaring cube keyed by join target. | | — | `type: geo` dimension | An Ossie field holds one expression and a geo dimension has two, so it **splits** into `_latitude` / `_longitude` (`Float`). Reconstruction data rides on the latitude half. See [Geo dimensions](#geo-dimensions). | | relationship | `joins[]` on a cube | `many_to_one` on cube A -> `from: A`(many), `to: B`(one). `one_to_many` is flipped so Ossie's `from` is the many side; the declared side and type are stashed so export restores the original. | -| `from_columns` / `to_columns` | join `sql` | Only an AND-chain of equalities mapping to **physical columns** converts. `{CUBE}.user_id` is already one; `{CUBE.user_key}` names a *member*, so it resolves to the column that member reads (`user_id`). A member reading an expression (`CONCAT(...)`) has no column for Ossie to name, so the whole join is preserved verbatim in the stash rather than described wrongly — as is anything else (non-equi, range, literal, third cube). | +| `from_columns` / `to_columns` | join `sql` | Only an AND-chain of equalities mapping to **physical columns of that dataset** converts. `{CUBE}.user_id` is already one; `{CUBE.user_key}` names a *member*, so it resolves to the column that member reads (`user_id`), following a chain of member references to its end with cycle detection. Everything else preserves the whole join verbatim in the stash rather than describing it wrongly: a member reading an expression (`CONCAT(...)`), a `case`/`switch` dimension (which reads no column at all), an alias belonging to another cube (`{users}.region_id`), or a clause that is not a two-column equality. | | metric | `measures[]` on the cube its expression references | Import hoists cube-scoped measures to the model level, qualifying a colliding name as `__` and stashing the original name and owning cube. | | `SUM`/`AVG`/`MIN`/`MAX(x)` | `type: sum`/`avg`/`min`/`max` + `sql` | | | `COUNT(DISTINCT x)` | `type: count_distinct` | | @@ -227,10 +227,14 @@ the measure's Cube type, and not on the cube it is declared on. Both shortcuts w wrong: a calculated `type: number` measure's type says nothing about the aggregates inside it, and the cube a measure is declared on is not necessarily the one an aggregate inside it *reads*. `SUM(users.ltv) / SUM(orders.amount)` sits on `orders` while `users` -is the fanned-out side. The idempotent set is an **allowlist** (`MIN`, `MAX`, -`COUNT(DISTINCT …)`, `APPROX_COUNT_DISTINCT`) because the set of aggregate functions is -open-ended — listing the unsafe ones silently declared `STDDEV`, `MEDIAN` and -`ARRAY_AGG` safe. +is the fanned-out side. The idempotent set is an **allowlist** — `MIN`, `MAX`, `APPROX_COUNT_DISTINCT`, and +*any* aggregate over a `DISTINCT` set (which collapses duplicates before the aggregate +sees them, so `SUM(DISTINCT x)` is as safe as `COUNT(DISTINCT x)`) — because the set of +aggregate functions is open-ended, and listing the unsafe ones silently declared +`STDDEV`, `MEDIAN` and `ARRAY_AGG` safe. Attribution walks the parse tree rather than +matching aggregate names in text, so an aggregate with no Cube mapping is attributed like +any other: `SUM(orders.amount) + STDDEV(users.ltv)` reports `users`, which name-matching +missed once it had found the `SUM`. The TPC-DS fixture carries a real example: `store_productivity` is `SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), 0)`, and diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index 449088e0..0a5b0562 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -523,6 +523,23 @@ def sub_outside_quotes(sql, transform): for text, quoted in quoted_runs(sql)) +def _lookup_keys(written): + """The keys a written identifier may match: its exact spelling, then normalized.""" + text = str(written).strip() + if len(text) >= 2 and text.startswith('"') and text.endswith('"'): + inner = text[1:-1].replace('""', '"') + return (inner, normalize_identifier(text)) + return (text, normalize_identifier(text)) + + +def _first(mapping, keys): + """The first of `keys` present in `mapping`, or None.""" + for key in keys: + if key in mapping: + return mapping[key] + return None + + def normalize_identifier(name): """An Ossie identifier in the spec's *normalized* form, for matching. @@ -608,15 +625,24 @@ def quoted_char_mask(sql): def lookup_map(names): - """{normalized identifier: the name to emit}. + """{lookup key: the name to emit}, keyed both ways an identifier can be written. Accepts a set of names (each maps to itself) or a mapping of *any* accepted spelling to the canonical one -- which is what lets an Ossie name and the Cube name it sanitizes to both resolve to the Cube spelling. + + Each name gets two keys: its normalized form (upper-cased, matching an unquoted + reference) and its exact spelling (matching a quoted one). The second is needed + because a name containing a space or mixed case *cannot* be written unquoted at all -- + `"Order Items"` is the only way to reference a dataset of that name, so exact-quoted + has to resolve or the name is unusable. """ - if isinstance(names, dict): - return {normalize_identifier(k): v for k, v in names.items()} - return {normalize_identifier(n): n for n in names} + pairs = names.items() if isinstance(names, dict) else ((n, n) for n in names) + out = {} + for spelling, canonical in pairs: + out.setdefault(str(spelling), canonical) + out.setdefault(normalize_identifier(spelling), canonical) + return out def source_part_count(source): @@ -697,7 +723,7 @@ def requalify_self_refs(sql, cube_name): def ossie_expr_to_cube_sql(expr, own_cube, own_members=(), cube_names=(), - inline_sql=None, members_by_cube=None): + inline_sql=None, members_by_cube=None, own_lookup=None): """Rewrite an Ossie expression into Cube member-reference form. Only *dotted* `cube.name` references are rewritten -- a bare identifier stays @@ -738,6 +764,7 @@ def ossie_expr_to_cube_sql(expr, own_cube, own_members=(), cube_names=(), for cube, fields in (inline_sql or {}).items() } + own_columns = lookup_map(own_lookup or {}) foreign_members = { normalize_identifier(cube): lookup_map(names) for cube, names in (members_by_cube or {}).items() @@ -748,28 +775,41 @@ def repl(m): # Ossie regular identifiers are case-insensitive, so the reference is matched # in normalized form; what is *emitted* is the canonical Cube spelling, since # Cube's own member lookup is case-sensitive. - head_n, name_n = normalize_identifier(head), normalize_identifier(name) + head_n, name_n = _lookup_keys(head), _lookup_keys(name) # Resolve the dataset first, then decide which branch applies. Comparing the # written spelling against `own_cube` directly was wrong once the two could # differ: dataset `Order Items` becomes cube `order_items`, so a reference to # `"ORDER ITEMS"` took the cross-cube branch on its own cube. - target = known.get(head_n) - is_own = target == own_cube or (target is None and head_n == own_norm) - substitute = (inline_sql_by_norm.get(head_n) or {}).get(name_n) + target = _first(known, head_n) + is_own = target == own_cube or (target is None and own_norm in head_n) + # Keyed on the *resolved* cube, not the token as written: a sanitized dataset name + # differs from the Ossie one, and looking inline SQL up by the written token meant + # a split geo half referenced through the Ossie name was never substituted. + inline_for = inline_sql_by_norm.get( + normalize_identifier(target)) if target else None + if inline_for is None: + inline_for = _first(inline_sql_by_norm, head_n) or {} + substitute = _first(inline_for, name_n) if substitute is not None: # Already-Cube SQL, so it bypasses the escaping above; `{CUBE}` inside # it means `head`, which only stays true while head is the own cube. return (str(substitute) if is_own else requalify_self_refs(substitute, target or head)) if is_own: - return ("{CUBE." + members[name_n] + "}" if name_n in members - else "{CUBE}." + name) + member = _first(members, name_n) + if member is not None: + return "{CUBE." + member + "}" + # A plain member is the same thing either way, but the *column* still has a + # canonical spelling -- emitting `"AMOUNT"` as written would force an exact + # uppercase match in the database against a column named `amount`. + column = _first(own_columns, name_n) + return "{CUBE}." + (column if column is not None else name) if target is not None: # A cross-cube member needs the target cube's own spelling for the same # reason: `{users.ID}` does not resolve when the member is declared `id`. - member = (foreign_members.get(normalize_identifier(target)) - or {}).get(name_n, name) - return "{" + target + "." + member + "}" + member = _first(foreign_members.get(normalize_identifier(target)) or {}, + name_n) + return "{" + target + "." + (member if member is not None else name) + "}" # Not a dataset in this model -- a genuine schema-qualified table # reference or an unrelated dotted token. Leave it alone. return m.group(0) diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 96ed2f82..3352d9e6 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -51,6 +51,8 @@ dump_yaml, filtered_operand, is_simple_identifier, + lookup_map, + normalize_identifier, referenced_datasets, join_source, load_yaml, @@ -67,9 +69,8 @@ ) from .converter_issues import IssueLog, IssueType from .expressions import ( - has_non_idempotent_aggregate, has_top_level_operator, - unsafe_aggregate_spans, + unsafe_aggregate_datasets, ) # Cube keys the converter maps natively at the cube level; everything else is @@ -923,10 +924,12 @@ def _decompose_join_sql(sql, own_cube, target, what, cubes, issues): return pairs or None -# The alias-dot form: `{CUBE}.column` / `{TABLE}.column` / `{cube}.column`. Whatever -# follows the dot is a raw physical column, not a member. +# The alias-dot form: `{CUBE}.column`. Group 1 is the alias, group 2 the raw physical +# column. The alias has to be checked against the *owning* cube -- `{users}.region_id` +# matches the same shape but reads another cube's column, and treating it as this cube's +# turned a transitive join into a relationship naming a column this dataset lacks. _ALIAS_COLUMN_RE = re.compile( - r"^\$?\{\s*[A-Za-z_][A-Za-z0-9_]*\s*\}\s*\.\s*([A-Za-z_][A-Za-z0-9_]*)\s*$") + r"^\$?\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\s*\.\s*([A-Za-z_][A-Za-z0-9_]*)\s*$") _JOIN_SIDE_RE = re.compile( r"^\s*\$?\{\s*([^{}]*?)\s*\}\s*(?:\.\s*([A-Za-z_][A-Za-z0-9_]*))?\s*$") @@ -954,15 +957,24 @@ def _column_of(cubes, cname, member, seen=()): continue if snake(dim.get("type") or "") == "geo": return None + if dim.get("case") is not None or snake(dim.get("type") or "") == "switch": + # A `case` or `switch` dimension carries conditions or enumerated values and + # no sql at all, so "no sql means the same-named column" does not apply -- + # there is no column of that name to name. + return None sql = dim.get("sql") if sql is None: return member - if _ALIAS_COLUMN_RE.match(str(sql).strip()): + alias = _ALIAS_COLUMN_RE.match(str(sql).strip()) + if alias: # The explicit raw-column form (`{CUBE}.tenant_user_id`) names a column, full # stop -- even if a dimension of that name also exists. Deciding on the # *translated* text lost that distinction, since both forms flatten to the # same bare name, and the join was parked over a column that was right there. - return _ALIAS_COLUMN_RE.match(str(sql).strip()).group(1) + # Only this cube's own alias counts, though. + if alias.group(1) in ("CUBE", "TABLE", cname): + return alias.group(2) + return None translated, _ = cube_sql_to_ossie(sql, cname) translated = translated.strip() if not is_simple_identifier(translated): @@ -1041,6 +1053,14 @@ def _rebuild_join_sql(target, pairs): # --- measures ------------------------------------------------------------------- +class _NoStaticForm(Exception): + """A measure this one depends on has no static Ossie form, so nor does this one.""" + + def __init__(self, dependency): + super().__init__(dependency) + self.dependency = dependency + + class _MeasureResolver: """Computes the Ossie expression for a Cube measure. @@ -1118,7 +1138,15 @@ def expression(self, cname, mname, stack=()): if sql is None: raise ConversionError( f"measure '{scope}': type '{mtype}' requires 'sql'") - expr = self._translate(sql, cname, stack + (key,)) + try: + expr = self._translate(sql, cname, stack + (key,)) + except _NoStaticForm as missing: + self._issues.add( + IssueType.MULTI_STAGE_MEASURE_PARKED, scope, + f"references '{missing.dependency}', which is computed over a grain " + f"other than the query's and has no Ossie form; this measure has " + f"none either and is preserved in custom_extensions only") + return self._remember(key, None) return self._remember(key, filtered_operand(expr, filter_exprs)) if mtype == "count": if sql is None: @@ -1181,9 +1209,11 @@ def _inline(self, body, cname, stack): return None inner = self.expression(target_cube, target_name, stack) if inner is None: - raise ConversionError( - f"measure '{cname}': references '{target_cube}.{target_name}', " - f"which has no static Ossie form") + # The referenced measure has no static Ossie form (it is windowed), so + # neither does this one. Aborting the whole conversion over it was wrong: the + # dependent is parked alongside its dependency, the same as any other measure + # Ossie cannot express. + raise _NoStaticForm(f"{target_cube}.{target_name}") # A lone `SUM(x)` needs no parentheses; only a term with its own top-level # operators does. Keeping them off means a decomposed metric inlines back to # exactly the expression it was split from. @@ -1219,18 +1249,18 @@ def _fanout_unsafe_datasets(expr, own_cube, dataset_names): different datasets. An aggregate naming no dataset is read as being over the cube the measure is declared on. """ - spans = unsafe_aggregate_spans(expr) - if spans: - found = set() - for start, end in spans: - found |= (referenced_datasets(expr[start:end], dataset_names) - or {own_cube}) - return found - if has_non_idempotent_aggregate(expr): - # An aggregate the span scanner does not know (STDDEV, MEDIAN, ARRAY_AGG...): - # there is no span to attribute, so every dataset the expression reads counts. + analysed = unsafe_aggregate_datasets(expr) + if analysed is None: + # Unparseable, so nothing can be attributed: assume every dataset it names. return referenced_datasets(expr, dataset_names) or {own_cube} - return set() + tables, unqualified = analysed + canonical = lookup_map(dataset_names) + found = {canonical[normalize_identifier(table)] for table in tables + if normalize_identifier(table) in canonical} + if unqualified: + # An unsafe aggregate over an unqualified column reads the declaring cube. + found.add(own_cube) + return found def _windowing_key(measure): diff --git a/converters/cube/src/ossie_cube/expressions.py b/converters/cube/src/ossie_cube/expressions.py index 4307bac9..63dc2b8f 100644 --- a/converters/cube/src/ossie_cube/expressions.py +++ b/converters/cube/src/ossie_cube/expressions.py @@ -179,7 +179,11 @@ def is_idempotent_aggregate(node): """True if duplicating input rows cannot change this aggregate's value.""" if isinstance(node, _IDEMPOTENT_NODES): return True - return isinstance(node, exp.Count) and _counts_distinct(node) + # DISTINCT collapses duplicates before the aggregate sees them, so *any* aggregate + # over a distinct set is duplication-invariant -- `SUM(DISTINCT ltv)` as much as + # `COUNT(DISTINCT id)`. Honouring it only for COUNT rejected the others in strict + # mode over a value fan-out cannot change. + return _aggregates_distinct(node) def has_non_idempotent_aggregate(expr): @@ -201,31 +205,48 @@ def has_non_idempotent_aggregate(expr): for node in tree.walk()) -def unsafe_aggregate_spans(expr): - """(start, end) of every aggregate in `expr` that duplication would inflate. - - Offsets index the original string, so a caller can ask which datasets each unsafe - aggregate reads -- the measure's own cube is not necessarily the fanned-out one. - """ - return [(start, end) for start, end in _scan_aggregates(str(expr)) - if not _parses_to_idempotent(str(expr)[start:end])] - - -def _parses_to_idempotent(text): - node = parse(text) - return node is not None and is_idempotent_aggregate(node) - - -def _counts_distinct(node): - """True for `COUNT(DISTINCT x)`, which duplication does not affect.""" - inner = node.this - if isinstance(inner, exp.Distinct): +def _aggregates_distinct(node): + """True when this aggregate is applied to a DISTINCT set.""" + if isinstance(node.this, exp.Distinct): + return True + if node.args.get("distinct"): return True # sqlglot may hang the DISTINCT off the function's argument list instead. return any(isinstance(arg, exp.Distinct) for arg in (node.args.get("expressions") or [])) +def unsafe_aggregate_datasets(expr): + """Which datasets each non-idempotent aggregate in `expr` reads. + + Returns `(datasets, unqualified)` -- the dataset names appearing inside an unsafe + aggregate, and whether any unsafe aggregate named none (so it reads the cube the + measure is declared on). Returns None when the expression does not parse, leaving the + caller to be conservative. + + Walks the parse tree rather than matching aggregate names in the text, so an + aggregate this converter has no Cube mapping for -- STDDEV, MEDIAN, ARRAY_AGG -- is + attributed like any other. Scanning for known names meant one recognized aggregate + was enough to stop the search, and an unrecognized one elsewhere in the same + expression went unattributed: `SUM(orders.amount) + STDDEV(users.ltv)` reported only + `orders`. + """ + tree = parse(expr) + if tree is None: + return None + datasets, unqualified = set(), False + for node in tree.walk(): + if not isinstance(node, exp.AggFunc) or is_idempotent_aggregate(node): + continue + tables = {column.table for column in node.find_all(exp.Column) + if column.table} + if tables: + datasets |= tables + else: + unqualified = True + return datasets, unqualified + + def has_top_level_operator(expr): """True if `expr` is not a single self-contained term. diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 2ae058e7..a26fcd66 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -213,7 +213,8 @@ def _convert_model(model, dialect, base_cube, issues): # generated view has to disambiguate against. emitted_members[cname] = [ m["name"] for key in ("dimensions", "measures", "segments") - for m in (cube.get(key) or []) if isinstance(m, dict) and m.get("name")] + for m in _member_entries(cube.get(key)) + if isinstance(m, dict) and m.get("name")] for vpath, views in _build_views(model, model_stash, cube_names, relationships, datasets, base_cube, emitted_members, @@ -708,6 +709,19 @@ def _dimension_type(field, stash, scope, issues): # --- joins ---------------------------------------------------------------------- +def _member_entries(collection): + """A member collection as a list of entries, from either Cube spelling. + + `dimensions`/`measures`/`segments` may be a list of entries carrying `name` or a + mapping keyed by name. Assuming a list meant mapping-form segments were skipped when + collecting the members a generated view has to disambiguate -- so a segment named + `users_id` and a prefixed `users.id` both reached the view as `users_id`. + """ + if isinstance(collection, dict): + return [{"name": name} for name in collection] + return list(collection or []) + + def _stashed_segment_names(stash): """Names of the segments a stash carries, in either Cube spelling. @@ -1048,7 +1062,8 @@ def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_ measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( inner, target, members, sanitized, inline_sql=inline_sql_by_cube, - members_by_cube=member_lookup) + members_by_cube=member_lookup, + own_lookup=(member_lookup or {}).get(target)) measure["type"] = agg return measure @@ -1056,7 +1071,8 @@ def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_ # these as a calculated measure whose sql carries the aggregation. measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( expr, target, members, sanitized, inline_sql=inline_sql_by_cube, - members_by_cube=member_lookup) + members_by_cube=member_lookup, + own_lookup=(member_lookup or {}).get(target)) measure["type"] = "number" return measure diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index e710e387..ac11fc72 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -2012,12 +2012,20 @@ def test_a_cross_cube_member_gets_the_target_cubes_own_spelling(): @pytest.mark.parametrize("reference,expected", [ # An ANSI double-quoted identifier is a *name*, not a string literal, so it is - # parsed. Per core-spec, a quoted identifier is force-matched to the normalized - # (upper) case: `"AMOUNT"` is the field `amount`, `"Amount"` is not. + # parsed. A quoted reference resolves against the normalized (upper) form, per + # core-spec, *and* against the name's exact spelling. + # + # The exact-spelling key is a deliberate superset of the spec's column-matching + # table, which says `"id"` does not match a column created as `id`. That rule is + # about physical database columns, folded by the database; an Ossie member name is + # whatever the model declares. And a name containing a space or mixed case cannot be + # written unquoted at all -- `"Order Items"` is the only way to reference a dataset + # of that name -- so without it such a name would be unreferenceable. ('orders."AMOUNT"', "SUM({CUBE.amount})"), ('"ORDERS"."AMOUNT"', "SUM({CUBE.amount})"), + ('orders."amount"', "SUM({CUBE.amount})"), + # Neither the exact spelling nor the normalized form, so it stays a raw column. ('orders."Amount"', 'SUM({CUBE}."Amount")'), - ('orders."amount"', 'SUM({CUBE}."amount")'), ]) def test_quoted_identifiers_are_parsed_not_skipped(reference, expected): """The whole double-quoted region used to be treated as opaque -- the same handling @@ -2233,3 +2241,102 @@ def test_an_explicit_raw_column_wins_over_a_dimension_of_that_name(): member = _files(m=model.replace("{KEY}", "{CUBE.tenant_user_id}")) ossie2, _, _ = _roundtrip(member) assert "relationships" not in model_of(ossie2) + + +# --- review round seven ---------------------------------------------------------- + +@pytest.mark.parametrize("sql,flagged", [ + # One recognized aggregate used to stop the search, leaving an unrecognized one + # elsewhere in the same expression unattributed. + ("SUM({CUBE}.amount) + STDDEV({users}.ltv)", True), + ("STDDEV({CUBE}.amount) + SUM({users}.ltv)", True), + ("MEDIAN({users}.ltv) - MIN({CUBE}.amount)", True), + # Only the safe cube is read unsafely. + ("STDDEV({CUBE}.amount) + MAX({users}.ltv)", False), + # DISTINCT collapses duplicates before the aggregate, so fan-out cannot change it. + ("SUM(DISTINCT {users}.ltv)", False), + ("AVG(DISTINCT {users}.ltv) / 2", False), +]) +def test_every_aggregate_is_attributed_including_unrecognized_ones(sql, flagged): + files = _files(m=_TWO_CUBE_CALC.replace("{SQL}", sql)) + _, issues = convert_cube_to_ossie(files) + assert bool(issues.of_type(IssueType.FANOUT_UNSAFE_METRIC)) is flagged + if flagged: + with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): + convert_cube_to_ossie(files, strict_fanout=True) + + +def _join_key_model(key, dims): + return _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: a.b.orders\n" + " joins:\n" + " - name: users\n" + f" sql: \"{{CUBE.{key}}} = {{users.id}}\"\n" + " relationship: many_to_one\n" + " dimensions:\n" + dims + + " - name: users\n" + " sql_table: a.b.users\n" + " dimensions:\n" + " - name: id\n sql: id\n type: number\n" + " primary_key: true\n" + " - name: region_id\n sql: region_id\n type: number\n" + )) + + +@pytest.mark.parametrize("key,dims,expected", [ + # A `case` dimension has conditions and no sql, so "no sql means the same-named + # column" does not apply -- there is no column of that name. + ("tier", + " - name: tier\n type: string\n case:\n when:\n" + " - sql: \"{CUBE}.x > 1\"\n label: hi\n", None), + # A `switch` dimension enumerates values and reads nothing. + ("tier", + " - name: tier\n type: switch\n values:\n - a\n", None), + # `{users}.region_id` reads *another* cube's column, so it is not this dataset's. + ("region_key", + " - name: region_key\n sql: \"{users}.region_id\"\n" + " type: number\n", None), + # This cube's own alias is a genuine raw column and still resolves. + ("k", + " - name: k\n sql: \"{CUBE}.user_id\"\n type: number\n", + ["user_id"]), +]) +def test_only_this_cubes_own_columns_resolve_a_join_key(key, dims, expected): + """Ossie relationship columns are physical columns of the dataset. Anything that is + not one has to park the join rather than name a column that does not exist.""" + files = _join_key_model(key, dims) + ossie, back, _ = _roundtrip(files) + rels = model_of(ossie).get("relationships") + if expected is None: + assert rels is None + else: + assert rels[0]["from_columns"] == expected + assert parse_files(back) == parse_files(files) + + +def test_a_measure_depending_on_a_windowed_one_is_parked_too(): + """A rolling/multi-stage measure has no static Ossie form, and neither does anything + referencing it. Raising aborted the whole import over one measure; both are parked + and restored together.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: a.b.orders\n" + " dimensions:\n" + " - name: id\n sql: id\n type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: rolling\n sql: amount\n type: sum\n" + " rolling_window:\n trailing: 3 month\n" + " - name: rolling_ratio\n sql: \"{rolling} / 100\"\n" + " type: number\n")) + ossie, back, issues = _roundtrip(files) + assert "metrics" not in model_of(ossie) + parked = {i.element_name for i in issues.of_type( + IssueType.MULTI_STAGE_MEASURE_PARKED)} + assert parked == {"orders.rolling", "orders.rolling_ratio"} + # Both come back, in their original positions. + cube = parse(back["model/cubes/m.yml"])["cubes"][0] + assert [m["name"] for m in cube["measures"]] == ["rolling", "rolling_ratio"] diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 935f168e..d10804c5 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -828,3 +828,81 @@ def test_mapping_form_stashed_segments_are_reserved_and_checked(): assert [m["name"] for m in cube["measures"]] == [ "ratio_part_2", "ratio_part_3", "ratio"] assert set(cube["segments"]) == {"ratio_part_1"} + + +def test_a_name_that_must_be_quoted_resolves_when_quoted_exactly(): + """`Order Items` cannot be written unquoted at all, so `"Order Items"` is the only + way to reference it -- exact-quoted has to resolve or the name is unusable.""" + ossie = ( + f"version: {OSSIE_VERSION}\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: Order Items\n" + " source: shop.public.oi\n" + " fields:\n" + " - name: Gross Amount\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: gross_raw * 2\n" + " datatype: Decimal\n" + " metrics:\n" + " - name: m\n expression:\n dialects:\n" + ' - dialect: ANSI_SQL\n expression: SUM("Order Items"."Gross Amount")\n' + ) + files, _ = convert_ossie_to_cube(ossie) + assert parse(files["model/cubes/order_items.yml"])["cubes"][0][ + "measures"] == [{"name": "m", "sql": "{CUBE.gross_amount}", "type": "sum"}] + + +def test_a_plain_field_reference_is_canonicalized_to_its_column(): + """A plain member is the same thing either way, but the column has a canonical + spelling: emitting `"AMOUNT"` as written would force an exact uppercase match in the + database against a column named `amount`.""" + ds = ( + " - name: orders\n" + " source: shop.public.orders\n" + " fields:\n" + " - name: amount\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: amount\n" + " datatype: Decimal\n" + ) + files, _ = convert_ossie_to_cube( + _ossie(ds, metrics=_metric("m", 'SUM(orders."AMOUNT")'))) + assert _cubes(files)["orders"]["measures"][0]["sql"] == "{CUBE}.amount" + + +def test_a_mapping_form_segment_is_counted_when_disambiguating_a_view(): + """Collecting the members a generated view must disambiguate assumed every collection + was a list, so a mapping-form segment was skipped -- and a segment named `users_id` + plus a prefixed `users.id` both reached the view under that one name.""" + import json + + stash = {"_v": 1, "cube_extras": {"segments": {"users_id": {"sql": "x"}}}} + datasets = ( + " - name: orders\n" + " source: shop.public.orders\n" + " primary_key:\n - id\n" + " fields:\n" + " - name: id\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: id\n" + " datatype: Integer\n" + " - name: user_id\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: user_id\n" + " datatype: Integer\n" + " custom_extensions:\n" + " - vendor_name: CUBE\n" + f" data: '{json.dumps(stash)}'\n" + " - name: users\n" + " source: shop.public.users\n" + " primary_key:\n - id\n" + " fields:\n" + " - name: id\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: id\n" + " datatype: Integer\n" + ) + rel = (" relationships:\n" + " - name: r\n from: orders\n to: users\n" + " from_columns: [user_id]\n to_columns: [id]\n") + files, issues = convert_ossie_to_cube(_ossie(datasets, rel)) + entry = parse(files["model/views/shop.yml"])["views"][0]["cubes"][1] + assert entry["excludes"] == ["id"] + assert issues.of_type(IssueType.APPROXIMATED) From 5eb67db2ec14e5d7d072fe56c5e2151a6f54e5b6 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Tue, 4 Aug 2026 21:44:41 +0500 Subject: [PATCH 34/46] Refactor: delete dead code, collapse the seven per-cube fact maps into one No behaviour change; 524 tests and both gates green before and after. Dead, found by counting references rather than by eye: - `FANOUT_SAFE_AGGS` -- unreferenced since fan-out moved to the resolved expression. - `FANOUT_UNSAFE_AGGS` -- a stale import, unused since the same change. - `DOTTED_REF_RE` -- superseded by the quoted-aware pattern, still imported by both converters and used by neither. The survivor takes the plain name back, so there is one notion of "a dotted reference" instead of two that had to be kept in step. - `has_non_idempotent_aggregate` -- unused by the converter since attribution started walking the parse tree, but still tested. Deleted, and its tests re-pointed at `unsafe_aggregate_datasets`, which is what production calls. The real smell: seven dicts keyed by cube name -- dimension names, reference members, all members, the name lookup, dropped fields, inline SQL, primary keys -- built in one loop and then threaded through the build functions one parameter at a time. `_build_measures` took thirteen parameters, `_decompose_measure` thirteen, `_build_cube` ten. They are seven answers about the same cube, so they now travel together as `_CubePlan`, and the four lookups every measure rewrite repeats are assembled once in `_to_cube_sql`. Worst parameter count in osi_to_cube.py: 13 -> 9. `_measure_from_expression` 9 -> 6, `_build_dimensions` 7 -> 4, `_build_cube` 10 -> 7. --- converters/cube/src/ossie_cube/_common.py | 31 +-- converters/cube/src/ossie_cube/cube_to_osi.py | 2 - converters/cube/src/ossie_cube/expressions.py | 19 -- converters/cube/src/ossie_cube/osi_to_cube.py | 215 ++++++++++-------- converters/cube/tests/_roundtrip_helpers.py | 4 +- converters/cube/tests/test_edge_cases.py | 23 +- 6 files changed, 144 insertions(+), 150 deletions(-) diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index 0a5b0562..16fc3e6b 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -55,19 +55,13 @@ # A bare SQL identifier (single column reference), e.g. `c_name`. _IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") -# `cube.member` -- a dotted reference an Ossie expression uses to point into a -# dataset. Guarded so `a.b.c` and `1.5` do not match. -DOTTED_REF_RE = re.compile( - r"(?). -FANOUT_SAFE_AGGS = frozenset({ - "count_distinct", "count_distinct_approx", "min", "max", -}) - -# Aggregates that over-count under row multiplication. Cube corrects for these at -# query time by deduplicating on the primary key; an Ossie expression cannot. -FANOUT_UNSAFE_AGGS = frozenset({"sum", "avg"}) - # The Ossie result datatype Cube itself declares for each aggregate. Only the # count family is listed: those are exactly the aggregates whose result type does # not depend on the operand. diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 3352d9e6..0c3c3eff 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -41,8 +41,6 @@ DIALECT_ANSI, DATATYPE_TO_DIM_TYPE, DIM_TYPE_TO_DATATYPE, - DOTTED_REF_RE, - FANOUT_UNSAFE_AGGS, JINJA_RE, OSSIE_VERSION, ConversionError, diff --git a/converters/cube/src/ossie_cube/expressions.py b/converters/cube/src/ossie_cube/expressions.py index 63dc2b8f..02643055 100644 --- a/converters/cube/src/ossie_cube/expressions.py +++ b/converters/cube/src/ossie_cube/expressions.py @@ -186,25 +186,6 @@ def is_idempotent_aggregate(node): return _aggregates_distinct(node) -def has_non_idempotent_aggregate(expr): - """True if `expr` contains an aggregate that over-counts duplicated rows. - - A Cube *calculated* measure (`type: number`) is classified by its outer type, which - says nothing about the aggregates inside it -- `SUM({CUBE}.ltv) / 100` is a `number` - measure whose value is still a sum. So fan-out safety has to be judged on the - resolved expression, not on the measure type. - - Conservative when sqlglot cannot parse the expression: an unparseable expression is - reported as unsafe rather than assumed safe, since the whole point is not to emit a - silently-inflated number. - """ - tree = parse(expr) - if tree is None: - return True - return any(isinstance(node, exp.AggFunc) and not is_idempotent_aggregate(node) - for node in tree.walk()) - - def _aggregates_distinct(node): """True when this aggregate is applied to a DISTINCT set.""" if isinstance(node.this, exp.Distinct): diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index a26fcd66..6329da69 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -31,13 +31,13 @@ ossie-cube export -i model.yaml -o model/ [--dialect SNOWFLAKE] [--base-cube orders] """ +import dataclasses import re from collections import deque from ._common import ( AGG_TO_RESULT_DATATYPE, DATATYPE_TO_DIM_TYPE, - DOTTED_REF_RE, DEFAULT_DATATYPE_FOR_CUBE_TYPE, OSSIE_FUNC_TO_AGG, OSSIE_VERSION, @@ -56,7 +56,7 @@ primary_key_operand, read_stash, referenced_datasets, - QUOTED_DOTTED_REF_RE, + DOTTED_REF_RE, lookup_map, normalize_identifier, quoted_runs, @@ -144,54 +144,18 @@ def _convert_model(model, dialect, base_cube, issues): model_stash = read_stash(model) - # Per-cube facts the join and measure stages need. - # Field -> dimension names are resolved once here and reused by every stage. - # Sanitizing per stage would let a collision go undetected in one place and be - # rejected in another, and would disagree about which members a cube actually - # has -- which decides `{CUBE.member}` vs `{CUBE}.column` and where a measure - # lands. - dim_names_by_cube = {} - members_by_cube = {} - all_members_by_cube = {} - member_lookup_by_cube = {} - dropped_by_cube = {} - inline_sql_by_cube = {} - pk_by_cube = {} - for ds_name, ds in datasets.items(): - cname = cube_names[ds_name] - dim_names_by_cube[cname], inline_sql_by_cube[cname] = ( - _resolve_dimension_names(ds, f"Model '{name}': dataset '{ds_name}'")) - # Not every member: only those the `{CUBE.member}` form is required for. - members_by_cube[cname] = _reference_members( - ds, dim_names_by_cube[cname], dialect) - # Every dimension name the cube will carry, and the fields that will not - # become one. A generated measure name has to avoid the first, and a metric - # over the second cannot be rendered at all. - # Every member the cube will carry, stashed ones included: a `switch` dimension - # or a multi-stage measure is restored verbatim on export, and a generated part - # name colliding with one fails the conversion at the very end. - ds_stash = read_stash(ds) - member_lookup_by_cube[cname] = dict(dim_names_by_cube[cname]) - member_lookup_by_cube[cname].update( - {dname: dname for dname in dim_names_by_cube[cname].values()}) - all_members_by_cube[cname] = ( - set(dim_names_by_cube[cname].values()) - | {str(item["dimension"]["name"]) - for item in (ds_stash.get("extra_dimensions") or []) - if (item.get("dimension") or {}).get("name")} - | {str(item["measure"]["name"]) - for item in (ds_stash.get("extra_measures") or []) - if (item.get("measure") or {}).get("name")} - | _stashed_segment_names(ds_stash)) - dropped_by_cube[cname] = _undialected_fields(ds, dialect) - pk_by_cube[cname] = [str(c) for c in (ds.get("primary_key") or [])] + # What every later stage needs to know about each cube, worked out once. Resolving + # any of it per stage would let the stages disagree -- about which names collide, + # about which members exist, and so about `{CUBE.member}` vs `{CUBE}.column` and + # where a measure lands. + plan = {cube_names[ds_name]: _CubePlan.of(ds, cube_names[ds_name], dialect, + f"Model '{name}': dataset '{ds_name}'") + for ds_name, ds in datasets.items()} joins_by_cube, join_parked_by_cube = _build_joins( relationships, cube_names, issues) measures_by_cube = _build_measures( - model, cube_names, members_by_cube, all_members_by_cube, - member_lookup_by_cube, dropped_by_cube, inline_sql_by_cube, pk_by_cube, - datasets, relationships, base_cube, dialect, issues) + model, cube_names, plan, datasets, relationships, base_cube, dialect, issues) # Cubes, grouped by the file they belong in: several datasets can share one # stashed original path, in which case they go back into the same file. @@ -200,9 +164,8 @@ def _convert_model(model, dialect, base_cube, issues): emitted_members = {} for ds_name, ds in datasets.items(): cname = cube_names[ds_name] - cube = _build_cube(ds, cname, dim_names_by_cube[cname], - inline_sql_by_cube[cname], members_by_cube[cname], - joins_by_cube.get(cname), measures_by_cube.get(cname), + cube = _build_cube(ds, plan[cname], joins_by_cube.get(cname), + measures_by_cube.get(cname), join_parked_by_cube.get(cname), dialect, issues) stashed = stashed_paths.get(cname) path = (safe_relative_path(stashed, f"cube '{cname}'") if stashed @@ -305,8 +268,8 @@ def _ordered(obj, order): # --- cubes ---------------------------------------------------------------------- -def _build_cube(ds, cname, dim_names, inline_sql, ref_members, joins, measures, - join_extensions, dialect, issues): +def _build_cube(ds, plan, joins, measures, join_extensions, dialect, issues): + cname = plan.cname ds_name = ds["name"] scope = f"dataset '{ds_name}'" stash = read_stash(ds) @@ -342,7 +305,7 @@ def _build_cube(ds, cname, dim_names, inline_sql, ref_members, joins, measures, "so this cube-level value has no effect in Cube") dimensions, by_name_scalar, by_column, by_name_computed = _build_dimensions( - ds, cname, dim_names, inline_sql, ref_members, dialect, issues) + ds, plan, dialect, issues) # Resolve each `primary_key` entry to the dimension Cube should mark. A # dimension only qualifies when it is *scalar* -- backed by a single source # column -- because `primary_key: true` in Cube declares that dimension's own @@ -441,6 +404,62 @@ def _reject_member_collisions(cname, dimensions, measures, cube_extras, issues): seen[key] = (kind, member["name"]) +@dataclasses.dataclass(frozen=True) +class _CubePlan: + """Everything the later stages need to know about one cube. + + These seven facts were seven parallel dicts keyed by cube name, threaded through the + build functions one parameter at a time -- `_build_measures` took thirteen. They are + all answers about the same cube, so they travel together. + + `names` Ossie field name -> the Cube dimension name it becomes. + `inline_sql` Ossie field name -> Cube SQL to substitute for it (a split geo half, + which exists only in Ossie and so has no member to reference). + `references` the members that must be addressed as `{CUBE.member}`, keyed by every + accepted spelling. + `lookup` every field/dimension name, keyed by every accepted spelling, for + canonicalizing a reference. + `members` every member name the cube will carry, stashed ones included -- what a + generated name has to avoid. + `dropped` fields with no expression in a usable dialect, which become no dimension. + `primary_key` the dataset's declared key columns. + """ + + cname: str + names: dict + inline_sql: dict + references: dict + lookup: dict + members: frozenset + dropped: frozenset + primary_key: tuple + + @classmethod + def of(cls, ds, cname, dialect, scope): + names, inline_sql = _resolve_dimension_names(ds, scope) + stash = read_stash(ds) + lookup = dict(names) + lookup.update({dname: dname for dname in names.values()}) + return cls( + cname=cname, + names=names, + inline_sql=inline_sql, + references=_reference_members(ds, names, dialect), + lookup=lookup, + members=frozenset( + set(names.values()) + | {str(item["dimension"]["name"]) + for item in (stash.get("extra_dimensions") or []) + if (item.get("dimension") or {}).get("name")} + | {str(item["measure"]["name"]) + for item in (stash.get("extra_measures") or []) + if (item.get("measure") or {}).get("name")} + | _stashed_segment_names(stash)), + dropped=frozenset(_undialected_fields(ds, dialect)), + primary_key=tuple(str(c) for c in (ds.get("primary_key") or [])), + ) + + def _reference_members(ds, dim_names, dialect): """Members that must be addressed as `{CUBE.member}` rather than `{CUBE}.column`. @@ -538,8 +557,7 @@ def _resolve_dimension_names(ds, scope): return names, inline_sql -def _build_dimensions(ds, cname, dim_names, inline_sql, ref_members, dialect, - issues): +def _build_dimensions(ds, plan, dialect, issues): """Build a cube's dimensions from an Ossie dataset's fields. Returns (dimensions, by_name_scalar, by_column, by_name_computed). @@ -552,10 +570,12 @@ def _build_dimensions(ds, cname, dim_names, inline_sql, ref_members, dialect, writes dimension *names* there and a computed key has no column to name instead. Fields carrying a `geo` stash are re-merged into the single Cube dimension they were split from. - Dimension names come from `dim_names` (see `_resolve_dimension_names`) rather + Dimension names come from `plan.names` (see `_resolve_dimension_names`) rather than being sanitized again here. """ ds_name = ds["name"] + cname = plan.cname + dim_names = plan.names by_name_scalar, by_column, by_name_computed = {}, {}, {} # Built by target dimension name rather than by list position: a geo dimension # is assembled from two fields that may appear in either order and need not be @@ -589,7 +609,8 @@ def _build_dimensions(ds, cname, dim_names, inline_sql, ref_members, dialect, dim["sql"] = stash["sql"] else: dim["sql"] = ossie_expr_to_cube_sql( - expr, cname, ref_members, (), inline_sql={cname: inline_sql}) + expr, cname, plan.references, (), + inline_sql={cname: plan.inline_sql}) if stash.get("case") is not None: # A `case` dimension carries its conditions instead of `sql`, and Cube # rejects a dimension declaring both ("dimensions.size does not match any @@ -823,9 +844,8 @@ def _build_joins(relationships, cube_names, issues): # --- measures ------------------------------------------------------------------- -def _build_measures(model, cube_names, members_by_cube, all_members_by_cube, - member_lookup_by_cube, dropped_by_cube, inline_sql_by_cube, - pk_by_cube, datasets, relationships, base_cube, dialect, issues): +def _build_measures(model, cube_names, plan, datasets, relationships, base_cube, + dialect, issues): """Group Ossie metrics into per-cube `measures` lists.""" name = model.get("name", "") # A metric is authored against Ossie names, and a name needing sanitization has a @@ -886,8 +906,7 @@ def resolve_base(): "no ANSI_SQL or preferred-dialect expression; metric dropped") continue - missing = _references_a_dropped_field(expr, sanitized, cube_names, - dropped_by_cube) + missing = _references_a_dropped_field(expr, sanitized, cube_names, plan) if missing: # The field has no expression in a usable dialect, so Cube gets no # dimension for it -- and a measure referencing one it did not get is a @@ -920,27 +939,24 @@ def resolve_base(): # applies its row-multiplication correction per aggregate instead of # seeing one opaque expression -- see _decompose_measure. public_sql = _decompose_measure( - expr, spans, mname, target, measures_by_cube, members_by_cube, - all_members_by_cube, member_lookup_by_cube, inline_sql_by_cube, - pk_by_cube, sanitized, name, reserved) + expr, spans, mname, target, measures_by_cube, plan, sanitized, + name, reserved) measure = {"name": mname, "sql": public_sql, "type": "number"} else: measure = _measure_from_expression( - expr, target, mname, stash, members_by_cube.get(target, set()), - inline_sql_by_cube, pk_by_cube.get(target, []), sanitized, - member_lookup_by_cube) + expr, target, mname, stash, plan, sanitized) _apply_measure_metadata(metric, measure, stash) _place(measures_by_cube, target, measure, name) return measures_by_cube -def _references_a_dropped_field(expr, sanitized, cube_names, dropped_by_cube): +def _references_a_dropped_field(expr, sanitized, cube_names, plan): """`dataset.field` references in `expr` naming a field that becomes no dimension.""" by_cube_name = {cname: ds_name for ds_name, cname in cube_names.items()} canonical = lookup_map(sanitized) dropped_norm = { cname: lookup_map(fields) - for cname, fields in dropped_by_cube.items() + for cname, fields in ((c, p.dropped) for c, p in plan.items()) } missing = set() for text, quoted in quoted_runs(expr): @@ -949,7 +965,7 @@ def _references_a_dropped_field(expr, sanitized, cube_names, dropped_by_cube): # The quoted-reference parser, so `orders."LEGACY_AMOUNT"` is seen as well: # matching only unquoted references let a metric over a dropped field survive # with a reference to a dimension that was never created. - for match in QUOTED_DOTTED_REF_RE.finditer(text): + for match in DOTTED_REF_RE.finditer(text): head, field = split_dotted_ref(match.group(0)) cname = canonical.get(normalize_identifier(head)) if cname is None: @@ -960,10 +976,8 @@ def _references_a_dropped_field(expr, sanitized, cube_names, dropped_by_cube): return missing -def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, - members_by_cube, all_members_by_cube, member_lookup, - inline_sql_by_cube, pk_by_cube, sanitized, model_name, - reserved): +def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, plan, + sanitized, model_name, reserved): """Emit one `public: false` measure per aggregate; return the sql referencing them. Cube corrects for row multiplication per measure, keyed on the cube that measure @@ -979,7 +993,7 @@ def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, # is unique across dimensions and measures alike -- so the check is over both, and # over every cube rather than the one part it happens to land on. taken = {m["name"].lower() for ms in measures_by_cube.values() for m in ms} - taken |= {n.lower() for ns in all_members_by_cube.values() for n in ns} + taken |= {n.lower() for p in plan.values() for n in p.members} # Names later metrics will claim, so allocation does not depend on metric order. taken |= {n for n in reserved if n != mname.lower()} out, cursor, index = [], 0, 0 @@ -997,25 +1011,20 @@ def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, taken.add(part_name.lower()) part = _measure_from_expression( - piece, part_target, part_name, {}, - members_by_cube.get(part_target, set()), inline_sql_by_cube, - pk_by_cube.get(part_target, []), sanitized, member_lookup) + piece, part_target, part_name, {}, plan, sanitized) part["public"] = False part["meta"] = {"ossie": {"part_of": mname}} _place(measures_by_cube, part_target, part, model_name) - out.append(ossie_expr_to_cube_sql( - expr[cursor:start], fallback, members_by_cube.get(fallback, set()), - sanitized, inline_sql=inline_sql_by_cube, - members_by_cube=member_lookup)) + out.append(_to_cube_sql( + expr[cursor:start], fallback, plan, sanitized)) # `{CUBE.x}` for a part on the same cube as the public measure: an explicit # name pins the reference to this cube and breaks if it is extended. qualifier = "CUBE" if part_target == fallback else part_target out.append("{" + f"{qualifier}.{part_name}" + "}") cursor = end - out.append(ossie_expr_to_cube_sql( - expr[cursor:], fallback, members_by_cube.get(fallback, set()), sanitized, - inline_sql=inline_sql_by_cube, members_by_cube=member_lookup)) + out.append(_to_cube_sql( + expr[cursor:], fallback, plan, sanitized)) return "".join(out) @@ -1028,8 +1037,24 @@ def _place(measures_by_cube, target, measure, model_name): bucket.append(measure) -def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_cube, - primary_key, sanitized, member_lookup=None): +def _to_cube_sql(text, target, plan, sanitized): + """One Ossie expression fragment as Cube SQL, resolved against the whole plan. + + Every measure rewrite needs the same four lookups -- the target's reference members, + its full name lookup, every cube's members, and the inline SQL of split geo halves -- + so they are assembled here rather than spelled out at each call site. + """ + own = plan.get(target) + return ossie_expr_to_cube_sql( + text, target, + own.references if own else (), + sanitized, + inline_sql={c: p.inline_sql for c, p in plan.items()}, + members_by_cube={c: p.lookup for c, p in plan.items()}, + own_lookup=own.lookup if own else None) + + +def _measure_from_expression(expr, target, mname, stash, plan, sanitized): """Turn an Ossie metric expression back into a structured Cube measure. `COUNT(DISTINCT )` is Cube's bare `type: count` -- @@ -1045,7 +1070,8 @@ def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_ distinct = _DISTINCT_RE.match(inner) if func == "COUNT" and distinct: inner = distinct.group(1).strip() - if primary_key and inner == primary_key_operand(target, primary_key): + key = list((plan.get(target).primary_key if target in plan else ())) + if key and inner == primary_key_operand(target, key): measure["type"] = "count" return measure func = "COUNT_DISTINCT" @@ -1059,20 +1085,15 @@ def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_ if not (func == "COUNT" and inner == "*"): agg = OSSIE_FUNC_TO_AGG.get(func) or ("count" if func == "COUNT" else None) if agg is not None: - measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( - inner, target, members, sanitized, - inline_sql=inline_sql_by_cube, - members_by_cube=member_lookup, - own_lookup=(member_lookup or {}).get(target)) + measure["sql"] = stash.get("sql") or _to_cube_sql( + inner, target, plan, sanitized) measure["type"] = agg return measure # A ratio, a window expression, or a multi-dataset aggregate: Cube expresses # these as a calculated measure whose sql carries the aggregation. - measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( - expr, target, members, sanitized, inline_sql=inline_sql_by_cube, - members_by_cube=member_lookup, - own_lookup=(member_lookup or {}).get(target)) + measure["sql"] = stash.get("sql") or _to_cube_sql( + expr, target, plan, sanitized) measure["type"] = "number" return measure diff --git a/converters/cube/tests/_roundtrip_helpers.py b/converters/cube/tests/_roundtrip_helpers.py index 3afefdd0..bea1fa2f 100644 --- a/converters/cube/tests/_roundtrip_helpers.py +++ b/converters/cube/tests/_roundtrip_helpers.py @@ -428,14 +428,14 @@ def _normalize_refs(expression): spelling -- so comparing raw text would report intended behaviour as a change. The normalization rules themselves are pinned by targeted tests, not by this one. """ - from ossie_cube._common import (QUOTED_DOTTED_REF_RE, normalize_identifier, + from ossie_cube._common import (DOTTED_REF_RE, normalize_identifier, split_dotted_ref) def repl(match): head, name = split_dotted_ref(match.group(0)) return f"{normalize_identifier(head)}.{normalize_identifier(name)}" - return QUOTED_DOTTED_REF_RE.sub(repl, expression) + return DOTTED_REF_RE.sub(repl, expression) def check_ossie_model(ossie_yaml): diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index ac11fc72..be6a6c02 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -1429,7 +1429,7 @@ def test_several_semantic_models_convert_the_first_with_an_issue(): ("'x' = a", [("'x'", True), (" = a", False)]), ("'it''s'", [("'it'", True), ("'s'", True)]), # A double-quoted run is an *identifier*, not a literal, so it stays parseable -- - # `QUOTED_DOTTED_REF_RE` matches it as one identifier part. + # `DOTTED_REF_RE` matches it as one identifier part. ('"col" = `c`', [('"col" = ', False), ("`c`", True)]), ("a = 'unterminated", [("a = ", False), ("'unterminated", True)]), ("plain", [("plain", False)]), @@ -1969,12 +1969,20 @@ def test_a_calculated_measure_is_judged_on_its_aggregates_not_its_type(): ("MIN(users.x) + MAX(users.y)", False), ("COUNT(DISTINCT users.id) / MAX(users.x)", False), ("users.a + users.b", False), # no aggregate at all - ("not valid sql (((", True), # unparseable: assume the worse ]) def test_non_idempotent_aggregate_detection(expr, unsafe): - from ossie_cube.expressions import has_non_idempotent_aggregate + from ossie_cube.expressions import unsafe_aggregate_datasets - assert has_non_idempotent_aggregate(expr) is unsafe + datasets, unqualified = unsafe_aggregate_datasets(expr) + assert bool(datasets or unqualified) is unsafe + + +def test_an_unparseable_expression_is_assumed_unsafe(): + """`None` means "cannot tell", and the caller then attributes every dataset the + expression names -- the point being not to emit a silently inflated number.""" + from ossie_cube.expressions import unsafe_aggregate_datasets + + assert unsafe_aggregate_datasets("not valid sql (((") is None def test_a_cross_cube_member_gets_the_target_cubes_own_spelling(): @@ -2185,9 +2193,12 @@ def test_fanout_is_judged_per_aggregate_and_per_dataset(sql, flagged): ("APPROX_COUNT_DISTINCT(users.x)", False), ]) def test_only_provably_idempotent_aggregates_are_treated_as_safe(expr, unsafe): - from ossie_cube.expressions import has_non_idempotent_aggregate + """Exercised through the function the converter actually calls, so the allowlist is + pinned on the production path rather than on a wrapper beside it.""" + from ossie_cube.expressions import unsafe_aggregate_datasets - assert has_non_idempotent_aggregate(expr) is unsafe + datasets, unqualified = unsafe_aggregate_datasets(expr) + assert bool(datasets or unqualified) is unsafe def test_a_cross_cube_alias_is_not_prefixed_with_the_own_cube(): From a7db53bc7df687ebca37463d4e63d4fb06f1deff Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Tue, 4 Aug 2026 21:56:13 +0500 Subject: [PATCH 35/46] Refactor: one notion of identifier matching, tables prepared once No behaviour change; 524 tests and both gates green before and after. Identifier resolution was four helpers that were one idea wearing different shapes: `normalize_identifier` (the spec's normalized form), `_lookup_keys` (a tuple of keys for a *written* token), `lookup_map` (keys for a *declared* name), and `_first` (try a tuple against a dict). Written and declared names were reduced to keys by different code, which is how the exact-spelling key came to exist on one side only. Now `match_keys` reduces either side the same way, `lookup_map` builds from it, and `resolve_identifier` answers "what does this written token name?". The three call sites still doing lookups by hand with a single normalized key -- `referenced_datasets`, the dropped-field check, and fan-out attribution -- go through it too, so they gain the exact-spelling match they were silently missing. That is a small widening rather than a pure refactor, and it is the consistent behaviour. `ossie_expr_to_cube_sql` took six name collections and rebuilt six lookup maps from them on *every call* -- once per measure, over the same names each time. It now takes `ReferenceTables`, prepared once per model: 7 parameters -> 3, and the repeated map building is gone. `_convert_measure` on the import side had the same shape as the export side before `_CubePlan`: nine parameters, four of them model-wide facts. They travel as `_MeasureContext`. `sanitized` was a third spelling of the dataset lookup, alongside `cube_names` and `tables.datasets`; the prepared table answers it via `datasets_in`. Worst parameter count across the package: 13 -> 9. `ossie_expr_to_cube_sql` 7 -> 3, `_convert_measure` 9 -> 5, `_build_dimensions` 7 -> 5. --- converters/cube/src/ossie_cube/_common.py | 191 +++++++++++------- converters/cube/src/ossie_cube/cube_to_osi.py | 52 ++++- converters/cube/src/ossie_cube/osi_to_cube.py | 94 ++++----- converters/cube/tests/_util.py | 17 ++ converters/cube/tests/test_edge_cases.py | 12 +- 5 files changed, 225 insertions(+), 141 deletions(-) diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index 16fc3e6b..bef78ae7 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -24,6 +24,7 @@ column references Ossie expressions use. """ +import dataclasses import datetime import json import re @@ -517,66 +518,84 @@ def sub_outside_quotes(sql, transform): for text, quoted in quoted_runs(sql)) -def _lookup_keys(written): - """The keys a written identifier may match: its exact spelling, then normalized.""" - text = str(written).strip() +# --- identifiers ----------------------------------------------------------------- +# +# Resolving a reference means matching what someone *wrote* against what the model +# *declares*, and Ossie's rules make those two different strings: a regular identifier is +# case-insensitive while a quoted one is exact. Both sides are reduced to the same small +# set of match keys, so there is one notion of "could these be the same identifier" +# rather than one per call site. + +def _unquoted(text): + """(content, was_quoted) for an ANSI-quoted identifier, with `""` unescaped.""" if len(text) >= 2 and text.startswith('"') and text.endswith('"'): - inner = text[1:-1].replace('""', '"') - return (inner, normalize_identifier(text)) - return (text, normalize_identifier(text)) - - -def _first(mapping, keys): - """The first of `keys` present in `mapping`, or None.""" - for key in keys: - if key in mapping: - return mapping[key] - return None + return text[1:-1].replace('""', '"'), True + return text, False def normalize_identifier(name): - """An Ossie identifier in the spec's *normalized* form, for matching. + """An Ossie identifier in the spec's *normalized* form. From core-spec/expression_language.md: "Regular identifiers (unquoted) should be case insensitive [...] Regular identifiers are upper cased; quoted identifiers have - their quotes stripped". So `orders.AMOUNT` addresses the field `amount`, and - matching them exactly -- as this converter used to -- emitted `{CUBE}.AMOUNT`, a raw - column that bypasses the member's own expression entirely. + their quotes stripped". So `orders.AMOUNT` addresses the field `amount`, and matching + them exactly -- as this converter used to -- emitted `{CUBE}.AMOUNT`, a raw column + that bypasses the member's own expression entirely. Cube identifiers, by contrast, *are* case-sensitive, so the canonical Cube spelling is what gets emitted; this form is only used to find it. """ - text = str(name).strip() - if len(text) >= 2 and text.startswith('"') and text.endswith('"'): - return text[1:-1].replace('""', '"') - return text.upper() + content, quoted = _unquoted(str(name).strip()) + return content if quoted else content.upper() -def referenced_datasets(expr, known): - """The dataset names an Ossie expression references, ignoring quoted text. +def match_keys(identifier): + """Every key this identifier can be matched by, most specific first. - Decides which cube a measure lands on and whether it crosses cubes, so a name - that only appears inside a string literal must not count -- otherwise - `SUM(orders.amount) || ' per users.id unit'` reads as a two-dataset metric and - gets attributed to the base cube rather than to `orders`. + A quoted identifier matches on its content alone -- that is what "exact" means, and + it is also how a name that *must* be quoted stays referenceable: `"Order Items"` is + the only way to write that dataset's name. An unquoted one matches its own spelling + (so a declared name spelled the same way is found) and its normalized form (so any + casing is). + """ + content, quoted = _unquoted(str(identifier).strip()) + if quoted: + return (content,) + return (content, content.upper()) + + +def datasets_in_expression(expr, prepared): + """Dataset names referenced in `expr`, resolved against a prepared lookup map. + + Split from `referenced_datasets` so a caller holding prepared tables does not rebuild + the map for every expression. """ - # `lookup_map`, not `canonical_map`: `known` may map an accepted spelling to the - # canonical name, and what callers need back is the canonical one -- a name they can - # place a measure under. Returning the spelling as written filed it under a cube that - # does not exist, and the measure vanished with no issue reported. - canonical = lookup_map(known) found = set() for text, quoted in quoted_runs(expr): if quoted: continue for match in DOTTED_REF_RE.finditer(text): head, _ = split_dotted_ref(match.group(0)) - name = canonical.get(normalize_identifier(head)) + name = resolve_identifier(prepared, head) if name is not None: found.add(name) return found +def referenced_datasets(expr, known): + """The dataset names an Ossie expression references, ignoring quoted text. + + Decides which cube a measure lands on and whether it crosses cubes, so a name + that only appears inside a string literal must not count -- otherwise + `SUM(orders.amount) || ' per users.id unit'` reads as a two-dataset metric and + gets attributed to the base cube rather than to `orders`. + """ + # What callers need back is the *canonical* name -- one they can place a measure + # under. Returning the spelling as written filed it under a cube that does not exist, + # and the measure vanished with no issue reported. + return datasets_in_expression(expr, lookup_map(known)) + + def split_dotted_ref(text): """Split a matched `cube.member` reference into its two identifier parts. @@ -619,26 +638,27 @@ def quoted_char_mask(sql): def lookup_map(names): - """{lookup key: the name to emit}, keyed both ways an identifier can be written. - - Accepts a set of names (each maps to itself) or a mapping of *any* accepted spelling - to the canonical one -- which is what lets an Ossie name and the Cube name it - sanitizes to both resolve to the Cube spelling. + """{match key: the name to emit}, from a set of names or a mapping of spellings. - Each name gets two keys: its normalized form (upper-cased, matching an unquoted - reference) and its exact spelling (matching a quoted one). The second is needed - because a name containing a space or mixed case *cannot* be written unquoted at all -- - `"Order Items"` is the only way to reference a dataset of that name, so exact-quoted - has to resolve or the name is unusable. + A mapping lets several accepted spellings resolve to one canonical name -- which is + how an Ossie dataset name and the Cube name it sanitizes to both reach the Cube one. """ pairs = names.items() if isinstance(names, dict) else ((n, n) for n in names) out = {} for spelling, canonical in pairs: - out.setdefault(str(spelling), canonical) - out.setdefault(normalize_identifier(spelling), canonical) + for key in match_keys(spelling): + out.setdefault(key, canonical) return out +def resolve_identifier(mapping, written): + """What `written` names in a `lookup_map`, or None.""" + for key in match_keys(written): + if key in mapping: + return mapping[key] + return None + + def source_part_count(source): """How many identifier parts a dotted dataset `source` has, or None for a query. @@ -716,8 +736,47 @@ def requalify_self_refs(sql, cube_name): ) -def ossie_expr_to_cube_sql(expr, own_cube, own_members=(), cube_names=(), - inline_sql=None, members_by_cube=None, own_lookup=None): +@dataclasses.dataclass(frozen=True) +class ReferenceTables: + """The prepared lookups `ossie_expr_to_cube_sql` resolves a reference against. + + Built once per model rather than per expression. Passing the raw name collections + instead meant every measure rewrite rebuilt a lookup map for every cube's members -- + six maps per call, over the same names each time -- which is the cost of threading + collections through a signature rather than preparing them once. + """ + + datasets: dict # match key -> Cube cube name + references: dict # cube -> match key -> member needing `{CUBE.member}` + columns: dict # cube -> match key -> canonical column/dimension name + inline_sql: dict # cube -> match key -> Cube SQL to substitute + + @classmethod + def of(cls, cube_names=(), references_by_cube=None, columns_by_cube=None, + inline_sql_by_cube=None): + def per_cube(source, prepare): + return {normalize_identifier(cube): prepare(value) + for cube, value in (source or {}).items()} + + return cls( + datasets=lookup_map(cube_names), + references=per_cube(references_by_cube, lookup_map), + columns=per_cube(columns_by_cube, lookup_map), + inline_sql=per_cube( + inline_sql_by_cube, + lambda fields: {normalize_identifier(f): sql + for f, sql in fields.items()}), + ) + + def for_cube(self, cube, attribute): + return getattr(self, attribute).get(normalize_identifier(cube)) or {} + + def datasets_in(self, expr): + """The datasets `expr` references, by canonical Cube name.""" + return datasets_in_expression(expr, self.datasets) + + +def ossie_expr_to_cube_sql(expr, own_cube, tables): """Rewrite an Ossie expression into Cube member-reference form. Only *dotted* `cube.name` references are rewritten -- a bare identifier stays @@ -749,60 +808,46 @@ def ossie_expr_to_cube_sql(expr, own_cube, own_members=(), cube_names=(), own text with a column reference. """ escaped = str(expr).replace("{", "\\{").replace("}", "\\}") - known = lookup_map(cube_names) - members = lookup_map(own_members) + known = tables.datasets + members = tables.for_cube(own_cube, "references") + own_columns = tables.for_cube(own_cube, "columns") own_norm = normalize_identifier(own_cube) if own_cube else None - inline_sql_by_norm = { - normalize_identifier(cube): {normalize_identifier(f): sql - for f, sql in fields.items()} - for cube, fields in (inline_sql or {}).items() - } - - own_columns = lookup_map(own_lookup or {}) - foreign_members = { - normalize_identifier(cube): lookup_map(names) - for cube, names in (members_by_cube or {}).items() - } def repl(m): head, name = split_dotted_ref(m.group(0)) # Ossie regular identifiers are case-insensitive, so the reference is matched # in normalized form; what is *emitted* is the canonical Cube spelling, since # Cube's own member lookup is case-sensitive. - head_n, name_n = _lookup_keys(head), _lookup_keys(name) + head_keys, name_keys = match_keys(head), match_keys(name) # Resolve the dataset first, then decide which branch applies. Comparing the # written spelling against `own_cube` directly was wrong once the two could # differ: dataset `Order Items` becomes cube `order_items`, so a reference to # `"ORDER ITEMS"` took the cross-cube branch on its own cube. - target = _first(known, head_n) - is_own = target == own_cube or (target is None and own_norm in head_n) + target = resolve_identifier(known, head) + is_own = target == own_cube or (target is None and own_norm in head_keys) # Keyed on the *resolved* cube, not the token as written: a sanitized dataset name # differs from the Ossie one, and looking inline SQL up by the written token meant # a split geo half referenced through the Ossie name was never substituted. - inline_for = inline_sql_by_norm.get( - normalize_identifier(target)) if target else None - if inline_for is None: - inline_for = _first(inline_sql_by_norm, head_n) or {} - substitute = _first(inline_for, name_n) + inline_for = tables.for_cube(target, "inline_sql") if target else {} + substitute = resolve_identifier(inline_for, name) if substitute is not None: # Already-Cube SQL, so it bypasses the escaping above; `{CUBE}` inside # it means `head`, which only stays true while head is the own cube. return (str(substitute) if is_own else requalify_self_refs(substitute, target or head)) if is_own: - member = _first(members, name_n) + member = resolve_identifier(members, name) if member is not None: return "{CUBE." + member + "}" # A plain member is the same thing either way, but the *column* still has a # canonical spelling -- emitting `"AMOUNT"` as written would force an exact # uppercase match in the database against a column named `amount`. - column = _first(own_columns, name_n) + column = resolve_identifier(own_columns, name) return "{CUBE}." + (column if column is not None else name) if target is not None: # A cross-cube member needs the target cube's own spelling for the same # reason: `{users.ID}` does not resolve when the member is declared `id`. - member = _first(foreign_members.get(normalize_identifier(target)) or {}, - name_n) + member = resolve_identifier(tables.for_cube(target, "columns"), name) return "{" + target + "." + (member if member is not None else name) + "}" # Not a dataset in this model -- a genuine schema-qualified table # reference or an unrelated dotted token. Leave it alone. diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 0c3c3eff..aac2de26 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -32,6 +32,7 @@ ossie-cube import -i model/ [-o model.yaml] [--name NAME] [--view VIEW] """ +import dataclasses import re from ._common import ( @@ -50,7 +51,7 @@ filtered_operand, is_simple_identifier, lookup_map, - normalize_identifier, + resolve_identifier, referenced_datasets, join_source, load_yaml, @@ -1253,8 +1254,8 @@ def _fanout_unsafe_datasets(expr, own_cube, dataset_names): return referenced_datasets(expr, dataset_names) or {own_cube} tables, unqualified = analysed canonical = lookup_map(dataset_names) - found = {canonical[normalize_identifier(table)] for table in tables - if normalize_identifier(table) in canonical} + found = {resolve_identifier(canonical, table) for table in tables} + found.discard(None) if unqualified: # An unsafe aggregate over an unqualified column reads the declaring cube. found.add(own_cube) @@ -1275,6 +1276,27 @@ def _is_generated_part(measure): return bool(((measure.get("meta") or {}).get("ossie") or {}).get("part_of")) +@dataclasses.dataclass(frozen=True) +class _MeasureContext: + """Model-wide facts every measure conversion needs. + + `resolver` produces a measure's Ossie expression, `fanned_out` says which datasets a + relationship multiplies, `dataset_names` is what a reference can resolve to, and + `plain_by_cube` says which members regenerate from a bare column name. Passing them + one at a time made `_convert_measure` a nine-parameter function whose signature said + nothing about what it does. + """ + + resolver: object + fanned_out: dict + dataset_names: frozenset + plain_by_cube: dict + issues: object + + def plain(self, cname): + return self.plain_by_cube.get(cname) or set() + + def _convert_measures(cubes, pk_by_cube, plain_by_cube, fanned_out, issues): """Hoist every cube's measures into Ossie model-level metrics. @@ -1288,7 +1310,13 @@ def _convert_measures(cubes, pk_by_cube, plain_by_cube, fanned_out, issues): otherwise vanish. They ride on the owning dataset's stash with their positions, the same protocol unconvertible joins use. """ - resolver = _MeasureResolver(cubes, pk_by_cube, issues) + context = _MeasureContext( + resolver=_MeasureResolver(cubes, pk_by_cube, issues), + fanned_out=fanned_out, + dataset_names=frozenset(cubes), + plain_by_cube=plain_by_cube, + issues=issues) + resolver = context.resolver counts = {} for (cname, mname), measure in resolver.measures().items(): @@ -1316,8 +1344,7 @@ def _convert_measures(cubes, pk_by_cube, plain_by_cube, fanned_out, issues): f"metric name '{metric_name}' derived twice; rename the " f"colliding measures in Cube") seen.add(metric_name) - metric = _convert_measure(cname, mname, metric_name, measure, resolver, - fanned_out, plain, set(cubes), issues) + metric = _convert_measure(cname, mname, metric_name, measure, context) if metric is not None: metrics.append(metric) else: @@ -1326,8 +1353,9 @@ def _convert_measures(cubes, pk_by_cube, plain_by_cube, fanned_out, issues): return metrics, extra_measures -def _convert_measure(cname, mname, metric_name, measure, resolver, fanned_out, - plain, dataset_names, issues): +def _convert_measure(cname, mname, metric_name, measure, context): + resolver, issues = context.resolver, context.issues + plain = context.plain(cname) scope = f"{cname}.{mname}" expr = resolver.expression(cname, mname) if expr is None: @@ -1356,13 +1384,15 @@ def _convert_measure(cname, mname, metric_name, measure, resolver, fanned_out, # nothing about the aggregates inside it, and the cube a measure is *declared* on is # not necessarily the one an aggregate inside it *reads* -- `SUM(users.ltv) / # SUM(orders.amount)` sits on `orders` while `users` is the fanned-out side. - for dataset in sorted(_fanout_unsafe_datasets(expr, cname, dataset_names)): - if dataset not in fanned_out: + for dataset in sorted( + _fanout_unsafe_datasets(expr, cname, context.dataset_names)): + if dataset not in context.fanned_out: continue issues.add( IssueType.FANOUT_UNSAFE_METRIC, scope, f"a non-idempotent aggregate reads dataset '{dataset}', which " - f"relationship '{fanned_out[dataset]}' fans out; Cube deduplicates on " + f"relationship '{context.fanned_out[dataset]}' fans out; Cube " + f"deduplicates on " f"the primary key at query time but a static Ossie expression cannot, so " f"a consumer joining through that relationship may over-count") diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 6329da69..5eaa9a69 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -50,6 +50,7 @@ instructions_of, is_simple_identifier, load_yaml, + ReferenceTables, ossie_expr_to_cube_sql, parse_source, pick_expression, @@ -58,7 +59,7 @@ referenced_datasets, DOTTED_REF_RE, lookup_map, - normalize_identifier, + resolve_identifier, quoted_runs, split_dotted_ref, require_str, @@ -152,10 +153,13 @@ def _convert_model(model, dialect, base_cube, issues): f"Model '{name}': dataset '{ds_name}'") for ds_name, ds in datasets.items()} + tables = _reference_tables(plan, cube_names) + joins_by_cube, join_parked_by_cube = _build_joins( relationships, cube_names, issues) measures_by_cube = _build_measures( - model, cube_names, plan, datasets, relationships, base_cube, dialect, issues) + model, cube_names, plan, tables, datasets, relationships, base_cube, dialect, + issues) # Cubes, grouped by the file they belong in: several datasets can share one # stashed original path, in which case they go back into the same file. @@ -164,7 +168,7 @@ def _convert_model(model, dialect, base_cube, issues): emitted_members = {} for ds_name, ds in datasets.items(): cname = cube_names[ds_name] - cube = _build_cube(ds, plan[cname], joins_by_cube.get(cname), + cube = _build_cube(ds, plan[cname], tables, joins_by_cube.get(cname), measures_by_cube.get(cname), join_parked_by_cube.get(cname), dialect, issues) stashed = stashed_paths.get(cname) @@ -268,7 +272,8 @@ def _ordered(obj, order): # --- cubes ---------------------------------------------------------------------- -def _build_cube(ds, plan, joins, measures, join_extensions, dialect, issues): +def _build_cube(ds, plan, tables, joins, measures, join_extensions, dialect, + issues): cname = plan.cname ds_name = ds["name"] scope = f"dataset '{ds_name}'" @@ -305,7 +310,7 @@ def _build_cube(ds, plan, joins, measures, join_extensions, dialect, issues): "so this cube-level value has no effect in Cube") dimensions, by_name_scalar, by_column, by_name_computed = _build_dimensions( - ds, plan, dialect, issues) + ds, plan, tables, dialect, issues) # Resolve each `primary_key` entry to the dimension Cube should mark. A # dimension only qualifies when it is *scalar* -- backed by a single source # column -- because `primary_key: true` in Cube declares that dimension's own @@ -557,7 +562,7 @@ def _resolve_dimension_names(ds, scope): return names, inline_sql -def _build_dimensions(ds, plan, dialect, issues): +def _build_dimensions(ds, plan, tables, dialect, issues): """Build a cube's dimensions from an Ossie dataset's fields. Returns (dimensions, by_name_scalar, by_column, by_name_computed). @@ -608,9 +613,7 @@ def _build_dimensions(ds, plan, dialect, issues): # The exact Cube spelling a prior import saw. dim["sql"] = stash["sql"] else: - dim["sql"] = ossie_expr_to_cube_sql( - expr, cname, plan.references, (), - inline_sql={cname: plan.inline_sql}) + dim["sql"] = ossie_expr_to_cube_sql(expr, cname, tables) if stash.get("case") is not None: # A `case` dimension carries its conditions instead of `sql`, and Cube # rejects a dimension declaring both ("dimensions.size does not match any @@ -844,16 +847,10 @@ def _build_joins(relationships, cube_names, issues): # --- measures ------------------------------------------------------------------- -def _build_measures(model, cube_names, plan, datasets, relationships, base_cube, - dialect, issues): +def _build_measures(model, cube_names, plan, tables, datasets, relationships, + base_cube, dialect, issues): """Group Ossie metrics into per-cube `measures` lists.""" name = model.get("name", "") - # A metric is authored against Ossie names, and a name needing sanitization has a - # different Cube name -- dataset `Order Items` becomes cube `order_items`. Accept - # either spelling and emit the Cube one. Passing only the Cube names left - # `SUM("ORDER ITEMS".amount)` as raw SQL naming a table that does not exist. - sanitized = dict(cube_names) - sanitized.update({cname: cname for cname in cube_names.values()}) base_cache = [] def resolve_base(): @@ -906,7 +903,7 @@ def resolve_base(): "no ANSI_SQL or preferred-dialect expression; metric dropped") continue - missing = _references_a_dropped_field(expr, sanitized, cube_names, plan) + missing = _references_a_dropped_field(expr, tables, cube_names, plan) if missing: # The field has no expression in a usable dialect, so Cube gets no # dimension for it -- and a measure referencing one it did not get is a @@ -916,7 +913,7 @@ def resolve_base(): f"expression in a usable dialect and so becomes no Cube " f"dimension; the metric is dropped with it") continue - referenced = referenced_datasets(expr, sanitized) + referenced = tables.datasets_in(expr) target = stash.get("cube") or ( next(iter(referenced)) if len(referenced) == 1 else resolve_base()) @@ -939,21 +936,21 @@ def resolve_base(): # applies its row-multiplication correction per aggregate instead of # seeing one opaque expression -- see _decompose_measure. public_sql = _decompose_measure( - expr, spans, mname, target, measures_by_cube, plan, sanitized, + expr, spans, mname, target, measures_by_cube, plan, tables, name, reserved) measure = {"name": mname, "sql": public_sql, "type": "number"} else: measure = _measure_from_expression( - expr, target, mname, stash, plan, sanitized) + expr, target, mname, stash, plan, tables) _apply_measure_metadata(metric, measure, stash) _place(measures_by_cube, target, measure, name) return measures_by_cube -def _references_a_dropped_field(expr, sanitized, cube_names, plan): +def _references_a_dropped_field(expr, tables, cube_names, plan): """`dataset.field` references in `expr` naming a field that becomes no dimension.""" by_cube_name = {cname: ds_name for ds_name, cname in cube_names.items()} - canonical = lookup_map(sanitized) + canonical = tables.datasets dropped_norm = { cname: lookup_map(fields) for cname, fields in ((c, p.dropped) for c, p in plan.items()) @@ -967,17 +964,17 @@ def _references_a_dropped_field(expr, sanitized, cube_names, plan): # with a reference to a dimension that was never created. for match in DOTTED_REF_RE.finditer(text): head, field = split_dotted_ref(match.group(0)) - cname = canonical.get(normalize_identifier(head)) + cname = resolve_identifier(canonical, head) if cname is None: continue - fname = (dropped_norm.get(cname) or {}).get(normalize_identifier(field)) + fname = resolve_identifier(dropped_norm.get(cname) or {}, field) if fname is not None: missing.add(f"'{by_cube_name.get(cname, cname)}.{fname}'") return missing def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, plan, - sanitized, model_name, reserved): + tables, model_name, reserved): """Emit one `public: false` measure per aggregate; return the sql referencing them. Cube corrects for row multiplication per measure, keyed on the cube that measure @@ -1000,7 +997,7 @@ def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, plan, for start, end in spans: piece = expr[start:end] # Each aggregate lands on the cube its own operand references. - refs = referenced_datasets(piece, sanitized) + refs = tables.datasets_in(piece) part_target = next(iter(refs)) if len(refs) == 1 else fallback index += 1 @@ -1011,20 +1008,18 @@ def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, plan, taken.add(part_name.lower()) part = _measure_from_expression( - piece, part_target, part_name, {}, plan, sanitized) + piece, part_target, part_name, {}, plan, tables) part["public"] = False part["meta"] = {"ossie": {"part_of": mname}} _place(measures_by_cube, part_target, part, model_name) - out.append(_to_cube_sql( - expr[cursor:start], fallback, plan, sanitized)) + out.append(ossie_expr_to_cube_sql(expr[cursor:start], fallback, tables)) # `{CUBE.x}` for a part on the same cube as the public measure: an explicit # name pins the reference to this cube and breaks if it is extended. qualifier = "CUBE" if part_target == fallback else part_target out.append("{" + f"{qualifier}.{part_name}" + "}") cursor = end - out.append(_to_cube_sql( - expr[cursor:], fallback, plan, sanitized)) + out.append(ossie_expr_to_cube_sql(expr[cursor:], fallback, tables)) return "".join(out) @@ -1037,24 +1032,23 @@ def _place(measures_by_cube, target, measure, model_name): bucket.append(measure) -def _to_cube_sql(text, target, plan, sanitized): - """One Ossie expression fragment as Cube SQL, resolved against the whole plan. +def _reference_tables(plan, cube_names): + """The prepared reference lookups for a whole model. - Every measure rewrite needs the same four lookups -- the target's reference members, - its full name lookup, every cube's members, and the inline SQL of split geo halves -- - so they are assembled here rather than spelled out at each call site. + A dataset resolves from either spelling -- its Ossie name or the Cube name it + sanitizes to -- because a metric is authored against the former and everything + downstream needs the latter. """ - own = plan.get(target) - return ossie_expr_to_cube_sql( - text, target, - own.references if own else (), - sanitized, - inline_sql={c: p.inline_sql for c, p in plan.items()}, - members_by_cube={c: p.lookup for c, p in plan.items()}, - own_lookup=own.lookup if own else None) + datasets = dict(cube_names) + datasets.update({cname: cname for cname in cube_names.values()}) + return ReferenceTables.of( + cube_names=datasets, + references_by_cube={c: p.references for c, p in plan.items()}, + columns_by_cube={c: p.lookup for c, p in plan.items()}, + inline_sql_by_cube={c: p.inline_sql for c, p in plan.items()}) -def _measure_from_expression(expr, target, mname, stash, plan, sanitized): +def _measure_from_expression(expr, target, mname, stash, plan, tables): """Turn an Ossie metric expression back into a structured Cube measure. `COUNT(DISTINCT )` is Cube's bare `type: count` -- @@ -1085,15 +1079,15 @@ def _measure_from_expression(expr, target, mname, stash, plan, sanitized): if not (func == "COUNT" and inner == "*"): agg = OSSIE_FUNC_TO_AGG.get(func) or ("count" if func == "COUNT" else None) if agg is not None: - measure["sql"] = stash.get("sql") or _to_cube_sql( - inner, target, plan, sanitized) + measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( + inner, target, tables) measure["type"] = agg return measure # A ratio, a window expression, or a multi-dataset aggregate: Cube expresses # these as a calculated measure whose sql carries the aggregation. - measure["sql"] = stash.get("sql") or _to_cube_sql( - expr, target, plan, sanitized) + measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( + expr, target, tables) measure["type"] = "number" return measure diff --git a/converters/cube/tests/_util.py b/converters/cube/tests/_util.py index 5e18bd28..c8c9563c 100644 --- a/converters/cube/tests/_util.py +++ b/converters/cube/tests/_util.py @@ -140,3 +140,20 @@ def walk(node): walk(obj) return obj + + +def to_cube_sql(expr, own_cube, members=(), cube_names=(), inline_sql=None): + """`ossie_expr_to_cube_sql` for a single-cube model, for the direct unit tests. + + The converter prepares its reference tables once per model; these tests care about + one expression, so the tables are assembled here rather than in each test. + """ + from ossie_cube._common import ReferenceTables, ossie_expr_to_cube_sql + + names = set(cube_names) | {own_cube} + tables = ReferenceTables.of( + cube_names=names, + references_by_cube={own_cube: members}, + columns_by_cube={own_cube: members}, + inline_sql_by_cube={own_cube: inline_sql or {}}) + return ossie_expr_to_cube_sql(expr, own_cube, tables) diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index be6a6c02..944bf0ee 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -2039,20 +2039,18 @@ def test_quoted_identifiers_are_parsed_not_skipped(reference, expected): """The whole double-quoted region used to be treated as opaque -- the same handling string literals get -- so a quoted reference was never rewritten and bypassed the member it named.""" - from ossie_cube._common import ossie_expr_to_cube_sql + from _util import to_cube_sql - assert ossie_expr_to_cube_sql( - f"SUM({reference})", "orders", {"amount"}, {"orders"}) == expected + assert to_cube_sql(f"SUM({reference})", "orders", {"amount"}) == expected def test_a_single_quoted_literal_is_still_opaque(): """The change above must not weaken literal handling: Cube compiles every string as an f-string, so a `{...}` emitted into a literal would be interpolated.""" - from ossie_cube._common import ossie_expr_to_cube_sql + from _util import to_cube_sql - assert ossie_expr_to_cube_sql( - "SUM(orders.amount) || ' orders.amount '", "orders", {"amount"}, - {"orders"}) == "SUM({CUBE.amount}) || ' orders.amount '" + assert to_cube_sql("SUM(orders.amount) || ' orders.amount '", "orders", + {"amount"}) == "SUM({CUBE.amount}) || ' orders.amount '" _CHAIN = ( From e90f60c62fe2e2d5362ee84bcb5a582b152c69b6 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Tue, 4 Aug 2026 23:26:50 +0500 Subject: [PATCH 36/46] Fix four more review findings in aggregate analysis and geo references All four reproduced first. The two P1s are the same hole from two directions: the analysis only understood one shape of aggregate. [P1] Qualified and unqualified operands were not tracked independently. An aggregate can read both, and `SUM(amount + line_items.qty)` reported only `line_items` -- the declaring cube, which the bare `amount` belongs to, went unmentioned, so a fan-out on it passed strict mode. [P1] Only `AggFunc` nodes were examined, which is one of three shapes sqlglot uses: - an *ordered-set* aggregate keeps its value-bearing column in the ORDER BY, on the `WithinGroup` wrapper rather than the inner function, so `PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY users.ltv)` was blamed on the declaring cube; - `LISTAGG(...) WITHIN GROUP (...)` is not modelled as an aggregate at all and disappeared from the analysis entirely. Aggregate *scope* now covers `AggFunc`, `WithinGroup` and unmodelled calls, with nesting resolved so an ordered-set aggregate counts once rather than reporting its inner function separately. An unmodelled call may equally be a scalar UDF, so this over-reports; that is the cheaper error, since the default is to warn rather than refuse, and the alternative is a silently inflated number. - `BOOL_OR`/`BOOL_AND` were reported unsafe. Duplicating a row cannot change whether any or all rows satisfy a predicate. `BIT_OR`/`BIT_AND` added for the same reason. - The inline-SQL table was keyed by the normalized identifier alone, unlike every other table, so an exact-quoted reference to a split geo half -- `users."home_latitude"` -- missed its substitution and came out as a raw column of a name that exists in Ossie and not in the database. It now uses the shared match-key logic. 541 tests with both gates, 516 with neither, 96% coverage. Interop unchanged. --- converters/cube/README.md | 21 +++--- converters/cube/src/ossie_cube/_common.py | 8 ++- converters/cube/src/ossie_cube/expressions.py | 69 +++++++++++++++--- converters/cube/tests/test_edge_cases.py | 71 +++++++++++++++++++ 4 files changed, 151 insertions(+), 18 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index e1a57a6f..844885c2 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -227,14 +227,19 @@ the measure's Cube type, and not on the cube it is declared on. Both shortcuts w wrong: a calculated `type: number` measure's type says nothing about the aggregates inside it, and the cube a measure is declared on is not necessarily the one an aggregate inside it *reads*. `SUM(users.ltv) / SUM(orders.amount)` sits on `orders` while `users` -is the fanned-out side. The idempotent set is an **allowlist** — `MIN`, `MAX`, `APPROX_COUNT_DISTINCT`, and -*any* aggregate over a `DISTINCT` set (which collapses duplicates before the aggregate -sees them, so `SUM(DISTINCT x)` is as safe as `COUNT(DISTINCT x)`) — because the set of -aggregate functions is open-ended, and listing the unsafe ones silently declared -`STDDEV`, `MEDIAN` and `ARRAY_AGG` safe. Attribution walks the parse tree rather than -matching aggregate names in text, so an aggregate with no Cube mapping is attributed like -any other: `SUM(orders.amount) + STDDEV(users.ltv)` reports `users`, which name-matching -missed once it had found the `SUM`. +is the fanned-out side. The idempotent set is an **allowlist** — `MIN`, `MAX`, `APPROX_COUNT_DISTINCT`, +`BOOL_OR`/`BOOL_AND`/`BIT_OR`/`BIT_AND`, and *any* aggregate over a `DISTINCT` set (which +collapses duplicates before the aggregate sees them, so `SUM(DISTINCT x)` is as safe as +`COUNT(DISTINCT x)`) — because the set of aggregate functions is open-ended, and listing +the unsafe ones silently declared `STDDEV`, `MEDIAN` and `ARRAY_AGG` safe. + +Attribution walks the parse tree rather than matching aggregate names in text, and counts +three shapes as one aggregate: an ordinary call, an *ordered-set* one (`PERCENTILE_CONT(…) +WITHIN GROUP (ORDER BY x)`, whose value-bearing column sits on the wrapper), and a call +SQL parsing does not model at all (`LISTAGG`, `APPROX_PERCENTILE`). The last of those may +equally be a scalar UDF, so treating it as an aggregate over-reports — the cheaper error, +since the default is to warn rather than refuse. Qualified and unqualified operands are +counted independently, because one aggregate can read both. The TPC-DS fixture carries a real example: `store_productivity` is `SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), 0)`, and diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index bef78ae7..d223fd85 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -762,10 +762,14 @@ def per_cube(source, prepare): datasets=lookup_map(cube_names), references=per_cube(references_by_cube, lookup_map), columns=per_cube(columns_by_cube, lookup_map), + # Keyed with the same match logic as every other table. Using only the + # normalized form meant an exact-quoted reference to a split geo half -- + # `users."home_latitude"` -- missed its substitution and came out as a raw + # column of that name, which exists in Ossie and not in the database. inline_sql=per_cube( inline_sql_by_cube, - lambda fields: {normalize_identifier(f): sql - for f, sql in fields.items()}), + lambda fields: {key: sql for f, sql in fields.items() + for key in match_keys(f)}), ) def for_cube(self, cube, attribute): diff --git a/converters/cube/src/ossie_cube/expressions.py b/converters/cube/src/ossie_cube/expressions.py index 02643055..b2cfc0cb 100644 --- a/converters/cube/src/ossie_cube/expressions.py +++ b/converters/cube/src/ossie_cube/expressions.py @@ -172,11 +172,46 @@ def _match_paren(text, open_at): # treated as unsafe -- an allowlist rather than a blocklist, because the set of # aggregate functions is open-ended (STDDEV, VARIANCE, MEDIAN, ARRAY_AGG, PERCENTILE...) # and listing the unsafe ones meant every unlisted one was silently declared safe. -_IDEMPOTENT_NODES = (exp.Min, exp.Max, exp.ApproxDistinct) +_IDEMPOTENT_NODES = ( + exp.Min, exp.Max, exp.ApproxDistinct, + # BOOL_OR / BOOL_AND: a duplicated row cannot change whether *any* or *all* rows + # satisfy the predicate. + exp.LogicalOr, exp.LogicalAnd, +) + +# Aggregates sqlglot leaves as an unmodelled call (`Anonymous`) but which duplication +# cannot affect either. Bitwise OR/AND of a value set is idempotent for the same reason +# the logical ones are. +_IDEMPOTENT_CALLS = frozenset({"BIT_OR", "BIT_AND", "BOOL_OR", "BOOL_AND"}) + + +def _is_aggregate_scope(node): + """True for a node that constitutes one aggregate, whatever shape sqlglot gave it. + + Three shapes, all of which have to count: + - `AggFunc`, the modelled aggregates (SUM, COUNT, PERCENTILE_CONT, ...); + - `WithinGroup`, an *ordered-set* aggregate -- the value-bearing column lives in the + ORDER BY, on the wrapper rather than on the inner function, so examining only the + inner one attributed `PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY users.ltv)` to + the declaring cube instead of `users`; + - `Anonymous`, a call sqlglot does not model at all -- which is how LISTAGG, + APPROX_PERCENTILE and BIT_OR arrive, and how `LISTAGG(...) WITHIN GROUP (...)` + vanished from the analysis entirely. + + An `Anonymous` may equally be a scalar UDF, so treating it as an aggregate + over-reports. That is the cheaper error: a warning by default (the fan-out policy + warns rather than refuses), against a silently inflated number the other way. + """ + return isinstance(node, (exp.AggFunc, exp.WithinGroup, exp.Anonymous)) def is_idempotent_aggregate(node): """True if duplicating input rows cannot change this aggregate's value.""" + if isinstance(node, exp.WithinGroup): + # An ordered-set aggregate is exactly as safe as the function being ordered. + return bool(node.this) and is_idempotent_aggregate(node.this) + if isinstance(node, exp.Anonymous): + return str(node.this or "").upper() in _IDEMPOTENT_CALLS if isinstance(node, _IDEMPOTENT_NODES): return True # DISTINCT collapses duplicates before the aggregate sees them, so *any* aggregate @@ -216,18 +251,36 @@ def unsafe_aggregate_datasets(expr): if tree is None: return None datasets, unqualified = set(), False - for node in tree.walk(): - if not isinstance(node, exp.AggFunc) or is_idempotent_aggregate(node): + for scope in _outermost_aggregate_scopes(tree): + if is_idempotent_aggregate(scope): continue - tables = {column.table for column in node.find_all(exp.Column) - if column.table} - if tables: - datasets |= tables - else: + columns = list(scope.find_all(exp.Column)) + # Qualified and unqualified operands are tracked *independently*: an aggregate can + # read both, and `SUM(amount + line_items.qty)` reported only `line_items` while + # the declaring cube -- which `amount` belongs to -- went unmentioned. + datasets |= {column.table for column in columns if column.table} + if not columns or any(not column.table for column in columns): unqualified = True return datasets, unqualified +def _outermost_aggregate_scopes(tree): + """Aggregate scopes that are not inside another one. + + Nesting is resolved so an ordered-set aggregate is counted once: `WithinGroup` and + the `PercentileCont` inside it are one aggregate, and treating the inner one as its + own scope would find no columns there and blame the declaring cube. + """ + scopes = [] + for node in tree.walk(): + if not _is_aggregate_scope(node): + continue + if any(any(inner is node for inner in scope.walk()) for scope in scopes): + continue + scopes.append(node) + return scopes + + def has_top_level_operator(expr): """True if `expr` is not a single self-contained term. diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 944bf0ee..cdd67a2b 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -2349,3 +2349,74 @@ def test_a_measure_depending_on_a_windowed_one_is_parked_too(): # Both come back, in their original positions. cube = parse(back["model/cubes/m.yml"])["cubes"][0] assert [m["name"] for m in cube["measures"]] == ["rolling", "rolling_ratio"] + + +# --- review round eight ---------------------------------------------------------- + +@pytest.mark.parametrize("sql,flagged", [ + # `users` is the fanned-out side; `orders` (the declaring cube) is not. + # + # An aggregate can read a qualified *and* an unqualified operand, and the two were + # not tracked independently: this reported `users` only, leaving the declaring cube + # -- which the bare `amount` belongs to -- unmentioned. Here it is `users` that must + # appear, which it did; the reverse case is covered below. + ("SUM({users}.ltv + {CUBE}.amount)", True), + # An *ordered-set* aggregate keeps its value-bearing column in the ORDER BY, on the + # wrapper rather than the inner function -- so examining only the inner one blamed + # the declaring cube and let this through. + ("PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY {users}.ltv)", True), + # sqlglot models LISTAGG as an unnamed call, so this vanished from the analysis + # entirely rather than merely being misattributed. + ("LISTAGG({users}.name, ',') WITHIN GROUP (ORDER BY {users}.name)", True), + # BOOL_OR / BOOL_AND cannot change when a row is duplicated, and were rejected. + ("BOOL_OR({users}.flag)", False), + ("BOOL_AND({users}.flag)", False), + ("BIT_OR({users}.mask)", False), + ("MAX({users}.ltv) - MIN({users}.ltv)", False), +]) +def test_fanout_covers_every_aggregate_shape(sql, flagged): + files = _files(m=_TWO_CUBE_CALC.replace("{SQL}", sql)) + _, issues = convert_cube_to_ossie(files) + assert bool(issues.of_type(IssueType.FANOUT_UNSAFE_METRIC)) is flagged + if flagged: + with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): + convert_cube_to_ossie(files, strict_fanout=True) + + +@pytest.mark.parametrize("expr,datasets,unqualified", [ + # Both kinds of operand, reported independently. + ("SUM(amount + line_items.qty)", {"line_items"}, True), + ("SUM(line_items.qty)", {"line_items"}, False), + ("SUM(amount)", set(), True), + # No columns at all is read as the declaring cube, which is what `COUNT(*)` means. + ("COUNT(*)", set(), True), + # An ordered-set aggregate's column is found on the wrapper. + ("PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY users.ltv)", {"users"}, False), + # An unrecognized call is treated as an aggregate: a warning if it is not one is + # cheaper than a silently inflated number if it is. + ("MY_UDF(users.x)", {"users"}, False), + # A nested aggregate is one scope, not two, so the inner one does not also report + # the declaring cube. + ("SUM(MY_UDF(users.x))", {"users"}, False), + # Idempotent, so nothing is attributed. + ("BOOL_OR(users.flag)", set(), False), + ("SUM(DISTINCT users.ltv)", set(), False), +]) +def test_aggregate_attribution(expr, datasets, unqualified): + from ossie_cube.expressions import unsafe_aggregate_datasets + + assert unsafe_aggregate_datasets(expr) == (datasets, unqualified) + + +def test_a_quoted_geo_half_reference_still_inlines_its_sql(): + """A split geo half exists only in Ossie, so a reference to it has to be replaced by + the half's own SQL. The inline table was keyed by the normalized form alone, unlike + every other table, so an exact-quoted reference missed the substitution and came out + as a raw column of a name the database does not have.""" + from _util import to_cube_sql + + for reference in ('users.home_latitude', 'users."HOME_LATITUDE"', + 'users."home_latitude"'): + assert to_cube_sql(f"AVG({reference})", "users", {"home"}, + inline_sql={"home_latitude": "{CUBE}.lat"}) == ( + "AVG({CUBE}.lat)") From c000c5aa9337d55207cfd55b3ecfc1ffd764ef9b Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Tue, 4 Aug 2026 23:44:23 +0500 Subject: [PATCH 37/46] Compare metric names case-insensitively; DISTINCT in unmodelled calls [P1] Model-level metric names were compared as exact strings, but Ossie regular identifiers are case-insensitive (core-spec/expression_language.md:73). So `revenue` on one cube and `Revenue` on another counted as two distinct names and both were emitted unqualified -- a document a consumer may reject or resolve to the wrong metric. Worse, nothing downstream catches it: the spec's own validator compares exact strings too, so its duplicate check passes and the test gate built on it could not see this class at all. Both the collision count and the derived-name check now normalize; the emitted name keeps its original spelling, so `orders__revenue` and `users__Revenue` come out qualified and still spelled as written. Also the follow-up flagged as non-blocking, since it was two lines: DISTINCT now counts for a call SQL parsing does not model, so `LISTAGG(DISTINCT name)` is idempotent for the same reason `SUM(DISTINCT x)` is. `Anonymous` keeps its arguments somewhere the modelled nodes do not, which is why the existing check missed them. 549 tests with both gates, 524 with neither, 96% coverage. Interop unchanged. --- converters/cube/README.md | 2 +- converters/cube/src/ossie_cube/cube_to_osi.py | 23 ++++++++---- converters/cube/src/ossie_cube/expressions.py | 14 +++++--- converters/cube/tests/test_cube_to_osi.py | 35 +++++++++++++++++++ converters/cube/tests/test_edge_cases.py | 15 ++++++++ 5 files changed, 78 insertions(+), 11 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index 844885c2..5e4334b3 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -140,7 +140,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | — | `type: geo` dimension | An Ossie field holds one expression and a geo dimension has two, so it **splits** into `_latitude` / `_longitude` (`Float`). Reconstruction data rides on the latitude half. See [Geo dimensions](#geo-dimensions). | | relationship | `joins[]` on a cube | `many_to_one` on cube A -> `from: A`(many), `to: B`(one). `one_to_many` is flipped so Ossie's `from` is the many side; the declared side and type are stashed so export restores the original. | | `from_columns` / `to_columns` | join `sql` | Only an AND-chain of equalities mapping to **physical columns of that dataset** converts. `{CUBE}.user_id` is already one; `{CUBE.user_key}` names a *member*, so it resolves to the column that member reads (`user_id`), following a chain of member references to its end with cycle detection. Everything else preserves the whole join verbatim in the stash rather than describing it wrongly: a member reading an expression (`CONCAT(...)`), a `case`/`switch` dimension (which reads no column at all), an alias belonging to another cube (`{users}.region_id`), or a clause that is not a two-column equality. | -| metric | `measures[]` on the cube its expression references | Import hoists cube-scoped measures to the model level, qualifying a colliding name as `__` and stashing the original name and owning cube. | +| metric | `measures[]` on the cube its expression references | Import hoists cube-scoped measures to the model level, qualifying a colliding name as `__` and stashing the original name and owning cube. Collision is judged on the **normalized** identifier, since Ossie's are case-insensitive: `orders.revenue` and `users.Revenue` are one name in the model-level namespace and both get qualified. The emitted name keeps its original spelling. | | `SUM`/`AVG`/`MIN`/`MAX(x)` | `type: sum`/`avg`/`min`/`max` + `sql` | | | `COUNT(DISTINCT x)` | `type: count_distinct` | | | `APPROX_COUNT_DISTINCT(x)` | `type: count_distinct_approx` | Cube resolves the warehouse-specific function itself. | diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index aac2de26..2383e8bf 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -51,6 +51,7 @@ filtered_operand, is_simple_identifier, lookup_map, + normalize_identifier, resolve_identifier, referenced_datasets, join_source, @@ -1318,10 +1319,16 @@ def _convert_measures(cubes, pk_by_cube, plain_by_cube, fanned_out, issues): issues=issues) resolver = context.resolver + # Counted by *normalized* name: Ossie regular identifiers are case-insensitive, so + # `revenue` on one cube and `Revenue` on another are one name in the model-level + # metric namespace. Counting them separately emitted both unqualified, which is a + # document a consumer may reject or resolve to the wrong metric -- and which the + # spec's own validator misses, since its duplicate check compares exact strings. counts = {} for (cname, mname), measure in resolver.measures().items(): if not _is_generated_part(measure): - counts[mname] = counts.get(mname, 0) + 1 + key = normalize_identifier(mname) + counts[key] = counts.get(key, 0) + 1 metrics = [] extra_measures = {} @@ -1338,12 +1345,16 @@ def _convert_measures(cubes, pk_by_cube, plain_by_cube, fanned_out, issues): # references inline back to the whole expression -- and export # regenerates it, so it is not stashed either. continue - metric_name = mname if counts[mname] == 1 else f"{cname}__{mname}" - if metric_name in seen: + unique = counts[normalize_identifier(mname)] == 1 + # The emitted name keeps its original spelling; only the *comparison* is + # normalized. + metric_name = mname if unique else f"{cname}__{mname}" + derived = normalize_identifier(metric_name) + if derived in seen: raise ConversionError( - f"metric name '{metric_name}' derived twice; rename the " - f"colliding measures in Cube") - seen.add(metric_name) + f"metric name '{metric_name}' derived twice (Ossie identifiers are " + f"case-insensitive); rename the colliding measures in Cube") + seen.add(derived) metric = _convert_measure(cname, mname, metric_name, measure, context) if metric is not None: metrics.append(metric) diff --git a/converters/cube/src/ossie_cube/expressions.py b/converters/cube/src/ossie_cube/expressions.py index b2cfc0cb..59a73391 100644 --- a/converters/cube/src/ossie_cube/expressions.py +++ b/converters/cube/src/ossie_cube/expressions.py @@ -211,7 +211,10 @@ def is_idempotent_aggregate(node): # An ordered-set aggregate is exactly as safe as the function being ordered. return bool(node.this) and is_idempotent_aggregate(node.this) if isinstance(node, exp.Anonymous): - return str(node.this or "").upper() in _IDEMPOTENT_CALLS + # DISTINCT applies here for the same reason it does to a modelled aggregate: + # `LISTAGG(DISTINCT name)` cannot be changed by a duplicated row. + return (str(node.this or "").upper() in _IDEMPOTENT_CALLS + or _aggregates_distinct(node)) if isinstance(node, _IDEMPOTENT_NODES): return True # DISTINCT collapses duplicates before the aggregate sees them, so *any* aggregate @@ -227,9 +230,12 @@ def _aggregates_distinct(node): return True if node.args.get("distinct"): return True - # sqlglot may hang the DISTINCT off the function's argument list instead. - return any(isinstance(arg, exp.Distinct) - for arg in (node.args.get("expressions") or [])) + # sqlglot may hang the DISTINCT off the argument list instead -- and for a call it + # does not model, `Anonymous.expressions` is where the arguments live. + arguments = list(node.args.get("expressions") or []) + if isinstance(node.this, list): + arguments += node.this + return any(isinstance(arg, exp.Distinct) for arg in arguments) def unsafe_aggregate_datasets(expr): diff --git a/converters/cube/tests/test_cube_to_osi.py b/converters/cube/tests/test_cube_to_osi.py index 15f6f26d..c1e60619 100644 --- a/converters/cube/tests/test_cube_to_osi.py +++ b/converters/cube/tests/test_cube_to_osi.py @@ -545,3 +545,38 @@ def test_reference_translation_in_a_field_context(sql, expected): def test_reference_translation_in_a_metric_context(sql, expected): """A metric expression is model-level, so own-cube references are qualified.""" assert cube_sql_to_ossie(sql, "orders", self_prefix="orders")[0] == expected + + +def _two_cube_measures(first, second): + return {"model/cubes/m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: a.b.orders\n" + " dimensions:\n" + " - name: id\n sql: id\n type: number\n" + " primary_key: true\n" + f" measures:\n - name: {first}\n sql: amount\n type: sum\n" + " - name: users\n" + " sql_table: a.b.users\n" + " dimensions:\n" + " - name: id\n sql: id\n type: number\n" + " primary_key: true\n" + f" measures:\n - name: {second}\n sql: ltv\n type: sum\n")} + + +@pytest.mark.parametrize("first,second,expected", [ + # Ossie regular identifiers are case-insensitive, so these are *one* name in the + # model-level metric namespace and both have to be qualified. Comparing exact strings + # emitted `revenue` and `Revenue` side by side -- a document a consumer may reject or + # resolve to the wrong metric, and one the spec's own validator passes because its + # duplicate check is exact too. + ("revenue", "Revenue", ["orders__revenue", "users__Revenue"]), + ("revenue", "revenue", ["orders__revenue", "users__revenue"]), + ("REVENUE", "revenue", ["orders__REVENUE", "users__revenue"]), + # Genuinely distinct names stay unqualified. + ("revenue", "lifetime", ["revenue", "lifetime"]), +]) +def test_metric_name_collisions_are_detected_case_insensitively(first, second, expected): + out, _ = convert_cube_to_ossie(_two_cube_measures(first, second)) + # The comparison is normalized; the emitted name keeps its original spelling. + assert [m["name"] for m in model_of(out)["metrics"]] == expected diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index cdd67a2b..8c662ddd 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -2420,3 +2420,18 @@ def test_a_quoted_geo_half_reference_still_inlines_its_sql(): assert to_cube_sql(f"AVG({reference})", "users", {"home"}, inline_sql={"home_latitude": "{CUBE}.lat"}) == ( "AVG({CUBE}.lat)") + + +@pytest.mark.parametrize("expr,unsafe", [ + # DISTINCT applies to a call SQL parsing does not model, for the same reason it + # applies to a modelled aggregate: a duplicated row cannot change the distinct set. + ("LISTAGG(DISTINCT users.name, ',')", False), + ("LISTAGG(users.name, ',')", True), + ("APPROX_PERCENTILE(DISTINCT users.x, 0.5)", False), + ("APPROX_PERCENTILE(users.x, 0.5)", True), +]) +def test_distinct_inside_an_unmodelled_call_is_idempotent(expr, unsafe): + from ossie_cube.expressions import unsafe_aggregate_datasets + + datasets, unqualified = unsafe_aggregate_datasets(expr) + assert bool(datasets or unqualified) is unsafe From 77943d6d2e2cd355b47db54743c57e5fb26c6a65 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Wed, 5 Aug 2026 00:20:11 +0500 Subject: [PATCH 38/46] Make another converter's output usable: dialect fallback and unique_keys as key Found by running the path we care about most end to end -- a Databricks metric view through Ossie into Cube, and back -- which nothing had exercised. Both directions were broken, and both failed *quietly*. - Every expression the Databricks converter emits is `DATABRICKS` with no ANSI_SQL alternative, and export required ANSI. So every field and metric was dropped and the result was an empty Cube model -- which Cube compiles, so neither the round-trip tests nor the compile gate saw anything wrong. Export now falls back to the expression's only dialect when that dialect is warehouse SQL, and reports it. MDX/TABLEAU/MAQL are not SQL a warehouse runs, so those still drop. - Cube refuses a cube that declares a join without a primary key, and a Databricks metric view has no primary-key concept. `unique_keys` identifies a row just as well and was already in the document -- parked in `meta.ossie` while Cube rejected the model for want of exactly it. It is now used as the key, and a dataset with a relationship and neither is reported with Cube's own wording, since nothing can be invented. With both, a metric view survives the full loop: source, joins, dimensions and measures all come back with the same names, and the Cube model compiles. Also a bug in the compile gate itself: it flattened model paths to basenames, so a cube and a view of the same name overwrote each other and a valid model looked malformed. That is exactly the shape this fixture has, since the Ossie model and its fact table share a name. `tests/fixtures/databricks_ossie.yaml` pins the path in this suite -- a document written by another converter, so nothing in it was shaped for Cube. 555 tests with both gates, 529 with neither, 96% coverage. --- converters/cube/README.md | 10 ++- converters/cube/src/ossie_cube/_common.py | 50 ++++++++---- converters/cube/src/ossie_cube/osi_to_cube.py | 71 +++++++++++++++-- converters/cube/tests/_cube_gate.py | 6 +- .../cube/tests/fixtures/databricks_ossie.yaml | 79 +++++++++++++++++++ converters/cube/tests/test_osi_to_cube.py | 68 ++++++++++++++++ converters/cube/tests/test_roundtrip.py | 14 ++++ 7 files changed, 273 insertions(+), 25 deletions(-) create mode 100644 converters/cube/tests/fixtures/databricks_ossie.yaml diff --git a/converters/cube/README.md b/converters/cube/README.md index 5e4334b3..21b8311d 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -124,7 +124,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | `dataset.description` | cube `description` | | | `dataset.ai_context` | cube `meta.ai_context` | Preserved for the round trip, but **inert in Cube** -- its agent ignores cube-level `ai_context`. Recorded as an issue. | | `dataset.primary_key` | dimension(s) with `primary_key: true` | Composite = several. A Cube key can be an *expression*, and then the only name Ossie can carry is the dimension's -- which the Ossie document alone cannot tell apart from a column name afterwards, so import records it (`computed_primary_key`) and export puts `primary_key: true` back on that dimension instead of synthesizing one that reads a column of that name. Export marks a dimension only when it is **scalar** — a single source column — since `primary_key: true` declares that dimension's own `sql` to be the key; a computed dimension or a merged `geo` one would declare the wrong thing even if its name matches. Anything left uncovered becomes a `public: false` scalar dimension, suffixed (`id_pk`) if the obvious name is taken. | -| `dataset.unique_keys` | `meta.ossie.unique_keys` | No native Cube slot; parked rather than dropped. | +| `dataset.unique_keys` | `meta.ossie.unique_keys` | No native Cube slot, so parked — and used as the Cube primary key when the dataset declares none. Cube refuses a cube that declares a join without one, and several source formats have no primary-key concept: a Databricks metric view does not. A dataset with a relationship and neither is reported, naming Cube's requirement, since nothing can be invented. | | field | `dimensions[]` entry | Export: a name that is not a valid Cube identifier is sanitized; a case-insensitive collision is an error, never a silent merge. | | `field.expression` | dimension `sql` | Dataset-scoped, so `{CUBE}.col` <-> `col`. Export emits `{CUBE}.column` for a raw column and `{CUBE.member}` for a declared member, and never spells the cube's own name (which would break under `extends`). | | `field.datatype` | dimension `type` (**required**) | `String`->`string`, `Boolean`->`boolean`, `Date`/`Time`/`DateTime`/`DateTimeTz`->`time`, `Integer`/`Decimal`/`Float`->`number`, `Opaque`->`string`. Import maps back, choosing `Decimal` for `number` -- Cube collapses three Ossie types into one, so any single answer is a guess, and a stated datatype is what another converter can act on. Export parks the exact one in `meta.ossie.datatype`, which import prefers when present, so `Integer` and `Float` still survive a round trip. | @@ -181,6 +181,14 @@ Ossie dialect enum has no `CUBE` entry -- so import emits `ANSI_SQL`, and export prefers `ANSI_SQL` with `--dialect` prepending a warehouse dialect (e.g. `SNOWFLAKE` for a Snowflake-backed Cube model). +Failing both, export falls back to the expression's **only** dialect when that dialect +is warehouse SQL (`SNOWFLAKE`, `DATABRICKS`, `BIGQUERY`), and reports it. This is what +makes another converter's output usable: everything the Databricks converter emits is +`DATABRICKS` with no ANSI alternative, and requiring ANSI dropped every field and metric +— producing an *empty* Cube model, which Cube compiles, so nothing downstream noticed. +`MDX`, `TABLEAU` and `MAQL` are query or calculation languages rather than warehouse SQL, +so those still drop. + **Braces are escaped in free text.** Cube compiles *every* string in a YAML model as a Python f-string (`f""` in `YamlCompiler`; only the handful of boolean-ish keys in the compiler's `nonStringFields` are exempt), so an unescaped `{` in a description, diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index d223fd85..beb3ba88 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -42,6 +42,12 @@ # the caller prepend a warehouse dialect the actual data source would accept. DIALECT_ANSI = "ANSI_SQL" +# Dialects whose expressions are SQL a warehouse executes, so Cube can pass them +# straight to the data source. The spec's enum also contains MDX, TABLEAU and MAQL, +# which are query or calculation languages rather than warehouse SQL -- an expression +# in one of those is not usable as a Cube `sql` at all. +WAREHOUSE_DIALECTS = frozenset({DIALECT_ANSI, "SNOWFLAKE", "DATABRICKS", "BIGQUERY"}) + # Bump when the shape of a stashed `data` blob changes. STASH_VERSION = 1 @@ -301,23 +307,37 @@ def foreign_vendor_extensions(obj): # --- expressions ---------------------------------------------------------------- def pick_expression(ossie_expression, preferred=None): - """Choose the SQL string for an Ossie expression. + """Choose the SQL string for an Ossie expression. Returns (sql, dialect). + + Preference order: the caller-chosen warehouse dialect (Cube passes SQL through to + the data source, so e.g. SNOWFLAKE SQL is valid on a Snowflake-backed Cube model), + then ANSI_SQL, then -- if the expression offers exactly one dialect and that dialect + is warehouse SQL -- that one. - Preference order: the caller-chosen warehouse dialect (Cube passes SQL - through to the data source, so e.g. SNOWFLAKE SQL is valid on a - Snowflake-backed Cube model), then ANSI_SQL. Returns None if neither is - present (the caller records an issue and skips). + The last step matters for real interop. Converters commonly emit their own dialect + and no ANSI: everything from the Databricks converter is `DATABRICKS`. Requiring + ANSI meant a Databricks-authored model exported to an *empty* Cube model, every + field and metric dropped. + + Only `WAREHOUSE_DIALECTS` qualify. An expression in MDX, TABLEAU or MAQL is not SQL + a warehouse can run, so there is nothing to fall back *to* -- those still drop, with + the issue saying so. `(None, None)` means nothing usable was found. """ - dialects = { - d.get("dialect"): d.get("expression") - for d in (ossie_expression or {}).get("dialects") or [] - } - expr = None - if preferred: - expr = dialects.get(preferred) - if expr is None: - expr = dialects.get(DIALECT_ANSI) - if expr is not None and not isinstance(expr, str): + dialects = [(d.get("dialect"), d.get("expression")) + for d in (ossie_expression or {}).get("dialects") or [] + if d.get("expression") is not None] + by_dialect = dict(dialects) + for candidate in (preferred, DIALECT_ANSI): + if candidate and candidate in by_dialect: + return _checked_expression(by_dialect[candidate]), candidate + if len(dialects) == 1 and dialects[0][0] in WAREHOUSE_DIALECTS: + dialect, expr = dialects[0] + return _checked_expression(expr), dialect + return None, None + + +def _checked_expression(expr): + if not isinstance(expr, str): raise ConversionError( f"expression must be a string, got {type(expr).__name__}") return expr diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 5eaa9a69..36a3d7ca 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -39,6 +39,7 @@ AGG_TO_RESULT_DATATYPE, DATATYPE_TO_DIM_TYPE, DEFAULT_DATATYPE_FOR_CUBE_TYPE, + DIALECT_ANSI, OSSIE_FUNC_TO_AGG, OSSIE_VERSION, ConversionError, @@ -320,7 +321,9 @@ def _build_cube(ds, plan, tables, joins, measures, join_extensions, dialect, pk_names = [] computed_keys = set(stash.get("computed_primary_key") or []) taken = {d["name"].lower() for d in dimensions} - for entry in (ds.get("primary_key") or []): + # `plan.primary_key`, not the dataset's own field: the plan is where the fallback to + # `unique_keys` is resolved, and every stage has to agree on the answer. + for entry in plan.primary_key: entry = str(entry) # Import records the *dimension name*, so the name match is checked first; # a hand-authored model naming the source column resolves by column. @@ -350,6 +353,17 @@ def _build_cube(ds, plan, tables, joins, measures, join_extensions, dialect, for dim in dimensions: if dim["name"] in pk_names: dim["primary_key"] = True + if plan.key_from_unique_keys and pk_names: + issues.add(IssueType.APPROXIMATED, scope, + f"no primary_key, so the first unique_keys entry " + f"({', '.join(plan.primary_key)}) is marked as the Cube primary key; " + f"Cube requires one on any cube that declares a join") + if joins and not pk_names: + issues.add(IssueType.DROPPED_NO_CUBE_EQUIVALENT, scope, + "declares a relationship but no primary_key or unique_keys, and Cube " + "requires a primary key on any cube with a join ('primary key for " + " is required when join is defined') -- the model will not " + "compile until the dataset declares one") # Dimensions a prior import could not express as an Ossie field (a `switch` one, # which has no sql) go back at their original positions. @@ -438,11 +452,13 @@ class _CubePlan: members: frozenset dropped: frozenset primary_key: tuple + key_from_unique_keys: bool @classmethod def of(cls, ds, cname, dialect, scope): names, inline_sql = _resolve_dimension_names(ds, scope) stash = read_stash(ds) + key, from_unique = _primary_key_columns(ds) lookup = dict(names) lookup.update({dname: dname for dname in names.values()}) return cls( @@ -461,10 +477,30 @@ def of(cls, ds, cname, dialect, scope): if (item.get("measure") or {}).get("name")} | _stashed_segment_names(stash)), dropped=frozenset(_undialected_fields(ds, dialect)), - primary_key=tuple(str(c) for c in (ds.get("primary_key") or [])), + primary_key=key, + key_from_unique_keys=from_unique, ) +def _primary_key_columns(ds): + """(key columns, whether they came from `unique_keys`). + + Cube needs a primary key on any cube that declares a join, and several source + formats have no primary-key concept at all -- a Databricks metric view does not -- + so the converter falls back to the first `unique_keys` entry, which identifies a row + just as well. Without that fallback the information sat parked in `meta.ossie` while + Cube refused the model for want of exactly it. + """ + declared = [str(c) for c in (ds.get("primary_key") or [])] + if declared: + return tuple(declared), False + for candidate in (ds.get("unique_keys") or []): + columns = [str(c) for c in (candidate or [])] + if columns: + return tuple(columns), True + return (), False + + def _reference_members(ds, dim_names, dialect): """Members that must be addressed as `{CUBE.member}` rather than `{CUBE}.column`. @@ -479,7 +515,7 @@ def _reference_members(ds, dim_names, dialect): dname = dim_names.get(fname) if not dname: continue - expr = pick_expression(field.get("expression"), dialect) + expr, _ = pick_expression(field.get("expression"), dialect) if expr is None: # No usable dialect: this field becomes no dimension at all, so claiming # it as a member would make a metric over it emit `{CUBE.name}` -- @@ -494,11 +530,26 @@ def _reference_members(ds, dim_names, dialect): return needed +def _report_dialect_fallback(issues, scope, used, preferred): + """Note when an expression came from a dialect that was not asked for. + + Emitted rather than dropped: a model whose only expressions are `DATABRICKS` still + converts, and Cube passes SQL through to the data source, so it is right whenever the + Cube model reads that warehouse. Pass `--dialect` to make the choice explicit. + """ + if used in (None, DIALECT_ANSI, preferred): + return + issues.add(IssueType.APPROXIMATED, scope, + f"no ANSI_SQL expression; used the only dialect on offer ('{used}'). " + f"Cube passes SQL to the data source, so this is correct where the " + f"model reads that warehouse -- pass --dialect {used} to say so.") + + def _undialected_fields(ds, dialect): """Field names with no expression in a usable dialect, so no Cube dimension.""" return {field.get("name") for field in (ds.get("fields") or []) if field.get("name") - and pick_expression(field.get("expression"), dialect) is None} + and pick_expression(field.get("expression"), dialect)[0] is None} def _resolve_dimension_names(ds, scope): @@ -602,11 +653,13 @@ def _build_dimensions(ds, plan, tables, dialect, issues): slot["host"] = geo["host"] continue - expr = pick_expression(field.get("expression"), dialect) + expr, used = pick_expression(field.get("expression"), dialect) if expr is None: issues.add(IssueType.NO_USABLE_DIALECT, f"{ds_name}.{fname}", - "no ANSI_SQL or preferred-dialect expression; field dropped") + "no ANSI_SQL expression and no warehouse dialect Cube could " + "pass through; field dropped") continue + _report_dialect_fallback(issues, f"{ds_name}.{fname}", used, dialect) dim = {"name": dname} if "sql" in stash: @@ -897,11 +950,13 @@ def resolve_base(): _place(measures_by_cube, target, measure, name) continue - expr = pick_expression(metric.get("expression"), dialect) + expr, used = pick_expression(metric.get("expression"), dialect) if expr is None: issues.add(IssueType.NO_USABLE_DIALECT, scope, - "no ANSI_SQL or preferred-dialect expression; metric dropped") + "no ANSI_SQL expression and no warehouse dialect Cube could " + "pass through; metric dropped") continue + _report_dialect_fallback(issues, scope, used, dialect) missing = _references_a_dropped_field(expr, tables, cube_names, plan) if missing: diff --git a/converters/cube/tests/_cube_gate.py b/converters/cube/tests/_cube_gate.py index 5ed95b03..e22121b9 100644 --- a/converters/cube/tests/_cube_gate.py +++ b/converters/cube/tests/_cube_gate.py @@ -68,7 +68,11 @@ def assert_cube_compiles(files, label=""): # A `.js`/`.ts` model needs Cube's transpiler and a `.py` one is # Jinja-driven; the converter preserves both without parsing them. continue - dest = pathlib.Path(tmp) / pathlib.Path(name).name + # The relative directory is kept. Flattening to the basename let a cube and + # a view of the same name overwrite each other -- `model/cubes/orders.yml` + # and `model/views/orders.yml` -- which made a valid model look malformed. + dest = pathlib.Path(tmp) / name + dest.parent.mkdir(parents=True, exist_ok=True) dest.write_text(text) paths.append(str(dest)) assert paths, f"{label}: nothing to compile" diff --git a/converters/cube/tests/fixtures/databricks_ossie.yaml b/converters/cube/tests/fixtures/databricks_ossie.yaml new file mode 100644 index 00000000..9eb1328f --- /dev/null +++ b/converters/cube/tests/fixtures/databricks_ossie.yaml @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# An Ossie model as the Databricks converter emits one, which differs from a +# Cube-authored document in two ways that matter here: +# +# - every expression is `DATABRICKS`, with no ANSI_SQL alternative, so requiring ANSI +# dropped every field and metric and produced an empty Cube model; +# - a Databricks metric view has no primary-key concept, and Cube requires one on any +# cube that declares a join -- so the key comes from `unique_keys`. +# +# Generated by `ossie-databricks import` from that converter's fixtureA metric view, with +# a `unique_keys` added to the fact table, which is what a metric-view author has to +# supply for the model to reach Cube at all. + +version: 0.2.0.dev0 +semantic_model: +- name: orders + description: Sales orders with customer attributes + datasets: + - name: orders + source: samples.tpch.orders + fields: + - name: o_orderkey + expression: + dialects: + - dialect: DATABRICKS + expression: o_orderkey + description: Order identifier + - name: o_orderdate + expression: + dialects: + - dialect: DATABRICKS + expression: o_orderdate + label: Order Date + ai_context: + synonyms: + - order date + - date + unique_keys: + - - o_orderkey + - name: customer + source: samples.tpch.customer + unique_keys: + - - c_custkey + fields: + - name: c_name + expression: + dialects: + - dialect: DATABRICKS + expression: c_name + description: Customer name + relationships: + - name: orders_to_customer + from: orders + to: customer + from_columns: + - o_custkey + to_columns: + - c_custkey + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "rely": {"at_most_one_match": true}}' + metrics: + - name: total_revenue + expression: + dialects: + - dialect: DATABRICKS + expression: SUM(o_totalprice) + description: Total order revenue + ai_context: + synonyms: + - revenue + - total revenue + - sales + - name: order_count + expression: + dialects: + - dialect: DATABRICKS + expression: COUNT(*) + description: Number of orders diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index d10804c5..32c1d4fe 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -906,3 +906,71 @@ def test_a_mapping_form_segment_is_counted_when_disambiguating_a_view(): entry = parse(files["model/views/shop.yml"])["views"][0]["cubes"][1] assert entry["excludes"] == ["id"] assert issues.of_type(IssueType.APPROXIMATED) + + +# --- a model from another converter ----------------------------------------------- +# +# Everything else here starts from a Cube model or from Ossie written for this test +# suite. A document another converter produced is shaped differently, and the two +# differences below both used to end the conversion in silence. + +def _databricks_ossie(): + from _util import load_fixture + + return load_fixture("databricks_ossie.yaml") + + +def test_a_model_with_only_a_warehouse_dialect_still_converts(): + """The Databricks converter emits `DATABRICKS` and no ANSI_SQL. Requiring ANSI meant + every field and metric was dropped and the export was an *empty* Cube model -- which + Cube compiles, so nothing downstream noticed either.""" + files, issues = convert_ossie_to_cube(_databricks_ossie()) + cube = _cubes(files)["orders"] + assert [d["name"] for d in cube["dimensions"]] == ["o_orderkey", "o_orderdate"] + assert [m["name"] for m in cube["measures"]] == ["total_revenue", "order_count"] + # Reported, because Cube will pass that SQL to whatever the data source is. + assert any("only dialect on offer" in i.detail + for i in issues.of_type(IssueType.APPROXIMATED)) + + +def test_a_non_sql_dialect_is_still_not_usable(): + """The fallback is to warehouse SQL only. MDX, TABLEAU and MAQL are query or + calculation languages, so there is nothing for Cube to pass through.""" + ds = ( + " - name: orders\n" + " source: shop.public.orders\n" + " fields:\n" + " - name: note\n expression:\n dialects:\n" + " - dialect: TABLEAU\n expression: note\n" + ) + files, issues = convert_ossie_to_cube(_ossie(ds)) + assert "dimensions" not in _cubes(files)["orders"] + assert issues.of_type(IssueType.NO_USABLE_DIALECT) + + +def test_unique_keys_supply_the_primary_key_cube_requires_for_a_join(): + """Cube refuses a cube that declares a join without a primary key, and several source + formats have no primary-key concept -- a Databricks metric view does not. The first + `unique_keys` entry identifies a row just as well, and was sitting parked in + `meta.ossie` while Cube rejected the model for want of exactly it.""" + files, issues = convert_ossie_to_cube(_databricks_ossie()) + keys = [d for d in _cubes(files)["orders"]["dimensions"] if d.get("primary_key")] + assert [d["sql"] for d in keys] == ["o_orderkey"] + assert any("unique_keys entry" in i.detail + for i in issues.of_type(IssueType.APPROXIMATED)) + # And it is still parked, so the round trip keeps it. + assert _cubes(files)["orders"]["meta"]["ossie"]["unique_keys"] == [["o_orderkey"]] + + +def test_a_join_with_no_key_at_all_says_what_cube_will_refuse(): + """Nothing can be invented here, so the issue names Cube's requirement and the + remedy rather than leaving a model that quietly will not load.""" + import yaml as _yaml + + doc = _yaml.safe_load(_databricks_ossie()) + for ds in doc["semantic_model"][0]["datasets"]: + ds.pop("unique_keys", None) + _, issues = convert_ossie_to_cube(_yaml.dump(doc, sort_keys=False)) + dropped = issues.of_type(IssueType.DROPPED_NO_CUBE_EQUIVALENT) + assert any("requires a primary key on any cube with a join" in i.detail + for i in dropped) diff --git a/converters/cube/tests/test_roundtrip.py b/converters/cube/tests/test_roundtrip.py index b309cea5..5b89ec6a 100644 --- a/converters/cube/tests/test_roundtrip.py +++ b/converters/cube/tests/test_roundtrip.py @@ -111,6 +111,20 @@ def test_the_fixture_and_its_round_trip_both_compile_in_cube(fixture): assert_cube_compiles(back, f"{fixture} (after a round trip)") +@validator_gate +def test_a_model_from_another_converter_is_valid_ossie(): + assert_ossie_is_valid(load_fixture("databricks_ossie.yaml"), "databricks_ossie.yaml") + + +@cube_gate +def test_a_model_from_another_converter_exports_to_a_model_cube_accepts(): + """The Databricks path end to end. Nothing here was written for Cube: the dialect is + `DATABRICKS` throughout and the primary key comes from `unique_keys`, because a + metric view has no primary-key concept and Cube demands one for a join.""" + files, _ = convert_ossie_to_cube(load_fixture("databricks_ossie.yaml")) + assert_cube_compiles(files, "databricks_ossie.yaml") + + @cube_gate def test_a_hand_authored_ossie_model_exports_to_a_model_cube_accepts(): """Nothing here came from Cube, so nothing is restored from a stash -- every key is From 942d3f60360ddea821eeea407f0afdedad9c448c Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Wed, 5 Aug 2026 00:32:08 +0500 Subject: [PATCH 39/46] Emit the dimension role block on every dimension Found the same way as the Databricks issues: by checking what a spoke actually made of our output rather than that it exited zero. Cube -> Ossie -> Snowflake produced a Cortex Analyst model with **zero dimensions and 27 facts** across TPC-DS -- every categorical column classified as a numeric measure. The cause is on our side. The Snowflake converter classifies "a field with no `dimension` block as a fact regardless of datatype", which is a fair reading: the block is the role marker. Import emitted it only for time dimensions, so everything else looked like a fact. A Cube `dimensions:` entry is a dimension by definition, so the block is now always emitted -- empty for a non-time one, which leaves the consumer to apply the spec's own default instead of this converter asserting `is_time: false`. Snowflake output for the same model, before -> after: store_sales dim=0 fact=9 -> dim=9 fact=0 customer dim=0 fact=6 -> dim=6 fact=0 date_dim dim=0 time=3 fact=2 -> dim=2 time=3 fact=0 which matches the shape of that converter's own committed example. No other spoke's result changed. Also covers the geo halves, whose fields are built on a separate path; and pins that the dialect fallback is not Databricks-specific -- Snowflake and BigQuery alone convert too. The two Ossie snapshot fixtures are regenerated. 559 tests with both gates, 533 with neither, 96% coverage. --- converters/cube/README.md | 1 + converters/cube/src/ossie_cube/cube_to_osi.py | 13 +++++++-- .../cube/tests/fixtures/fixtureA_ossie.yaml | 8 +++++ .../cube/tests/fixtures/tpcds_ossie.yaml | 29 +++++++++++++++++++ converters/cube/tests/test_cube_to_osi.py | 18 ++++++++++++ converters/cube/tests/test_osi_to_cube.py | 22 ++++++++++++++ 6 files changed, 89 insertions(+), 2 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index 21b8311d..765b7410 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -128,6 +128,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | field | `dimensions[]` entry | Export: a name that is not a valid Cube identifier is sanitized; a case-insensitive collision is an error, never a silent merge. | | `field.expression` | dimension `sql` | Dataset-scoped, so `{CUBE}.col` <-> `col`. Export emits `{CUBE}.column` for a raw column and `{CUBE.member}` for a declared member, and never spells the cube's own name (which would break under `extends`). | | `field.datatype` | dimension `type` (**required**) | `String`->`string`, `Boolean`->`boolean`, `Date`/`Time`/`DateTime`/`DateTimeTz`->`time`, `Integer`/`Decimal`/`Float`->`number`, `Opaque`->`string`. Import maps back, choosing `Decimal` for `number` -- Cube collapses three Ossie types into one, so any single answer is a guess, and a stated datatype is what another converter can act on. Export parks the exact one in `meta.ossie.datatype`, which import prefers when present, so `Integer` and `Float` still survive a round trip. | +| `field.dimension` | a `dimensions[]` entry | Import always emits the block, because a Cube `dimensions:` entry *is* a dimension and the block's absence is what other converters read as "not one" — the Snowflake converter classifies a field without it as a fact regardless of datatype, so omitting it made every non-time dimension a Cortex Analyst fact. Left empty for a non-time dimension, so the consumer applies the spec's default rather than this converter asserting `is_time: false`. | | `field.dimension.is_time` | `type: time` | Import sets `is_time: true` for a time dimension. A field carrying `is_time` but *no* `datatype` records the absence (`meta.ossie.untyped`), since the spec says not to infer a scalar type from `is_time` alone — otherwise `type: time` would come back asserting `DateTime`. | | `field.label` / `description` | dimension `title` / `description` | | | `field.ai_context.instructions` | dimension `meta.ai_context` | Cube's documented AI-only context field. | diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 2383e8bf..55645c9c 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -690,8 +690,14 @@ def _finish_dimension_field(cname, dname, dim, field, stash, issues): # With no datatype there is nothing to regenerate from, so the type is recorded. if DATATYPE_TO_DIM_TYPE.get(field.get("datatype")) != dtype: stash["dim_type"] = dtype - if dtype == "time": - field["dimension"] = {"is_time": True} + # Every Cube `dimensions:` entry is a dimension, so the role block is always + # emitted. Its *absence* is what other converters read as "not a dimension" -- the + # Snowflake converter classifies a field with no `dimension` block as a fact + # "regardless of datatype", so omitting it turned every non-time dimension into a + # Cortex Analyst fact. Left empty for a non-time one, which lets the consumer apply + # the spec's default (`is_time` false for a non-temporal datatype) rather than this + # converter asserting it. + field["dimension"] = {"is_time": True} if dtype == "time" else {} if dim.get("title"): field["label"] = unescape_braces_from_cube(dim["title"]) if dim.get("description"): @@ -782,6 +788,9 @@ def _convert_geo_dimension(cname, dname, dim, issues): "dialects": [{"dialect": DIALECT_ANSI, "expression": expr}] }, "datatype": "Float", + # A coordinate is a dimension like any other; this path builds its fields + # directly, so it needs the role block spelled out here too. + "dimension": {}, } geo = {"of": dname, "part": part, "sql": sub} if part == "latitude" and host_extras: diff --git a/converters/cube/tests/fixtures/fixtureA_ossie.yaml b/converters/cube/tests/fixtures/fixtureA_ossie.yaml index 577fd8ed..19435d23 100644 --- a/converters/cube/tests/fixtures/fixtureA_ossie.yaml +++ b/converters/cube/tests/fixtures/fixtureA_ossie.yaml @@ -44,18 +44,21 @@ semantic_model: - dialect: ANSI_SQL expression: id datatype: Decimal + dimension: {} - name: user_id expression: dialects: - dialect: ANSI_SQL expression: user_id datatype: Decimal + dimension: {} - name: status expression: dialects: - dialect: ANSI_SQL expression: status datatype: String + dimension: {} label: Order Status description: Current order status ai_context: @@ -74,6 +77,7 @@ semantic_model: - dialect: ANSI_SQL expression: amount > 500 datatype: Boolean + dimension: {} primary_key: - id - name: users @@ -85,18 +89,21 @@ semantic_model: - dialect: ANSI_SQL expression: id datatype: Decimal + dimension: {} - name: city expression: dialects: - dialect: ANSI_SQL expression: city datatype: String + dimension: {} - name: location_latitude expression: dialects: - dialect: ANSI_SQL expression: lat datatype: Float + dimension: {} custom_extensions: - vendor_name: CUBE data: '{"_v": 1, "geo": {"of": "location", "part": "latitude", "sql": "{CUBE}.lat"}}' @@ -106,6 +113,7 @@ semantic_model: - dialect: ANSI_SQL expression: lon datatype: Float + dimension: {} custom_extensions: - vendor_name: CUBE data: '{"_v": 1, "geo": {"of": "location", "part": "longitude", "sql": "{CUBE}.lon"}}' diff --git a/converters/cube/tests/fixtures/tpcds_ossie.yaml b/converters/cube/tests/fixtures/tpcds_ossie.yaml index 0b323fb9..682ac98d 100644 --- a/converters/cube/tests/fixtures/tpcds_ossie.yaml +++ b/converters/cube/tests/fixtures/tpcds_ossie.yaml @@ -75,6 +75,7 @@ semantic_model: - dialect: ANSI_SQL expression: ss_sold_date_sk datatype: Decimal + dimension: {} description: Foreign key to date dimension ai_context: synonyms: @@ -86,6 +87,7 @@ semantic_model: - dialect: ANSI_SQL expression: ss_item_sk datatype: Decimal + dimension: {} description: Foreign key to item dimension ai_context: synonyms: @@ -97,6 +99,7 @@ semantic_model: - dialect: ANSI_SQL expression: ss_customer_sk datatype: Decimal + dimension: {} description: Foreign key to customer dimension ai_context: synonyms: @@ -108,6 +111,7 @@ semantic_model: - dialect: ANSI_SQL expression: ss_store_sk datatype: Decimal + dimension: {} description: Foreign key to store dimension ai_context: synonyms: @@ -119,6 +123,7 @@ semantic_model: - dialect: ANSI_SQL expression: ss_quantity datatype: Decimal + dimension: {} description: Quantity of items sold ai_context: synonyms: @@ -130,6 +135,7 @@ semantic_model: - dialect: ANSI_SQL expression: ss_sales_price datatype: Decimal + dimension: {} description: Sales price per unit ai_context: synonyms: @@ -141,6 +147,7 @@ semantic_model: - dialect: ANSI_SQL expression: ss_ext_sales_price datatype: Decimal + dimension: {} description: Extended sales price (quantity * price) ai_context: synonyms: @@ -152,6 +159,7 @@ semantic_model: - dialect: ANSI_SQL expression: ss_net_profit datatype: Decimal + dimension: {} description: Net profit from the sale ai_context: synonyms: @@ -163,6 +171,7 @@ semantic_model: - dialect: ANSI_SQL expression: ss_ticket_number datatype: String + dimension: {} custom_extensions: - vendor_name: CUBE data: '{"_v": 1, "public": false}' @@ -186,6 +195,7 @@ semantic_model: - dialect: ANSI_SQL expression: d_date_sk datatype: Decimal + dimension: {} description: Surrogate key for date - name: d_date expression: @@ -206,6 +216,7 @@ semantic_model: - dialect: ANSI_SQL expression: d_year datatype: Decimal + dimension: {} description: Year ai_context: synonyms: @@ -254,6 +265,7 @@ semantic_model: - dialect: ANSI_SQL expression: c_customer_sk datatype: Decimal + dimension: {} description: Surrogate key for customer - name: c_customer_id expression: @@ -261,6 +273,7 @@ semantic_model: - dialect: ANSI_SQL expression: c_customer_id datatype: String + dimension: {} description: Business key for customer ai_context: synonyms: @@ -272,6 +285,7 @@ semantic_model: - dialect: ANSI_SQL expression: c_first_name datatype: String + dimension: {} description: Customer first name - name: c_last_name expression: @@ -279,6 +293,7 @@ semantic_model: - dialect: ANSI_SQL expression: c_last_name datatype: String + dimension: {} description: Customer last name - name: customer_full_name expression: @@ -286,6 +301,7 @@ semantic_model: - dialect: ANSI_SQL expression: c_first_name || ' ' || c_last_name datatype: String + dimension: {} description: Customer full name (computed field) ai_context: synonyms: @@ -297,6 +313,7 @@ semantic_model: - dialect: ANSI_SQL expression: c_email_address datatype: String + dimension: {} description: Customer email address ai_context: synonyms: @@ -321,6 +338,7 @@ semantic_model: - dialect: ANSI_SQL expression: i_item_sk datatype: Decimal + dimension: {} description: Surrogate key for item - name: i_item_id expression: @@ -328,6 +346,7 @@ semantic_model: - dialect: ANSI_SQL expression: i_item_id datatype: String + dimension: {} description: Business key for item ai_context: synonyms: @@ -340,6 +359,7 @@ semantic_model: - dialect: ANSI_SQL expression: i_item_desc datatype: String + dimension: {} description: Item description ai_context: synonyms: @@ -351,6 +371,7 @@ semantic_model: - dialect: ANSI_SQL expression: i_brand datatype: String + dimension: {} description: Brand name ai_context: synonyms: @@ -362,6 +383,7 @@ semantic_model: - dialect: ANSI_SQL expression: i_category datatype: String + dimension: {} description: Item category ai_context: synonyms: @@ -373,6 +395,7 @@ semantic_model: - dialect: ANSI_SQL expression: i_current_price datatype: Decimal + dimension: {} description: Current price of the item ai_context: synonyms: @@ -397,6 +420,7 @@ semantic_model: - dialect: ANSI_SQL expression: s_store_sk datatype: Decimal + dimension: {} description: Surrogate key for store - name: s_store_id expression: @@ -404,6 +428,7 @@ semantic_model: - dialect: ANSI_SQL expression: s_store_id datatype: String + dimension: {} description: Business key for store ai_context: synonyms: @@ -415,6 +440,7 @@ semantic_model: - dialect: ANSI_SQL expression: s_store_name datatype: String + dimension: {} description: Store name ai_context: synonyms: @@ -426,6 +452,7 @@ semantic_model: - dialect: ANSI_SQL expression: s_city datatype: String + dimension: {} description: City where store is located ai_context: synonyms: @@ -437,6 +464,7 @@ semantic_model: - dialect: ANSI_SQL expression: s_state datatype: String + dimension: {} description: State where store is located ai_context: synonyms: @@ -448,6 +476,7 @@ semantic_model: - dialect: ANSI_SQL expression: s_number_employees datatype: Decimal + dimension: {} description: Number of employees at the store ai_context: synonyms: diff --git a/converters/cube/tests/test_cube_to_osi.py b/converters/cube/tests/test_cube_to_osi.py index c1e60619..6cba2db3 100644 --- a/converters/cube/tests/test_cube_to_osi.py +++ b/converters/cube/tests/test_cube_to_osi.py @@ -580,3 +580,21 @@ def test_metric_name_collisions_are_detected_case_insensitively(first, second, e out, _ = convert_cube_to_ossie(_two_cube_measures(first, second)) # The comparison is normalized; the emitted name keeps its original spelling. assert [m["name"] for m in model_of(out)["metrics"]] == expected + + +def test_every_dimension_carries_the_role_block(model_a): + """A Cube `dimensions:` entry is a dimension, and the `dimension` block is what says + so. Its *absence* is what other converters read as "not a dimension": the Snowflake + converter classifies a field with no block as a fact "regardless of datatype", so + emitting it for time fields only turned every other dimension into a Cortex Analyst + fact -- 0 dimensions and 27 facts across the TPC-DS model. + + Left empty for a non-time dimension, so the consumer applies the spec's default + rather than this converter asserting `is_time: false`. + """ + model, _ = model_a + fields = by_name(by_name(model["datasets"])["orders"]["fields"]) + assert fields["created_at"]["dimension"] == {"is_time": True} + assert fields["status"]["dimension"] == {} + assert all("dimension" in f + for ds in model["datasets"] for f in ds.get("fields", [])) diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 32c1d4fe..7b7db21c 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -974,3 +974,25 @@ def test_a_join_with_no_key_at_all_says_what_cube_will_refuse(): dropped = issues.of_type(IssueType.DROPPED_NO_CUBE_EQUIVALENT) assert any("requires a primary key on any cube with a join" in i.detail for i in dropped) + + +@pytest.mark.parametrize("dialect", ["SNOWFLAKE", "DATABRICKS", "BIGQUERY"]) +def test_any_warehouse_dialect_alone_is_enough_to_convert(dialect): + """The fallback is not Databricks-specific: a model carrying only Snowflake or + BigQuery SQL converts too, since Cube passes SQL to whatever the data source is.""" + ds = ( + " - name: orders\n" + " source: shop.public.orders\n" + " primary_key:\n - id\n" + " fields:\n" + " - name: id\n expression:\n dialects:\n" + f" - dialect: {dialect}\n expression: id\n" + " datatype: Integer\n" + ) + files, issues = convert_ossie_to_cube(_ossie(ds, metrics=_metric( + "n", "COUNT(DISTINCT orders.id)"))) + cube = _cubes(files)["orders"] + assert [d["name"] for d in cube["dimensions"]] == ["id"] + assert cube["measures"] == [{"name": "n", "type": "count"}] + assert any(dialect in i.detail + for i in issues.of_type(IssueType.APPROXIMATED)) From bf3fe5803c8714ad752fc92a78f0a5e14590afb6 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Wed, 5 Aug 2026 00:56:46 +0500 Subject: [PATCH 40/46] Record provenance for the three Cube-shaped choices export has to make All three blockers are regressions from the previous two commits: each fixed the forward direction by making a choice Cube requires, and each choice was one-way, so `Ossie -> Cube -> Ossie` no longer returned the document it was given. The pattern is the one the rest of the converter already uses -- record the choice in `meta.ossie`, undo it on the way back: - A warehouse dialect used in place of ANSI is recorded, so re-import labels the SQL as that dialect instead of calling vendor SQL `ANSI_SQL`. On measures as well as fields; metrics carry expressions too. - A `unique_keys` entry promoted to satisfy Cube's join requirement is recorded, so re-import does not hand back a declared `primary_key` the model never had. The dimension the promotion synthesized is recorded too, so it does not come back as a field for a column the Ossie model never described. - An Ossie field with no `dimension` block is recorded, so it returns as the fact it was rather than as a dimension. Cube has one kind of dimension, so the block still goes out on every member -- that is what the Snowflake classification needs. `test_a_model_from_another_converter_survives_the_round_trip_exactly` pins all of it on the committed Databricks-authored fixture: dialects, keys, fields and roles compared before and after. It would have failed on each of the three. Worth noting why the property tests missed these: the generator draws ANSI expressions, declares a primary key, and gives every field a dimension role -- so none of the three shapes can occur in a generated model. The fixture from another converter is the only thing in the suite that has them. 560 tests with both gates, 533 with neither, 96% coverage. Snowflake classification and the interop matrix unchanged. --- converters/cube/README.md | 8 ++-- converters/cube/src/ossie_cube/cube_to_osi.py | 39 +++++++++++++------ converters/cube/src/ossie_cube/osi_to_cube.py | 33 ++++++++++++++-- converters/cube/tests/test_edge_cases.py | 13 ++++--- converters/cube/tests/test_osi_to_cube.py | 3 +- converters/cube/tests/test_roundtrip.py | 35 +++++++++++++++++ 6 files changed, 106 insertions(+), 25 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index 765b7410..78e39976 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -124,11 +124,11 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | `dataset.description` | cube `description` | | | `dataset.ai_context` | cube `meta.ai_context` | Preserved for the round trip, but **inert in Cube** -- its agent ignores cube-level `ai_context`. Recorded as an issue. | | `dataset.primary_key` | dimension(s) with `primary_key: true` | Composite = several. A Cube key can be an *expression*, and then the only name Ossie can carry is the dimension's -- which the Ossie document alone cannot tell apart from a column name afterwards, so import records it (`computed_primary_key`) and export puts `primary_key: true` back on that dimension instead of synthesizing one that reads a column of that name. Export marks a dimension only when it is **scalar** — a single source column — since `primary_key: true` declares that dimension's own `sql` to be the key; a computed dimension or a merged `geo` one would declare the wrong thing even if its name matches. Anything left uncovered becomes a `public: false` scalar dimension, suffixed (`id_pk`) if the obvious name is taken. | -| `dataset.unique_keys` | `meta.ossie.unique_keys` | No native Cube slot, so parked — and used as the Cube primary key when the dataset declares none. Cube refuses a cube that declares a join without one, and several source formats have no primary-key concept: a Databricks metric view does not. A dataset with a relationship and neither is reported, naming Cube's requirement, since nothing can be invented. | +| `dataset.unique_keys` | `meta.ossie.unique_keys` | No native Cube slot, so parked — and used as the Cube primary key when the dataset declares none, recorded as `meta.ossie.key_from_unique_keys` so re-import does not hand back a `primary_key` the model never declared. Cube refuses a cube that declares a join without one, and several source formats have no primary-key concept: a Databricks metric view does not. A dataset with a relationship and neither is reported, naming Cube's requirement, since nothing can be invented. | | field | `dimensions[]` entry | Export: a name that is not a valid Cube identifier is sanitized; a case-insensitive collision is an error, never a silent merge. | | `field.expression` | dimension `sql` | Dataset-scoped, so `{CUBE}.col` <-> `col`. Export emits `{CUBE}.column` for a raw column and `{CUBE.member}` for a declared member, and never spells the cube's own name (which would break under `extends`). | | `field.datatype` | dimension `type` (**required**) | `String`->`string`, `Boolean`->`boolean`, `Date`/`Time`/`DateTime`/`DateTimeTz`->`time`, `Integer`/`Decimal`/`Float`->`number`, `Opaque`->`string`. Import maps back, choosing `Decimal` for `number` -- Cube collapses three Ossie types into one, so any single answer is a guess, and a stated datatype is what another converter can act on. Export parks the exact one in `meta.ossie.datatype`, which import prefers when present, so `Integer` and `Float` still survive a round trip. | -| `field.dimension` | a `dimensions[]` entry | Import always emits the block, because a Cube `dimensions:` entry *is* a dimension and the block's absence is what other converters read as "not one" — the Snowflake converter classifies a field without it as a fact regardless of datatype, so omitting it made every non-time dimension a Cortex Analyst fact. Left empty for a non-time dimension, so the consumer applies the spec's default rather than this converter asserting `is_time: false`. | +| `field.dimension` | a `dimensions[]` entry | Import always emits the block, because a Cube `dimensions:` entry *is* a dimension and the block's absence is what other converters read as "not one" — the Snowflake converter classifies a field without it as a fact regardless of datatype, so omitting it made every non-time dimension a Cortex Analyst fact. Left empty for a non-time dimension, so the consumer applies the spec's default rather than this converter asserting `is_time: false`. An Ossie field that had *no* block is recorded (`meta.ossie.no_role`) and gets none back: it was a fact and stays one. | | `field.dimension.is_time` | `type: time` | Import sets `is_time: true` for a time dimension. A field carrying `is_time` but *no* `datatype` records the absence (`meta.ossie.untyped`), since the spec says not to infer a scalar type from `is_time` alone — otherwise `type: time` would come back asserting `DateTime`. | | `field.label` / `description` | dimension `title` / `description` | | | `field.ai_context.instructions` | dimension `meta.ai_context` | Cube's documented AI-only context field. | @@ -183,7 +183,9 @@ prefers `ANSI_SQL` with `--dialect` prepending a warehouse dialect (e.g. `SNOWFLAKE` for a Snowflake-backed Cube model). Failing both, export falls back to the expression's **only** dialect when that dialect -is warehouse SQL (`SNOWFLAKE`, `DATABRICKS`, `BIGQUERY`), and reports it. This is what +is warehouse SQL (`SNOWFLAKE`, `DATABRICKS`, `BIGQUERY`), records which one in +`meta.ossie.dialect`, and reports it. The record matters: without it re-import would +label vendor-specific SQL as `ANSI_SQL` and mislead the next converter. This is what makes another converter's output usable: everything the Databricks converter emits is `DATABRICKS` with no ANSI alternative, and requiring ANSI dropped every field and metric — producing an *empty* Cube model, which Cube compiles, so nothing downstream noticed. diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 55645c9c..e5399854 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -569,7 +569,7 @@ def _convert_cube(cname, cube, plain, extra_joins, extra_measures, issues): if extra_dimensions: stash["extra_dimensions"] = extra_dimensions primary_key = _primary_key_of(cube, cname) - if primary_key: + if primary_key and not parked.get("key_from_unique_keys"): ds["primary_key"] = primary_key # Ossie's `primary_key` names columns, but a Cube key can be an expression # (`CONCAT(tenant_id, id)`), and then the only name there is to write is the @@ -630,7 +630,8 @@ def _convert_dimension(cname, dname, dim, plain, issues): "expression": { "dialects": [{"dialect": DIALECT_ANSI, "expression": expr}]}, } - return [_finish_dimension_field(cname, dname, dim, field, stash, issues)] + built = _finish_dimension_field(cname, dname, dim, field, stash, issues) + return [built] if built is not None else [] if dim.get("sub_query"): # `sub_query: true` means the sql references a *measure* (`{users.count}`), # which Cube resolves by aggregating in a subquery. An Ossie field expression @@ -665,7 +666,8 @@ def _convert_dimension(cname, dname, dim, plain, issues): "name": dname, "expression": {"dialects": [{"dialect": DIALECT_ANSI, "expression": expr}]}, } - return [_finish_dimension_field(cname, dname, dim, field, stash, issues)] + built = _finish_dimension_field(cname, dname, dim, field, stash, issues) + return [built] if built is not None else [] def _finish_dimension_field(cname, dname, dim, field, stash, issues): @@ -678,6 +680,14 @@ def _finish_dimension_field(cname, dname, dim, field, stash, issues): # A precise datatype parked by a previous export wins over the default the Cube # type maps to, since Cube itself cannot hold the distinction. parked = parked_of(dim.get("meta")) + if parked.get("synthetic_key"): + # A dimension export added only to carry Cube's primary key; the Ossie model had + # no field for that column, so it gets none back. + return None + if parked.get("dialect"): + # The expression came from a warehouse dialect, so it is labelled as that one -- + # calling vendor SQL `ANSI_SQL` would mislead the next converter. + field["expression"]["dialects"][0]["dialect"] = parked["dialect"] # A field that carried no datatype keeps carrying none: Ossie says not to infer a # scalar type from `is_time` alone, so emitting DateTime for a `type: time` # dimension would assert something the model never said. @@ -690,14 +700,16 @@ def _finish_dimension_field(cname, dname, dim, field, stash, issues): # With no datatype there is nothing to regenerate from, so the type is recorded. if DATATYPE_TO_DIM_TYPE.get(field.get("datatype")) != dtype: stash["dim_type"] = dtype - # Every Cube `dimensions:` entry is a dimension, so the role block is always - # emitted. Its *absence* is what other converters read as "not a dimension" -- the - # Snowflake converter classifies a field with no `dimension` block as a fact - # "regardless of datatype", so omitting it turned every non-time dimension into a - # Cortex Analyst fact. Left empty for a non-time one, which lets the consumer apply - # the spec's default (`is_time` false for a non-temporal datatype) rather than this - # converter asserting it. - field["dimension"] = {"is_time": True} if dtype == "time" else {} + # A Cube `dimensions:` entry is a dimension, and the block's *absence* is what other + # converters read as "not one" -- the Snowflake converter classifies a field without + # it as a fact regardless of datatype, so omitting it turned every non-time dimension + # into a Cortex Analyst fact. Empty for a non-time one, leaving the consumer to apply + # the spec's default rather than this converter asserting `is_time: false`. + # + # Unless export recorded that the Ossie field had no role of its own: that field was a + # fact, and handing back a dimension would change what it means. + if not parked.get("no_role"): + field["dimension"] = {"is_time": True} if dtype == "time" else {} if dim.get("title"): field["label"] = unescape_braces_from_cube(dim["title"]) if dim.get("description"): @@ -1416,9 +1428,12 @@ def _convert_measure(cname, mname, metric_name, measure, context): f"the primary key at query time but a static Ossie expression cannot, so " f"a consumer joining through that relationship may over-count") + parked_measure = parked_of(measure.get("meta")) metric = { "name": metric_name, - "expression": {"dialects": [{"dialect": DIALECT_ANSI, "expression": expr}]}, + "expression": {"dialects": [ + {"dialect": parked_measure.get("dialect") or DIALECT_ANSI, + "expression": expr}]}, } # A datatype parked by a previous export wins: Cube has no field for a measure's # result type, and only the count family can be inferred from the aggregate. diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 36a3d7ca..7446fedd 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -294,6 +294,12 @@ def _build_cube(ds, plan, tables, joins, measures, join_extensions, dialect, foreign = foreign_vendor_extensions(ds) if foreign: parked["custom_extensions"] = foreign + if plan.key_from_unique_keys: + # Recorded before the meta is assembled, so re-import does not hand back a + # `primary_key` the Ossie model never declared: `unique_keys` is restored on its + # own, and promoting it was a Cube requirement rather than something the document + # said. + parked["key_from_unique_keys"] = True if join_extensions: parked["join_extensions"] = join_extensions issues.add(IssueType.PARKED_IN_META, scope, @@ -347,8 +353,13 @@ def _build_cube(ds, plan, tables, joins, measures, join_extensions, dialect, if name != entry: detail += f", named '{name}' to avoid colliding with the existing member" issues.add(IssueType.APPROXIMATED, scope, detail) - dimensions.append({"name": name, "sql": entry, "type": "string", - "primary_key": True, "public": False}) + dimensions.append({ + "name": name, "sql": entry, "type": "string", + "primary_key": True, "public": False, + # This dimension exists only to carry Cube's primary key; the Ossie model + # had no field for the column. Marked so re-import does not invent one. + "meta": {"ossie": {"synthetic_key": True}}, + }) pk_names.append(name) for dim in dimensions: if dim["name"] in pk_names: @@ -687,6 +698,16 @@ def _build_dimensions(ds, plan, tables, dialect, issues): # recover it. `meta.ossie` is Cube-side, so this costs the Ossie document # nothing -- unlike a custom_extension, which every other spoke would warn # about and discard. + if "dimension" not in field: + # The Ossie field carried no dimension role -- it is a fact. Cube has one + # kind of dimension, so the block is emitted regardless; recording its + # absence is what stops re-import handing back a dimension. + parked["no_role"] = True + if used not in (None, DIALECT_ANSI): + # The expression came from a warehouse dialect, not ANSI. Re-import would + # otherwise label vendor-specific SQL as ANSI_SQL and mislead the next + # converter. + parked["dialect"] = used dt = field.get("datatype") if dt and DEFAULT_DATATYPE_FOR_CUBE_TYPE.get(dim["type"]) != dt: parked["datatype"] = dt @@ -997,7 +1018,7 @@ def resolve_base(): else: measure = _measure_from_expression( expr, target, mname, stash, plan, tables) - _apply_measure_metadata(metric, measure, stash) + _apply_measure_metadata(metric, measure, stash, used) _place(measures_by_cube, target, measure, name) return measures_by_cube @@ -1147,7 +1168,7 @@ def _measure_from_expression(expr, target, mname, stash, plan, tables): return measure -def _apply_measure_metadata(metric, measure, stash): +def _apply_measure_metadata(metric, measure, stash, used_dialect=None): if stash.get("title"): # Not escaped: this came out of the stash, so it is already whatever Cube # needs it to be. Escaping it again turned a valid `Revenue \{USD\}` into @@ -1165,6 +1186,10 @@ def _apply_measure_metadata(metric, measure, stash): datatype = metric.get("datatype") if datatype and datatype != AGG_TO_RESULT_DATATYPE.get(measure.get("type")): parked["datatype"] = datatype + if used_dialect not in (None, DIALECT_ANSI): + # The expression came from a warehouse dialect; re-import would otherwise label + # vendor-specific SQL as ANSI_SQL. + parked["dialect"] = used_dialect meta = _build_meta(metric.get("ai_context"), stash.get("meta"), parked) if meta: measure["meta"] = meta diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 8c662ddd..c14bd8c2 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -1021,8 +1021,9 @@ def test_a_computed_dimension_does_not_cover_a_primary_key(): assert "primary_key" not in dims["id"] assert dims["id"]["sql"] == "LOWER(email)" # A private scalar dimension carries the key instead, under a free name. - assert dims["id_pk"] == {"name": "id_pk", "sql": "id", "type": "string", - "primary_key": True, "public": False} + assert dims["id_pk"] == { + "name": "id_pk", "sql": "id", "type": "string", "primary_key": True, + "public": False, "meta": {"ossie": {"synthetic_key": True}}} assert issues.of_type(IssueType.APPROXIMATED) @@ -1038,7 +1039,8 @@ def test_a_merged_geo_dimension_does_not_cover_a_primary_key(): assert "primary_key" not in dims["location"] assert dims["location_pk"] == { "name": "location_pk", "sql": "location", "type": "string", - "primary_key": True, "public": False} + "primary_key": True, "public": False, + "meta": {"ossie": {"synthetic_key": True}}} def test_a_scalar_dimension_backed_by_the_key_column_covers_it(): @@ -1085,8 +1087,9 @@ def test_a_synthesized_key_name_avoids_every_existing_member(): dims = by_name(_dims(files)) keys = [n for n, d in dims.items() if d.get("primary_key")] assert keys == ["id_pk_3"] - assert dims["id_pk_3"] == {"name": "id_pk_3", "sql": "id", "type": "string", - "primary_key": True, "public": False} + assert dims["id_pk_3"] == { + "name": "id_pk_3", "sql": "id", "type": "string", "primary_key": True, + "public": False, "meta": {"ossie": {"synthetic_key": True}}} # Nothing was overwritten. assert dims["id"]["sql"] == "LOWER(email)" assert dims["id_pk"]["sql"] == "UPPER(email)" diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 7b7db21c..48d5526b 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -201,7 +201,8 @@ def test_primary_key_column_without_a_field_is_synthesized(): dims = by_name(_cubes(files)["orders"]["dimensions"]) assert dims["ticket_no"] == { "name": "ticket_no", "sql": "ticket_no", "type": "string", - "primary_key": True, "public": False} + "primary_key": True, "public": False, + "meta": {"ossie": {"synthetic_key": True}}} # `type: string` is chosen by the converter, not carried by Ossie. assert issues.of_type(IssueType.APPROXIMATED) diff --git a/converters/cube/tests/test_roundtrip.py b/converters/cube/tests/test_roundtrip.py index 5b89ec6a..a84eb65c 100644 --- a/converters/cube/tests/test_roundtrip.py +++ b/converters/cube/tests/test_roundtrip.py @@ -196,3 +196,38 @@ def test_ossie_only_constructs_are_parked_not_dropped(): assert ds["orders"]["unique_keys"] == [["order_number"]] vendors = {e["vendor_name"] for e in ds["orders"]["custom_extensions"]} assert "SNOWFLAKE" in vendors + + +@validator_gate +def test_a_model_from_another_converter_survives_the_round_trip_exactly(): + """`Ossie -> Cube -> Ossie` on a document written by another converter. + + Everything this fixture exercises is a place where the *forward* direction has to + make a Cube-shaped choice, and each of those choices was one-way until provenance was + recorded: a warehouse dialect became `ANSI_SQL`, a `unique_keys` promoted to satisfy + Cube's join requirement came back as a declared `primary_key`, the dimension the + promotion synthesized came back as a field, and a fact came back as a dimension + because Cube has only the one kind. + """ + src = load_fixture("databricks_ossie.yaml") + files, _ = convert_ossie_to_cube(src) + back, _ = convert_cube_to_ossie(files) + assert_ossie_is_valid(back, "databricks_ossie.yaml round trip") + + before = parse(src)["semantic_model"][0] + after = parse(back)["semantic_model"][0] + + def shape(model): + return { + "datasets": {ds["name"]: { + "primary_key": ds.get("primary_key"), + "unique_keys": ds.get("unique_keys"), + "fields": {f["name"]: (f["expression"]["dialects"][0]["dialect"], + "dimension" in f, f.get("datatype")) + for f in ds.get("fields", [])}} + for ds in model["datasets"]}, + "metrics": {m["name"]: m["expression"]["dialects"][0]["dialect"] + for m in model.get("metrics", [])}, + } + + assert shape(after) == shape(before) From 2f427d15f9618ae2421ce1a8aa7b374f88f8188b Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Wed, 5 Aug 2026 01:05:21 +0500 Subject: [PATCH 41/46] Generate the shapes the last three blockers lived in Three review rounds found bugs in one blind spot: the generator drew ANSI expressions, always declared a `primary_key`, and gave every field a `dimension` role -- so none of the 210 generated models per run could contain the shapes that were breaking. The committed Databricks-authored fixture was the only thing in the suite that had them, which is why the same class of defect came back three times. The generator now draws all three, since each is a place where export must make a choice Cube requires and then be able to undo it: - a dialect per field and per metric, often a warehouse one with no ANSI alternative; - either `primary_key` or `unique_keys` (never neither -- Cube rightly refuses a cube with a join and no key); - a `dimension` role or none, the latter being a fact. And the property compares what those choices affect -- dialects, keys, roles and datatypes -- not just expressions, which is how one-way fixes slipped past it before. Checked that it can fail: reverting each of the three provenance records in turn breaks 61, 61 and 34 of the 122 property cases. A green test that cannot fail is not a test. 560 tests with both gates, 534 with neither, 96% coverage. --- converters/cube/README.md | 9 ++- converters/cube/tests/_roundtrip_helpers.py | 86 +++++++++++++++------ 2 files changed, 69 insertions(+), 26 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index 78e39976..ab71599d 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -480,7 +480,14 @@ composite metrics (the decomposition path), mixed-case and quoted references, co fields, and join keys in all three reference forms. It asserts the generated document is spec-valid, that `Ossie -> Cube -> Ossie` preserves every metric and field expression modulo the identifier-case canonicalization, and -- when a Cube checkout is available -- -that Cube compiles the export. It found a defect on its first run: a generated view over +that Cube compiles the export. + +It draws the shapes a converter's output has and hand-written test models do not: a +warehouse dialect with no ANSI alternative, `unique_keys` in place of `primary_key`, and +fields with no `dimension` role. Each is a place where export has to make a choice Cube +requires and then be able to undo it, and the property compares dialects, keys and roles +rather than expressions alone -- reverting any one of those three provenance records fails +it. It found a defect on its first run: a generated view over an ordinary star schema, where the fact's `users_id` foreign key collides with `users.id` once prefixed. diff --git a/converters/cube/tests/_roundtrip_helpers.py b/converters/cube/tests/_roundtrip_helpers.py index bea1fa2f..b354fe25 100644 --- a/converters/cube/tests/_roundtrip_helpers.py +++ b/converters/cube/tests/_roundtrip_helpers.py @@ -326,18 +326,29 @@ def build_ossie_model(rnd): fields_by_dataset[name] = fields lines.append(f" - name: {name}") lines.append(f" source: shop.public.{name}") - lines.append(" primary_key:") - lines.append(" - id") + # Either way of declaring the key. `unique_keys` is what a source format with no + # primary-key concept produces -- a Databricks metric view has none -- and export + # has to promote it, because Cube demands a key on any cube with a join. One of + # the two is always present: with neither, Cube rightly refuses the model. + if rnd.chance(0.7): + lines.append(" primary_key:") + lines.append(" - id") + else: + lines.append(" unique_keys:") + lines.append(" - - id") if rnd.chance(0.4): lines.append(f" description: {_yaml_text(rnd.text())}") lines.append(" fields:") - for fname, expr, datatype in fields: + for fname, expr, datatype, dialect, has_role in fields: lines.append(f" - name: {fname}") lines.append(" expression:") lines.append(" dialects:") - lines.append(" - dialect: ANSI_SQL") + lines.append(f" - dialect: {dialect}") lines.append(f" expression: {expr}") lines.append(f" datatype: {datatype}") + if has_role: + lines.append(" dimension:") + lines.append(" is_time: false") # Every dimension dataset is reachable from the fact, so a generated view has an # unambiguous root and cross-dataset metrics have a join path. @@ -355,20 +366,35 @@ def build_ossie_model(rnd): return "\n".join(lines) + "\n" +# Dialects whose SQL Cube can pass to a data source. A converter commonly emits its own +# and no ANSI -- everything from the Databricks converter is `DATABRICKS` -- so export has +# to use it and record which one, or vendor SQL comes back labelled `ANSI_SQL`. +OSSIE_DIALECTS = ["ANSI_SQL", "ANSI_SQL", "DATABRICKS", "SNOWFLAKE", "BIGQUERY"] + + def _ossie_fields(rnd, dataset, dim_names): - """(name, ANSI expression, datatype) for one dataset's fields.""" - fields = [("id", "id", "Integer")] + """(name, expression, datatype, dialect, has_dimension_role) per field. + + The last two are drawn rather than fixed, because both are places where export must + make a Cube-shaped choice and then be able to undo it. A field with no `dimension` + block is a *fact*; Cube has one kind of dimension, so export marks every member as one + and has to remember which were not. + """ + def entry(name, expr, datatype): + return (name, expr, datatype, rnd.pick(OSSIE_DIALECTS), rnd.chance(0.5)) + + fields = [entry("id", "id", "Integer")] for d in dim_names: - fields.append((f"{d}_id", f"{d}_id", "Integer")) + fields.append(entry(f"{d}_id", f"{d}_id", "Integer")) for i in range(rnd.count(1, 2)): name = f"attr_{i}" if rnd.chance(0.7) else f"Attr{i}" if rnd.chance(0.3): # A computed field, which must be referenced as `{CUBE.member}` so Cube # inlines its expression rather than reading a column of that name. - fields.append((name, f"LOWER({name}_raw)", "String")) + fields.append(entry(name, f"LOWER({name}_raw)", "String")) else: - fields.append((name, name, "String")) - fields.append(("value", "value", "Decimal")) + fields.append(entry(name, name, "String")) + fields.append(entry("value", "value", "Decimal")) return fields @@ -380,7 +406,7 @@ def block(name, expression): entry = [f" - name: {name}", " expression:", " dialects:", - " - dialect: ANSI_SQL", + f" - dialect: {rnd.pick(OSSIE_DIALECTS)}", f" expression: {expression}"] if rnd.chance(0.3): entry.insert(1, f" description: {_yaml_text(rnd.text())}") @@ -443,24 +469,34 @@ def check_ossie_model(ossie_yaml): original = load_yaml(ossie_yaml)["semantic_model"][0] returned = load_yaml(back)["semantic_model"][0] - # Metrics must come back with the same names and the same expressions: a composite + # Metrics must come back with the same names, expressions *and* dialects. A composite # one is split into hidden measures on the way out and inlined back on the way in, - # which is the most intricate path in the converter and had no generated coverage. + # which is the most intricate path in the converter. def metrics(model): - return {m["name"]: _normalize_refs(m["expression"]["dialects"][0]["expression"]) - for m in (model.get("metrics") or [])} + return {m["name"]: _dialected(m) for m in (model.get("metrics") or [])} assert metrics(returned) == metrics(original), ( "Ossie -> Cube -> Ossie changed the metrics") - # And the fields, which is where a computed expression has to survive as a member - # reference rather than being flattened to a column of the same name. - def fields(model): - return {ds["name"]: {f["name"]: _normalize_refs( - f["expression"]["dialects"][0]["expression"]) - for f in (ds.get("fields") or [])} - for ds in model["datasets"]} - - assert fields(returned) == fields(original), ( - "Ossie -> Cube -> Ossie changed the fields") + # Fields likewise, plus the two things Cube forces a choice about: it has one kind of + # dimension, so a field with no role has to be recorded as having none; and it needs a + # key on any cube with a join, so a `unique_keys` promoted to supply one must not come + # back as a declared `primary_key`. + def datasets(model): + return {ds["name"]: { + "primary_key": ds.get("primary_key"), + "unique_keys": ds.get("unique_keys"), + "fields": {f["name"]: (_dialected(f), "dimension" in f, + f.get("datatype")) + for f in (ds.get("fields") or [])}, + } for ds in model["datasets"]} + + assert datasets(returned) == datasets(original), ( + "Ossie -> Cube -> Ossie changed the datasets") return files + + +def _dialected(entry): + """(normalized expression, dialect) for a field or metric.""" + dialect = entry["expression"]["dialects"][0] + return _normalize_refs(dialect["expression"]), dialect["dialect"] From e602ac34c947fd95631b8070c0a7f1e3cbaeb350 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Wed, 5 Aug 2026 01:33:57 +0500 Subject: [PATCH 42/46] Preserve key columns and every dialect of an expression Both blockers were provenance recorded by halves. - Ossie names a primary key by *column*; Cube marks a *dimension*. The two differ whenever the dimension carrying the key is not named after its column -- a field `order_id` reading column `id`, or a synthesized `id_pk` where a computed field shadows the column -- and import rebuilt the key from dimension names, so it came back naming something the table need not have. The column list is recorded when it cannot be read back off the dimensions, and only then, so a model whose names already agree keeps a clean Cube round trip. `_primary_key_of` returns columns now, which also fixes the rebuilt `COUNT(DISTINCT ...)`: it was naming the synthesized dimension, a member the Ossie side does not have at all. - Recording only the chosen dialect's *name* lost the alternatives. Cube holds one `sql` per member, so nothing short of the whole expression object brings a multi-dialect expression back; it is parked entire, on measures as well as fields. The generator now draws several dialects per expression and the property compares them all rather than `dialects[0]`, which is what let the second one through. That immediately found two more: - an expression offering two warehouse dialects and no ANSI was dropped outright, because the fallback insisted on a sole candidate. It takes the first in document order and reports it -- Cube passes SQL to one data source, and the alternatives are parked. - `COUNT(DISTINCT )` drifted to the synthesized dimension name, above. Also a flaw in the generator itself: it picked an alternative dialect out of a `set`, whose iteration order varies between processes, so the seeded sweep produced different models each run and could not name a reproducible seed. Sorted now -- the same suite ran green and red in consecutive invocations before this. Checked both fixes can fail: reverting each breaks 60 and 9 of 122 property cases. Metric drift across 400 generated models is zero. 560 tests with both gates, 534 with neither, 97% coverage. --- converters/cube/README.md | 14 +++-- converters/cube/src/ossie_cube/_common.py | 15 +++-- converters/cube/src/ossie_cube/cube_to_osi.py | 40 +++++++++---- converters/cube/src/ossie_cube/osi_to_cube.py | 56 +++++++++++++------ converters/cube/tests/_roundtrip_helpers.py | 50 ++++++++++++----- 5 files changed, 124 insertions(+), 51 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index ab71599d..6768ae10 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -123,7 +123,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | `dataset.source` (`SELECT ...`) | `sql` | Cube requires exactly one of `sql` / `sql_table`. | | `dataset.description` | cube `description` | | | `dataset.ai_context` | cube `meta.ai_context` | Preserved for the round trip, but **inert in Cube** -- its agent ignores cube-level `ai_context`. Recorded as an issue. | -| `dataset.primary_key` | dimension(s) with `primary_key: true` | Composite = several. A Cube key can be an *expression*, and then the only name Ossie can carry is the dimension's -- which the Ossie document alone cannot tell apart from a column name afterwards, so import records it (`computed_primary_key`) and export puts `primary_key: true` back on that dimension instead of synthesizing one that reads a column of that name. Export marks a dimension only when it is **scalar** — a single source column — since `primary_key: true` declares that dimension's own `sql` to be the key; a computed dimension or a merged `geo` one would declare the wrong thing even if its name matches. Anything left uncovered becomes a `public: false` scalar dimension, suffixed (`id_pk`) if the obvious name is taken. | +| `dataset.primary_key` | dimension(s) with `primary_key: true` | Composite = several. Ossie names the key by *column* while Cube marks a *dimension*, and the two differ whenever the dimension carrying the key is not named after its column — so the column list is recorded (`meta.ossie.primary_key`) when it cannot be read back off the dimensions. It is what the rebuilt `COUNT(DISTINCT …)` uses too, which otherwise named a member the Ossie model does not have. A Cube key can be an *expression*, and then the only name Ossie can carry is the dimension's -- which the Ossie document alone cannot tell apart from a column name afterwards, so import records it (`computed_primary_key`) and export puts `primary_key: true` back on that dimension instead of synthesizing one that reads a column of that name. Export marks a dimension only when it is **scalar** — a single source column — since `primary_key: true` declares that dimension's own `sql` to be the key; a computed dimension or a merged `geo` one would declare the wrong thing even if its name matches. Anything left uncovered becomes a `public: false` scalar dimension, suffixed (`id_pk`) if the obvious name is taken. | | `dataset.unique_keys` | `meta.ossie.unique_keys` | No native Cube slot, so parked — and used as the Cube primary key when the dataset declares none, recorded as `meta.ossie.key_from_unique_keys` so re-import does not hand back a `primary_key` the model never declared. Cube refuses a cube that declares a join without one, and several source formats have no primary-key concept: a Databricks metric view does not. A dataset with a relationship and neither is reported, naming Cube's requirement, since nothing can be invented. | | field | `dimensions[]` entry | Export: a name that is not a valid Cube identifier is sanitized; a case-insensitive collision is an error, never a silent merge. | | `field.expression` | dimension `sql` | Dataset-scoped, so `{CUBE}.col` <-> `col`. Export emits `{CUBE}.column` for a raw column and `{CUBE.member}` for a declared member, and never spells the cube's own name (which would break under `extends`). | @@ -182,10 +182,14 @@ Ossie dialect enum has no `CUBE` entry -- so import emits `ANSI_SQL`, and export prefers `ANSI_SQL` with `--dialect` prepending a warehouse dialect (e.g. `SNOWFLAKE` for a Snowflake-backed Cube model). -Failing both, export falls back to the expression's **only** dialect when that dialect -is warehouse SQL (`SNOWFLAKE`, `DATABRICKS`, `BIGQUERY`), records which one in -`meta.ossie.dialect`, and reports it. The record matters: without it re-import would -label vendor-specific SQL as `ANSI_SQL` and mislead the next converter. This is what +Failing both, export falls back to the **first** dialect on offer that is warehouse SQL +(`SNOWFLAKE`, `DATABRICKS`, `BIGQUERY`), records which one in `meta.ossie.dialect`, and +reports it. The record matters: without it re-import would label vendor-specific SQL as +`ANSI_SQL` and mislead the next converter. + +Cube holds one `sql` per member, so an expression offering **several** dialects cannot +keep its alternatives natively — the whole expression object is parked and restored, since +nothing less brings them back. This is what makes another converter's output usable: everything the Databricks converter emits is `DATABRICKS` with no ANSI alternative, and requiring ANSI dropped every field and metric — producing an *empty* Cube model, which Cube compiles, so nothing downstream noticed. diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index beb3ba88..91bb322d 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -311,8 +311,7 @@ def pick_expression(ossie_expression, preferred=None): Preference order: the caller-chosen warehouse dialect (Cube passes SQL through to the data source, so e.g. SNOWFLAKE SQL is valid on a Snowflake-backed Cube model), - then ANSI_SQL, then -- if the expression offers exactly one dialect and that dialect - is warehouse SQL -- that one. + then ANSI_SQL, then the first dialect on offer that is warehouse SQL. The last step matters for real interop. Converters commonly emit their own dialect and no ANSI: everything from the Databricks converter is `DATABRICKS`. Requiring @@ -322,6 +321,12 @@ def pick_expression(ossie_expression, preferred=None): Only `WAREHOUSE_DIALECTS` qualify. An expression in MDX, TABLEAU or MAQL is not SQL a warehouse can run, so there is nothing to fall back *to* -- those still drop, with the issue saying so. `(None, None)` means nothing usable was found. + + Taking the *first* rather than insisting on a sole candidate matters: an expression + offering SNOWFLAKE and BIGQUERY but no ANSI has no single obvious choice, and requiring + one dropped the field altogether. Cube passes SQL to one data source, so picking in + document order and reporting it keeps the model; the alternatives are parked, so + nothing is lost on the way back. """ dialects = [(d.get("dialect"), d.get("expression")) for d in (ossie_expression or {}).get("dialects") or [] @@ -330,9 +335,9 @@ def pick_expression(ossie_expression, preferred=None): for candidate in (preferred, DIALECT_ANSI): if candidate and candidate in by_dialect: return _checked_expression(by_dialect[candidate]), candidate - if len(dialects) == 1 and dialects[0][0] in WAREHOUSE_DIALECTS: - dialect, expr = dialects[0] - return _checked_expression(expr), dialect + for dialect, expr in dialects: + if dialect in WAREHOUSE_DIALECTS: + return _checked_expression(expr), dialect return None, None diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index e5399854..8712d761 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -500,11 +500,21 @@ def _plain_members(cube, cname): def _primary_key_of(cube, cname): - """The names of a cube's `primary_key: true` dimensions. + """A cube's primary key, as the *columns* Ossie names it by. - Read directly off the dimensions so the stages that need it -- measures, and the - fan-out check -- do not have to wait for the dataset to be built. + Read off the cube rather than the built dataset, so the stages that need it -- the + dataset, measures, and the fan-out check -- all get the same answer without waiting + for each other. + + A recorded column list wins over the dimension names. The two differ whenever the key + is not a same-named scalar dimension: a field `order_id` reading column `id` carries + the key, and export synthesizes `id_pk` when a computed field shadows the column. Using + the dimension name then put that name in the rebuilt `COUNT(DISTINCT ...)` too, so the + metric referenced a member that does not exist on the Ossie side. """ + recorded = parked_of(cube.get("meta")).get("primary_key") + if recorded: + return [str(column) for column in recorded] return [require_str(dim, "name", f"cube '{cname}': dimension") for dim in _as_named_list(cube.get("dimensions"), f"cube '{cname}' dimensions") @@ -684,10 +694,7 @@ def _finish_dimension_field(cname, dname, dim, field, stash, issues): # A dimension export added only to carry Cube's primary key; the Ossie model had # no field for that column, so it gets none back. return None - if parked.get("dialect"): - # The expression came from a warehouse dialect, so it is labelled as that one -- - # calling vendor SQL `ANSI_SQL` would mislead the next converter. - field["expression"]["dialects"][0]["dialect"] = parked["dialect"] + _restore_expression(field, parked) # A field that carried no datatype keeps carrying none: Ossie says not to infer a # scalar type from `is_time` alone, so emitting DateTime for a `type: time` # dimension would assert something the model never said. @@ -774,6 +781,19 @@ def _case_label(cname, dname, holder): return "'" + text.replace("'", "''") + "'" +def _restore_expression(target, parked): + """Put back the dialect, or the whole expression, that export had to set aside. + + Cube holds one `sql` per member, so an Ossie expression carrying several dialects + cannot survive natively -- export parks it whole, and it comes back as it went in. + A single non-ANSI dialect needs only its label restored. + """ + if parked.get("expression"): + target["expression"] = parked["expression"] + elif parked.get("dialect"): + target["expression"]["dialects"][0]["dialect"] = parked["dialect"] + + def _convert_geo_dimension(cname, dname, dim, issues): """Split a `type: geo` dimension into a latitude and a longitude field. @@ -1428,13 +1448,11 @@ def _convert_measure(cname, mname, metric_name, measure, context): f"the primary key at query time but a static Ossie expression cannot, so " f"a consumer joining through that relationship may over-count") - parked_measure = parked_of(measure.get("meta")) metric = { "name": metric_name, - "expression": {"dialects": [ - {"dialect": parked_measure.get("dialect") or DIALECT_ANSI, - "expression": expr}]}, + "expression": {"dialects": [{"dialect": DIALECT_ANSI, "expression": expr}]}, } + _restore_expression(metric, parked_of(measure.get("meta"))) # A datatype parked by a previous export wins: Cube has no field for a measure's # result type, and only the count family can be inferred from the aggregate. parked_dt = parked_of(measure.get("meta")).get("datatype") diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 7446fedd..f1dcf577 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -308,13 +308,6 @@ def _build_cube(ds, plan, tables, joins, measures, join_extensions, dialect, f"parked under meta.ossie.join_extensions") cube_extras = dict(stash.get("cube_extras") or {}) stashed_meta = cube_extras.pop("meta", None) - meta = _build_meta(ds.get("ai_context"), stashed_meta, parked) - if meta: - cube["meta"] = meta - if "ai_context" in meta: - issues.add(IssueType.CUBE_LEVEL_AI_CONTEXT_INERT, scope, - "Cube's agent reads ai_context only on views and members, " - "so this cube-level value has no effect in Cube") dimensions, by_name_scalar, by_column, by_name_computed = _build_dimensions( ds, plan, tables, dialect, issues) @@ -364,6 +357,17 @@ def _build_cube(ds, plan, tables, joins, measures, join_extensions, dialect, for dim in dimensions: if dim["name"] in pk_names: dim["primary_key"] = True + # Whichever way the key arrived -- declared, or promoted from `unique_keys` -- what + # matters is whether the columns can be read back off the dimensions carrying them. + key_columns = list(plan.primary_key) + if key_columns and pk_names != key_columns: + # Import rebuilds the key from Cube *dimension* names, which are the columns only + # when they happen to coincide: a field `order_id` reading column `id` is marked + # as the key and comes back named `order_id`, and a synthesized `id_pk` comes back + # as `id_pk`. Both name something the table need not have, so the column list is + # recorded -- and only then, so a model whose names already agree keeps a clean + # round trip. + parked["primary_key"] = key_columns if plan.key_from_unique_keys and pk_names: issues.add(IssueType.APPROXIMATED, scope, f"no primary_key, so the first unique_keys entry " @@ -376,6 +380,14 @@ def _build_cube(ds, plan, tables, joins, measures, join_extensions, dialect, " is required when join is defined') -- the model will not " "compile until the dataset declares one") + meta = _build_meta(ds.get("ai_context"), stashed_meta, parked) + if meta: + cube["meta"] = meta + if "ai_context" in meta: + issues.add(IssueType.CUBE_LEVEL_AI_CONTEXT_INERT, scope, + "Cube's agent reads ai_context only on views and members, " + "so this cube-level value has no effect in Cube") + # Dimensions a prior import could not express as an Ossie field (a `switch` one, # which has no sql) go back at their original positions. for item in sorted(stash.get("extra_dimensions") or [], @@ -541,6 +553,25 @@ def _reference_members(ds, dim_names, dialect): return needed +def _park_expression(parked, expression, used): + """Record what it takes to hand this expression back unchanged. + + Cube holds one `sql` per member, so only the dialect export chose survives natively. + Two things can be lost on the way back: + + - the *label*: vendor SQL emitted as Cube's `sql` would be re-imported as `ANSI_SQL`, + which misleads the next converter. Recording the dialect name is enough for that. + - the *alternatives*: an Ossie expression may carry several dialects, and the others + have nowhere to go in Cube at all. Nothing short of the whole object brings them + back, so a multi-dialect expression is parked entire. + """ + dialects = (expression or {}).get("dialects") or [] + if len(dialects) > 1: + parked["expression"] = expression + elif used not in (None, DIALECT_ANSI): + parked["dialect"] = used + + def _report_dialect_fallback(issues, scope, used, preferred): """Note when an expression came from a dialect that was not asked for. @@ -703,11 +734,7 @@ def _build_dimensions(ds, plan, tables, dialect, issues): # kind of dimension, so the block is emitted regardless; recording its # absence is what stops re-import handing back a dimension. parked["no_role"] = True - if used not in (None, DIALECT_ANSI): - # The expression came from a warehouse dialect, not ANSI. Re-import would - # otherwise label vendor-specific SQL as ANSI_SQL and mislead the next - # converter. - parked["dialect"] = used + _park_expression(parked, field.get("expression"), used) dt = field.get("datatype") if dt and DEFAULT_DATATYPE_FOR_CUBE_TYPE.get(dim["type"]) != dt: parked["datatype"] = dt @@ -1186,10 +1213,7 @@ def _apply_measure_metadata(metric, measure, stash, used_dialect=None): datatype = metric.get("datatype") if datatype and datatype != AGG_TO_RESULT_DATATYPE.get(measure.get("type")): parked["datatype"] = datatype - if used_dialect not in (None, DIALECT_ANSI): - # The expression came from a warehouse dialect; re-import would otherwise label - # vendor-specific SQL as ANSI_SQL. - parked["dialect"] = used_dialect + _park_expression(parked, metric.get("expression"), used_dialect) meta = _build_meta(metric.get("ai_context"), stash.get("meta"), parked) if meta: measure["meta"] = meta diff --git a/converters/cube/tests/_roundtrip_helpers.py b/converters/cube/tests/_roundtrip_helpers.py index b354fe25..90ae8451 100644 --- a/converters/cube/tests/_roundtrip_helpers.py +++ b/converters/cube/tests/_roundtrip_helpers.py @@ -339,12 +339,13 @@ def build_ossie_model(rnd): if rnd.chance(0.4): lines.append(f" description: {_yaml_text(rnd.text())}") lines.append(" fields:") - for fname, expr, datatype, dialect, has_role in fields: + for fname, expr, datatype, forms, has_role in fields: lines.append(f" - name: {fname}") lines.append(" expression:") lines.append(" dialects:") - lines.append(f" - dialect: {dialect}") - lines.append(f" expression: {expr}") + for dialect, text in forms: + lines.append(f" - dialect: {dialect}") + lines.append(f" expression: {text}") lines.append(f" datatype: {datatype}") if has_role: lines.append(" dimension:") @@ -372,16 +373,33 @@ def build_ossie_model(rnd): OSSIE_DIALECTS = ["ANSI_SQL", "ANSI_SQL", "DATABRICKS", "SNOWFLAKE", "BIGQUERY"] +def _dialect_forms(rnd, expr): + """[(dialect, expression)] for one Ossie expression -- sometimes more than one. + + Cube has room for a single `sql` per member, so every dialect but the one export picks + has nowhere to go. Generating one dialect per expression could never show that. + """ + forms = [(rnd.pick(OSSIE_DIALECTS), expr)] + if rnd.chance(0.3): + # Sorted, not set order: a set iterates differently between processes, which + # made the seeded sweep generate different models per run and cost it the one + # thing it is for -- naming a seed a failure can be reproduced from. + alternative = rnd.pick(sorted(set(OSSIE_DIALECTS) - {forms[0][0]})) + forms.append((alternative, f"CAST({expr} AS VARCHAR)")) + return forms + + def _ossie_fields(rnd, dataset, dim_names): - """(name, expression, datatype, dialect, has_dimension_role) per field. + """(name, expression, datatype, dialects, has_dimension_role) per field. The last two are drawn rather than fixed, because both are places where export must make a Cube-shaped choice and then be able to undo it. A field with no `dimension` block is a *fact*; Cube has one kind of dimension, so export marks every member as one - and has to remember which were not. + and has to remember which were not. And Cube holds one `sql` per member, so an + expression offering several dialects loses the alternatives unless they are kept. """ def entry(name, expr, datatype): - return (name, expr, datatype, rnd.pick(OSSIE_DIALECTS), rnd.chance(0.5)) + return (name, expr, datatype, _dialect_forms(rnd, expr), rnd.chance(0.5)) fields = [entry("id", "id", "Integer")] for d in dim_names: @@ -403,11 +421,10 @@ def _ossie_metrics(rnd, fact, dim_names, fields_by_dataset): out = [] def block(name, expression): - entry = [f" - name: {name}", - " expression:", - " dialects:", - f" - dialect: {rnd.pick(OSSIE_DIALECTS)}", - f" expression: {expression}"] + entry = [f" - name: {name}", " expression:", " dialects:"] + for dialect, text in _dialect_forms(rnd, expression): + entry.append(f" - dialect: {dialect}") + entry.append(f" expression: {text}") if rnd.chance(0.3): entry.insert(1, f" description: {_yaml_text(rnd.text())}") out.append(entry) @@ -497,6 +514,11 @@ def datasets(model): def _dialected(entry): - """(normalized expression, dialect) for a field or metric.""" - dialect = entry["expression"]["dialects"][0] - return _normalize_refs(dialect["expression"]), dialect["dialect"] + """Every dialect of a field or metric expression, as (dialect, expression) pairs. + + All of them, not just the first: Cube keeps one `sql` per member, so the alternatives + are exactly what a round trip can quietly drop -- and comparing `dialects[0]` alone + would not notice. + """ + return tuple((d.get("dialect"), _normalize_refs(d.get("expression", ""))) + for d in entry["expression"]["dialects"]) From e34d56c44e73115ed0068d45b9f62de799f1bbe4 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Wed, 5 Aug 2026 02:03:56 +0500 Subject: [PATCH 43/46] Keep the second export cycle identical to the first [P1] The recorded key column list is *columns*, but the `computed_primary_key` inference read it as dimension names -- so a key column `id` alongside a computed field also named `id` came back flagged as computed, and the second export marked `LOWER(email)` as the key instead of synthesizing `id_pk`. Cube then deduplicated on a different value, which changes the counts it returns. The inference is skipped when `meta.ossie.primary_key` supplied the key, because those entries are columns by construction. Both Ossie documents were identical in that case; only the *Cube* model changed. So the property now runs a second export and requires it to reproduce the first, which is the only way to see a record that one side writes and the other reads differently. That found two more, neither reachable in a single cycle: - A decomposed metric's public measure was stashed verbatim and restored with references to hidden parts the next export no longer generated. Cube's verdict on the second cycle: "fact.crossing_part_1 cannot be resolved" -- a broken model. The public half is marked, so re-import rebuilds it from its expression and both halves are regenerated together. - `COUNT(DISTINCT DIM_0.ID)` was not recognized as the primary-key count because the comparison was case-sensitive, so cycle 1 emitted `count_distinct` and cycle 2 -- reading a canonically regenerated expression -- emitted the bare `count`. Compared on normalized identifiers now, which also means a metric spelling the key in any case gets Cube's fan-out-safe form. Non-blocking wording fixed too: the fallback may pick the first of several warehouse dialects, not only a sole one. Checked the new checks can fail: reverting the inference fix breaks 9 cases across the property sweep and the targeted two-cycle test. 561 tests with both gates, 534 with neither, 97% coverage. --- converters/cube/src/ossie_cube/_common.py | 14 ++++++++ converters/cube/src/ossie_cube/cube_to_osi.py | 25 ++++++++++++-- converters/cube/src/ossie_cube/osi_to_cube.py | 19 ++++++++--- converters/cube/tests/_roundtrip_helpers.py | 13 +++++++ converters/cube/tests/test_osi_to_cube.py | 8 +++-- converters/cube/tests/test_roundtrip.py | 34 +++++++++++++++++++ 6 files changed, 104 insertions(+), 9 deletions(-) diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index 91bb322d..a43ee3ac 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -589,6 +589,20 @@ def match_keys(identifier): return (content, content.upper()) +def normalized_expression(expr): + """`expr` with every `dataset.member` reference in its normalized form. + + For comparing two expressions that mean the same thing. Ossie identifiers are + case-insensitive, so `COUNT(DISTINCT DIM_0.ID)` and `COUNT(DISTINCT dim_0.id)` are one + expression -- and comparing them exactly made the primary-key count go unrecognized + whenever the metric spelled the key in another case. + """ + return DOTTED_REF_RE.sub( + lambda m: ".".join(normalize_identifier(part) + for part in split_dotted_ref(m.group(0))), + str(expr)) + + def datasets_in_expression(expr, prepared): """Dataset names referenced in `expr`, resolved against a prepared lookup map. diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 8712d761..0fbed899 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -587,9 +587,15 @@ def _convert_cube(cname, cube, plain, extra_joins, extra_measures, issues): # document afterwards -- a hand-authored model may name a real column that a # computed field happens to share a name with -- so it is recorded here rather # than guessed on the way back. - computed = [n for n in primary_key if n not in plain] - if computed: - stash["computed_primary_key"] = computed + # + # Not inferred when `meta.ossie.primary_key` supplied the key: those entries are + # Ossie *columns* by construction, and reading them as dimension names moved the + # key onto a computed dimension of the same name on the next cycle -- changing what + # Cube deduplicates on, and so the counts it returns. + if not parked.get("primary_key"): + computed = [n for n in primary_key if n not in plain] + if computed: + stash["computed_primary_key"] = computed if extra_joins: stash["extra_joins"] = extra_joins if extra_measures: @@ -1426,6 +1432,13 @@ def _convert_measure(cname, mname, metric_name, measure, context): and mtype not in CALCULATED_MEASURE_TYPES and not measure.get("filters") ) + decomposed = bool(parked_of(measure.get("meta")).get("decomposed")) + if decomposed: + # The public half of a decomposition. Its expression is the whole metric -- the + # references to its hidden parts inline back into it -- so export can rebuild both + # halves from that. Stashing the measure verbatim instead kept references to parts + # the next export does not generate, and Cube refused the result. + reconstructible = True # Fan-out: a non-idempotent aggregate over a dataset the graph can multiply. Cube # fixes this at query time by deduplicating on the primary key; a static expression @@ -1471,6 +1484,12 @@ def _convert_measure(cname, mname, metric_name, measure, context): snake(k): v for k, v in measure.items() if snake(k) not in ("description", "meta") } + elif decomposed: + # Nothing about the Cube spelling is worth keeping: its references point at hidden + # parts, and the next export regenerates both halves from the expression. Keeping + # the sql suppressed that regeneration, so the parts were never rebuilt and the + # measure referenced members that no longer existed. + pass elif sql is not None and not sql_is_reversible(sql, plain, cname): # Only a reference export cannot regenerate needs the original spelling: a # non-plain member (whose own SQL is inlined) or a cross-cube reference diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index f1dcf577..0b975155 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -61,6 +61,7 @@ DOTTED_REF_RE, lookup_map, resolve_identifier, + normalized_expression, quoted_runs, split_dotted_ref, require_str, @@ -582,7 +583,8 @@ def _report_dialect_fallback(issues, scope, used, preferred): if used in (None, DIALECT_ANSI, preferred): return issues.add(IssueType.APPROXIMATED, scope, - f"no ANSI_SQL expression; used the only dialect on offer ('{used}'). " + f"no ANSI_SQL expression; used the first warehouse dialect on offer " + f"('{used}'). " f"Cube passes SQL to the data source, so this is correct where the " f"model reads that warehouse -- pass --dialect {used} to say so.") @@ -1045,7 +1047,8 @@ def resolve_base(): else: measure = _measure_from_expression( expr, target, mname, stash, plan, tables) - _apply_measure_metadata(metric, measure, stash, used) + _apply_measure_metadata(metric, measure, stash, used, + decomposed=len(spans) > 1) _place(measures_by_cube, target, measure, name) return measures_by_cube @@ -1168,7 +1171,8 @@ def _measure_from_expression(expr, target, mname, stash, plan, tables): if func == "COUNT" and distinct: inner = distinct.group(1).strip() key = list((plan.get(target).primary_key if target in plan else ())) - if key and inner == primary_key_operand(target, key): + if key and (normalized_expression(inner) + == normalized_expression(primary_key_operand(target, key))): measure["type"] = "count" return measure func = "COUNT_DISTINCT" @@ -1195,7 +1199,8 @@ def _measure_from_expression(expr, target, mname, stash, plan, tables): return measure -def _apply_measure_metadata(metric, measure, stash, used_dialect=None): +def _apply_measure_metadata(metric, measure, stash, used_dialect=None, + decomposed=False): if stash.get("title"): # Not escaped: this came out of the stash, so it is already whatever Cube # needs it to be. Escaping it again turned a valid `Revenue \{USD\}` into @@ -1214,6 +1219,12 @@ def _apply_measure_metadata(metric, measure, stash, used_dialect=None): if datatype and datatype != AGG_TO_RESULT_DATATYPE.get(measure.get("type")): parked["datatype"] = datatype _park_expression(parked, metric.get("expression"), used_dialect) + if decomposed: + # The public half of a decomposition. Recorded so a re-import rebuilds it from its + # expression instead of restoring it verbatim: restoring kept its references to + # hidden parts the next export no longer generates, and Cube then refused the + # model -- "fact.crossing_part_1 cannot be resolved". + parked["decomposed"] = True meta = _build_meta(metric.get("ai_context"), stash.get("meta"), parked) if meta: measure["meta"] = meta diff --git a/converters/cube/tests/_roundtrip_helpers.py b/converters/cube/tests/_roundtrip_helpers.py index 90ae8451..bb3ed56a 100644 --- a/converters/cube/tests/_roundtrip_helpers.py +++ b/converters/cube/tests/_roundtrip_helpers.py @@ -510,9 +510,22 @@ def datasets(model): assert datasets(returned) == datasets(original), ( "Ossie -> Cube -> Ossie changed the datasets") + + # A second cycle has to produce the same Cube model as the first. Comparing only the + # Ossie ends misses a whole class: a record meant to be read one way that the next + # export reads another. `primary_key` recorded as *columns* was inferred back as + # *dimension names*, which moved Cube's deduplication key onto a computed dimension -- + # invisible in the Ossie comparison, and a different number out of Cube. + again, _ = convert_ossie_to_cube(back) + assert load_yaml_files(again) == load_yaml_files(files), ( + "Ossie -> Cube -> Ossie -> Cube did not reproduce the first Cube model") return files +def load_yaml_files(files): + return {name: load_yaml(text, name) for name, text in files.items()} + + def _dialected(entry): """Every dialect of a field or metric expression, as (dialect, expression) pairs. diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 48d5526b..42fab6e7 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -412,7 +412,11 @@ def test_a_ratio_is_split_into_one_measure_per_aggregate(): # stays correct when the cube is extended. assert orders["aov"] == { "name": "aov", "type": "number", - "sql": "{CUBE.aov_part_1} / {users.aov_part_2}"} + "sql": "{CUBE.aov_part_1} / {users.aov_part_2}", + # Marked as the public half of a decomposition, so a re-import rebuilds it from + # its expression rather than restoring sql that names parts the next export has + # not generated yet. + "meta": {"ossie": {"decomposed": True}}} def test_a_dotted_token_inside_a_string_literal_is_left_alone(): @@ -930,7 +934,7 @@ def test_a_model_with_only_a_warehouse_dialect_still_converts(): assert [d["name"] for d in cube["dimensions"]] == ["o_orderkey", "o_orderdate"] assert [m["name"] for m in cube["measures"]] == ["total_revenue", "order_count"] # Reported, because Cube will pass that SQL to whatever the data source is. - assert any("only dialect on offer" in i.detail + assert any("first warehouse dialect on offer" in i.detail for i in issues.of_type(IssueType.APPROXIMATED)) diff --git a/converters/cube/tests/test_roundtrip.py b/converters/cube/tests/test_roundtrip.py index a84eb65c..dfefbb54 100644 --- a/converters/cube/tests/test_roundtrip.py +++ b/converters/cube/tests/test_roundtrip.py @@ -36,6 +36,7 @@ parse_files) from ossie_cube import convert_cube_to_ossie, convert_ossie_to_cube +from ossie_cube._common import OSSIE_VERSION FIXTURES = ["fixtureA_cube", "tpcds_cube"] @@ -231,3 +232,36 @@ def shape(model): } assert shape(after) == shape(before) + + +@cube_gate +def test_two_export_cycles_produce_the_same_cube_model(): + """`Ossie -> Cube -> Ossie -> Cube`, on the collision that made the first differ from + the second: a key column `id` alongside a computed field also named `id`. + + Comparing only the Ossie ends cannot see this class. A record meant to be read one way + that the next export reads another leaves both Ossie documents identical while the Cube + model changes -- here the key moved off the column and onto `LOWER(email)`, so Cube + deduplicated on a different value and returned different counts. + """ + ossie = ( + f"version: {OSSIE_VERSION}\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: shop.public.orders\n" + " primary_key:\n - id\n" + " fields:\n" + " - name: id\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: LOWER(email)\n" + " datatype: String\n" + ) + first, _ = convert_ossie_to_cube(ossie) + second, _ = convert_ossie_to_cube(convert_cube_to_ossie(first)[0]) + assert parse_files(second) == parse_files(first) + + keys = [d for d in parse_files(first)["model/cubes/orders.yml"]["cubes"][0][ + "dimensions"] if d.get("primary_key")] + assert [(d["name"], d["sql"]) for d in keys] == [("id_pk", "id")] + assert_cube_compiles(second, "second export cycle") From db392a21a7cab3f3fcf2930146da4f73c97f0d3c Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Wed, 5 Aug 2026 14:49:14 +0500 Subject: [PATCH 44/46] Emit a view that cannot collide with a cube MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cube keeps cubes and views in one global namespace, so a view may not share a name with a cube. The exporter did not check, and an Ossie model named after one of its own datasets produced exactly that -- Cube rejected the whole model with "Cannot read properties of undefined (reading 'toString')". Not an exotic input: it is what every Databricks metric view over a same-named table converts to, and `databricks_ossie.yaml` is one. The generated view becomes `_view` and the model's own name is recorded in `meta.ossie.model_name`, since the mapped view's name is the model's name on the way back and the rename would otherwise stick. Renamed rather than refused because a cube is addressed by joins and by every member reference, a generated view by nothing. That went unnoticed because the gate meant to catch it was dropping half its input. `cube_compile.js` keyed model files by basename; Cube's own FileRepository keys them by path relative to the model root, so `cubes/orders.yml` and `views/orders.yml` collided and one was discarded silently. A valid cube plus an invalid same-named view reported COMPILED OK, while the identical pair under distinct names failed as it should. Keyed by relative path now, matching Cube, and duplicate keys are refused outright rather than resolved by chance -- a gate that quietly compiles less than it was given is worse than no gate. The same flattening was already fixed a layer up in `_cube_gate.py`, where the temp files are written; the JS undid it. With the gate honest, a second defect surfaced one cycle out: a `DATABRICKS` metric came back as `ANSI_SQL` on the second export. The verbatim-restore path hands back the Cube SQL a previous import stashed instead of picking a dialect, so it had no dialect to pass to `_park_expression` and the label was dropped. It falls back to the sole declared dialect, which is the one that SQL came from. `Ossie -> Cube -> Ossie` is byte-stable from the first cycle now; the existing one-cycle comparison could not see this, since cycle one was correct. Also, `validation/validate.py` reports a missing `jsonschema` by calling `sys.exit(1)` at import time, and SystemExit does not derive from Exception -- so it escaped the guard around the validator import and aborted pytest *collection*. The whole suite refused to run on any machine without jsonschema, the exact case `validator_gate` exists to skip. Smaller, from the same review: - The CLI's file I/O used the platform default encoding, so a model carrying any non-ASCII text died under a non-UTF-8 locale: `title: Größe` gave `UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3`. Pinned to UTF-8. - The property generator hard-coded the spec version instead of using `OSSIE_VERSION`. - `has_top_level_operator` treated any depth-0 whitespace as structure, including the trailing newline off a YAML block scalar, and parenthesized a lone `SUM(x)` needlessly. - The interop matrix counted failures by adding a bool; made explicit. Checked the new checks can fail. Reverting the gate keying (and its duplicate guard) leaves the collision test passing a model Cube refuses; reverting the view rename breaks 3 tests including the Databricks compile; reverting the dialect fallback breaks the two-cycle stability test; reverting the encoding breaks the non-UTF-8 CLI test. 566 tests with both gates, 399 with neither, 97% coverage. --- converters/cube/README.md | 2 +- converters/cube/src/ossie_cube/cli.py | 8 +- converters/cube/src/ossie_cube/cube_to_osi.py | 7 +- converters/cube/src/ossie_cube/expressions.py | 5 +- converters/cube/src/ossie_cube/osi_to_cube.py | 53 +++++++++++- converters/cube/tests/_cube_gate.py | 7 +- converters/cube/tests/_roundtrip_helpers.py | 4 +- converters/cube/tests/test_cli.py | 39 +++++++++ converters/cube/tests/test_roundtrip.py | 81 ++++++++++++++++++- converters/cube/tools/cube_compile.js | 41 ++++++++-- converters/cube/tools/interop_matrix.py | 3 +- 11 files changed, 230 insertions(+), 20 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index 6768ae10..4473715e 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -116,7 +116,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | Apache Ossie | Cube | Notes | |---|---|---| | `semantic_model` | a **view** | Cube users are view-first, and Cube's agent reads `meta.ai_context` only from views and members -- so the view, not any cube, is the model boundary. | -| `semantic_model.name` | view name | Import: the mapped view's name (override with `--name`). | +| `semantic_model.name` | view name | Import: the mapped view's name (override with `--name`). Cube keeps cubes and views in one namespace, so a model named after one of its own datasets — what a Databricks metric view over a same-named table produces — would emit two members of one name and Cube refuses the model. The generated view becomes `_view` and the model's own name is recorded in `meta.ossie.model_name`, so import hands back the original rather than the renamed view's. | | `model.description` / `ai_context.instructions` | view `description` / `meta.ai_context` | Import: taken from the sole view, or `--view`. | | dataset | `cubes[]` entry in `model/cubes/.yml` | Import: a non-canonical original path is stashed and restored on export. | | `dataset.source` (dotted) | `sql_table` | Passed through verbatim; Cube interpolates it straight into `FROM`, so no catalog/schema split is needed. | diff --git a/converters/cube/src/ossie_cube/cli.py b/converters/cube/src/ossie_cube/cli.py index d665be0c..82d7ab76 100644 --- a/converters/cube/src/ossie_cube/cli.py +++ b/converters/cube/src/ossie_cube/cli.py @@ -141,7 +141,7 @@ def _collect_file(files, path, anchor): raise ConversionError( f"two inputs both resolve to '{rel}'; pass their common parent " f"directory instead, or rename one") - with open(path) as fh: + with open(path, encoding="utf-8") as fh: files[rel] = fh.read() @@ -157,14 +157,14 @@ def main(argv=None): args = _build_parser().parse_args(argv) try: if args.command == "export": - with open(args.input) as fh: + with open(args.input, encoding="utf-8") as fh: ossie_yaml = fh.read() files, issues = convert_ossie_to_cube( ossie_yaml, dialect=args.dialect, base_cube=args.base_cube) for rel, text in files.items(): dest = os.path.join(args.output, *rel.split("/")) os.makedirs(os.path.dirname(dest) or ".", exist_ok=True) - with open(dest, "w") as fh: + with open(dest, "w", encoding="utf-8") as fh: fh.write(text) print(f"Wrote {len(files)} file(s) to {args.output}", file=sys.stderr) _report(issues) @@ -175,7 +175,7 @@ def main(argv=None): files, model_name=args.name, view=args.view, strict_fanout=args.strict_fanout) if args.output: - with open(args.output, "w") as fh: + with open(args.output, "w", encoding="utf-8") as fh: fh.write(out) else: sys.stdout.write(out) diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 0fbed899..44bd3c34 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -135,7 +135,12 @@ def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=False mapped_view = views.get(mapped_name) or {} cubes = _order_by_view(cubes, mapped_view) - model = {"name": model_name or mapped_name or "cube_model"} + # A previous export records the model's own name when it could not also be the + # view's -- Cube gives cubes and views one namespace, so a model named after one of + # its datasets has its view renamed. Without this the name would silently become + # the renamed view's on the way back. + parked_model_name = parked_of(mapped_view.get("meta")).get("model_name") + model = {"name": model_name or parked_model_name or mapped_name or "cube_model"} if mapped_view.get("description"): model["description"] = unescape_braces_from_cube( mapped_view["description"]) diff --git a/converters/cube/src/ossie_cube/expressions.py b/converters/cube/src/ossie_cube/expressions.py index 59a73391..1e841a28 100644 --- a/converters/cube/src/ossie_cube/expressions.py +++ b/converters/cube/src/ossie_cube/expressions.py @@ -294,7 +294,10 @@ def has_top_level_operator(expr): parentheses: a lone `SUM(x)` does not, `SUM(x) / 2` does. """ depth, quote = 0, None - for ch in str(expr): + # Stripped first: interior whitespace is what implies structure, so a trailing + # newline off a YAML block scalar (`expression: |`) is not evidence of any, and + # counting it wrapped a lone `SUM(x)\n` in parentheses it did not need. + for ch in str(expr).strip(): if quote: if ch == quote: quote = None diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 0b975155..f6f3ecc5 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -569,7 +569,15 @@ def _park_expression(parked, expression, used): dialects = (expression or {}).get("dialects") or [] if len(dialects) > 1: parked["expression"] = expression - elif used not in (None, DIALECT_ANSI): + return + if used is None and len(dialects) == 1: + # The verbatim-restore path hands back the Cube SQL a previous import stashed + # rather than picking a dialect, so it has no `used` to pass -- but the label + # still has to survive, or Cube's `sql` is re-imported as ANSI_SQL. A + # `DATABRICKS` metric restored from the stash lost its label on exactly this + # path, one cycle later than the label loss already fixed for the direct one. + used = dialects[0].get("dialect") + if used not in (None, DIALECT_ANSI): parked["dialect"] = used @@ -1244,6 +1252,32 @@ def _balanced(s): # --- views ---------------------------------------------------------------------- +def _uncollided_view_name(vname, cube_names, model, parked, issues): + """A generated view name that no cube already owns. + + Returns `vname` untouched when it is free. Otherwise appends `_view` (then + `_view_2`, ...) and records the model's real name under `meta.ossie.model_name`, so + the next import restores it rather than adopting the renamed view's. + + Renaming rather than refusing: an Ossie model whose name matches one of its own + datasets is a perfectly ordinary document -- every Databricks metric view over a + same-named table produces one -- and it is the model most worth converting. + """ + taken = {str(name).lower() for name in cube_names.values()} + if vname.lower() not in taken: + return vname + candidate, suffix = f"{vname}_view", 2 + while candidate.lower() in taken: + candidate, suffix = f"{vname}_view_{suffix}", suffix + 1 + parked["model_name"] = model.get("name") + issues.add( + IssueType.PARKED_IN_META, f"view '{candidate}'", + f"the model name '{vname}' is also a cube name, and Cube keeps cubes and " + f"views in one namespace, so the view is emitted as '{candidate}'; the " + f"model's name is preserved under meta.ossie.model_name") + return candidate + + def _build_views(model, model_stash, cube_names, relationships, datasets, base_cube, emitted_members, issues): """Return {file path: [view dict, ...]}. @@ -1263,10 +1297,25 @@ def _build_views(model, model_stash, cube_names, relationships, datasets, if foreign: parked["custom_extensions"] = foreign + # Cube keeps cubes and views in one global namespace, so a view may not share a + # name with a cube. The model name and a dataset name being equal is not exotic -- + # it is what the Databricks metric-view converter produces, a metric view `orders` + # over a table `orders` -- and the collision made Cube reject the whole model with + # `Cannot read properties of undefined`. The view is what gets renamed: cubes are + # referred to by joins and by every member reference, the view by nothing. + model_vname = sanitize_name(model.get("name", "model"), "Model", set()) + out = {} if "views" in model_stash: mapped = model_stash.get("mapped_view") paths = model_stash.get("view_files") or {} + # The mapped view's name is the model's name on re-import, so a difference + # between them has to be recorded or the next import adopts the view's name. + # This is also what keeps a renamed view stable across a second cycle: the + # rename below is not re-derived from the stash, and `meta.ossie` does not + # survive stashing. + if mapped is not None and mapped != model_vname: + parked["model_name"] = model.get("name") if foreign and mapped is None: # The model's own metadata rides on the view that represents it, and # there isn't one: the source Cube model had several views and none was @@ -1297,7 +1346,7 @@ def _build_views(model, model_stash, cube_names, relationships, datasets, out.setdefault(path, []).append(view) return out - vname = sanitize_name(model.get("name", "model"), "Model", set()) + vname = _uncollided_view_name(model_vname, cube_names, model, parked, issues) view = {"name": vname} if model.get("description"): view["description"] = escape_braces_for_cube(model["description"]) diff --git a/converters/cube/tests/_cube_gate.py b/converters/cube/tests/_cube_gate.py index e22121b9..6bb1724d 100644 --- a/converters/cube/tests/_cube_gate.py +++ b/converters/cube/tests/_cube_gate.py @@ -108,7 +108,12 @@ def _load_validator(): _VALIDATOR_MODULE = _load_validator() _SCHEMA = json.loads( (_REPO_ROOT / "core-spec" / "osi-schema.json").read_text()) -except Exception as exc: # missing jsonschema, a moved schema, a changed script +# SystemExit is deliberately included: `validate.py` reports a missing `jsonschema` by +# calling `sys.exit(1)` at import time, and SystemExit derives from BaseException, so an +# `except Exception` let it escape and abort pytest *collection* -- the entire suite +# refused to run on any machine without jsonschema, which is the exact situation this +# gate is supposed to skip over. +except (Exception, SystemExit) as exc: # missing dep, a moved schema, a changed script _VALIDATOR_ERROR = f"{type(exc).__name__}: {exc}" diff --git a/converters/cube/tests/_roundtrip_helpers.py b/converters/cube/tests/_roundtrip_helpers.py index bb3ed56a..ff6befd2 100644 --- a/converters/cube/tests/_roundtrip_helpers.py +++ b/converters/cube/tests/_roundtrip_helpers.py @@ -50,7 +50,7 @@ import string from ossie_cube import convert_cube_to_ossie, convert_ossie_to_cube -from ossie_cube._common import dump_yaml, load_yaml +from ossie_cube._common import OSSIE_VERSION, dump_yaml, load_yaml # Aggregates whose value survives duplicate rows, so they are safe on any cube. IDEMPOTENT_AGGS = ["count_distinct", "count_distinct_approx", "min", "max"] @@ -315,7 +315,7 @@ def build_ossie_model(rnd): dim_names = [f"dim_{i}" for i in range(rnd.count(1, 2))] fact = "fact" - lines = ["version: 0.2.0.dev0", "semantic_model:", "- name: shop"] + lines = [f"version: {OSSIE_VERSION}", "semantic_model:", "- name: shop"] if rnd.chance(0.5): lines.append(f" description: {_yaml_text(rnd.text())}") lines.append(" datasets:") diff --git a/converters/cube/tests/test_cli.py b/converters/cube/tests/test_cli.py index 2314280d..e6f9cd43 100644 --- a/converters/cube/tests/test_cli.py +++ b/converters/cube/tests/test_cli.py @@ -22,6 +22,11 @@ goes, since those are the converter's contract with a shell script. """ +import os +import pathlib +import subprocess +import sys + import pytest from _util import REPO_ROOT, load_fixture_dir, parse @@ -310,3 +315,37 @@ def test_no_subcommand_is_a_usage_error(): with pytest.raises(SystemExit) as excinfo: main([]) assert excinfo.value.code == 2 + + +def test_a_non_ascii_model_converts_under_a_non_utf8_locale(tmp_path): + """A German title or a Russian description must not depend on the machine's locale. + + Python's `open()` defaults to the locale's preferred encoding, so on any host that is + not UTF-8 -- a Windows console, a container with `LC_ALL=C` -- reading the model died + with `UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3`. Run in a subprocess + because the encoding is fixed from the environment at interpreter start, so an + in-process check would only ever see the test runner's own UTF-8. + """ + model = tmp_path / "model" + model.mkdir() + (model / "orders.yml").write_text( + _ORDERS.replace(" primary_key: true\n", + " primary_key: true\n title: Größe\n"), + encoding="utf-8") + out = tmp_path / "out.yaml" + + env = { + **os.environ, + "LC_ALL": "C", "LANG": "C", + # Both would otherwise put the interpreter back into UTF-8 and hide the point. + "PYTHONUTF8": "0", "PYTHONCOERCECLOCALE": "0", + "PYTHONPATH": str(pathlib.Path(__file__).resolve().parents[1] / "src"), + } + result = subprocess.run( + [sys.executable, "-c", + "import sys; from ossie_cube.cli import main; sys.exit(main(sys.argv[1:]))", + "import", "-i", str(model), "-o", str(out)], + capture_output=True, text=True, env=env) + + assert result.returncode == 0, f"{result.stdout}\n{result.stderr}" + assert "Größe" in out.read_text(encoding="utf-8") diff --git a/converters/cube/tests/test_roundtrip.py b/converters/cube/tests/test_roundtrip.py index dfefbb54..b31355b6 100644 --- a/converters/cube/tests/test_roundtrip.py +++ b/converters/cube/tests/test_roundtrip.py @@ -35,7 +35,7 @@ from _util import (REPO_ROOT, canon, load_fixture, load_fixture_dir, parse, parse_files) -from ossie_cube import convert_cube_to_ossie, convert_ossie_to_cube +from ossie_cube import IssueType, convert_cube_to_ossie, convert_ossie_to_cube from ossie_cube._common import OSSIE_VERSION FIXTURES = ["fixtureA_cube", "tpcds_cube"] @@ -234,6 +234,85 @@ def shape(model): assert shape(after) == shape(before) +@cube_gate +def test_the_compile_gate_does_not_silently_drop_a_same_named_file(): + """A meta-test: the gate has to actually see every file it is handed. + + Cube keys model files by their path relative to the model root, and the gate passed + basenames instead -- so `cubes/orders.yml` and `views/orders.yml` collided and one was + dropped without a word. An invalid model then reported COMPILED OK, which is how the + cube/view namespace collision above went unnoticed. The converter emits exactly this + pair of names, so this is the arrangement that has to fail loudly. + """ + files = { + "model/cubes/orders.yml": + "cubes:\n- name: orders\n sql_table: public.orders\n" + " dimensions:\n - name: id\n sql: id\n type: number\n" + " primary_key: true\n", + # Same basename, different directory, and invalid: it includes a member no cube + # defines. If the gate drops this file, it reports success. + "model/views/orders.yml": + "views:\n- name: orders_view\n cubes:\n - join_path: orders\n" + " includes:\n - id\n - no_such_member\n", + } + with pytest.raises(AssertionError, match="Cube refused the model"): + assert_cube_compiles(files, "same-basename files") + + +def test_a_model_named_after_one_of_its_datasets_does_not_collide_in_cube(): + """Cube keeps cubes and views in one namespace, so a model named `orders` over a + dataset named `orders` cannot emit both under that name -- Cube rejected the whole + model with `Cannot read properties of undefined (reading 'toString')`. + + This is the shape every Databricks metric view over a same-named table produces, so + the view is renamed rather than the model refused, and the model's own name is + recorded so the trip back does not adopt the renamed view's. + """ + src = load_fixture("databricks_ossie.yaml") + assert parse(src)["semantic_model"][0]["name"] == "orders" + + files, issues = convert_ossie_to_cube(src) + assert set(files) == {"model/cubes/orders.yml", "model/cubes/customer.yml", + "model/views/orders_view.yml"} + view = parse(files["model/views/orders_view.yml"])["views"][0] + assert view["name"] == "orders_view" + assert view["meta"]["ossie"]["model_name"] == "orders" + assert any("one namespace" in i.detail + for i in issues.of_type(IssueType.PARKED_IN_META)) + + # And the name comes back, rather than becoming `orders_view`. + back, _ = convert_cube_to_ossie(files) + assert parse(back)["semantic_model"][0]["name"] == "orders" + + +@cube_gate +def test_a_renamed_view_still_compiles_and_stays_renamed(): + """The rename has to survive a second export too. It is not re-derived from the + stash -- `meta.ossie` is stripped when a view is stashed -- so the recorded model + name is what keeps cycle two from emitting a colliding view again.""" + files, _ = convert_ossie_to_cube(load_fixture("databricks_ossie.yaml")) + assert_cube_compiles(files, "model named after its own dataset") + second, _ = convert_ossie_to_cube(convert_cube_to_ossie(files)[0]) + assert set(second) == set(files) + assert parse(second["model/views/orders_view.yml"])["views"][0][ + "name"] == "orders_view" + assert_cube_compiles(second, "model named after its own dataset (cycle 2)") + + +def test_a_model_from_another_converter_is_stable_after_one_cycle(): + """`Ossie -> Cube -> Ossie` twice, compared byte-for-byte. + + The one-cycle comparison above cannot see a value that survives the first trip and + is dropped on the second: the `DATABRICKS` label on a metric restored verbatim from + the stash came back as `ANSI_SQL` on cycle two, because the verbatim path hands back + stashed Cube SQL instead of picking a dialect and so had no label to re-park. + """ + first, _ = convert_cube_to_ossie( + convert_ossie_to_cube(load_fixture("databricks_ossie.yaml"))[0]) + second, _ = convert_cube_to_ossie(convert_ossie_to_cube(first)[0]) + assert second == first + + @cube_gate def test_two_export_cycles_produce_the_same_cube_model(): """`Ossie -> Cube -> Ossie -> Cube`, on the collision that made the first differ from diff --git a/converters/cube/tools/cube_compile.js b/converters/cube/tools/cube_compile.js index 3e97920b..62023b80 100644 --- a/converters/cube/tools/cube_compile.js +++ b/converters/cube/tools/cube_compile.js @@ -29,8 +29,8 @@ * exactly that kind were found by running this. * * Needs a built Cube checkout (`yarn build` in the monorepo, or an installed - * node_modules with dist/). `tests/test_cube_compiles.py` skips when there isn't one, - * so this is a local and release-time gate rather than a CI one. + * node_modules with dist/). The tests behind `cube_gate` skip when there isn't one, so + * this is a local and release-time gate rather than a CI one. */ const fs = require('fs'); @@ -76,15 +76,44 @@ if (!files.length) { process.exit(2); } -// Cube keys models by file name, and its loader does not care about directories, so a -// flat list is enough -- `cubes/orders.yml` and `views/sales.yml` compile together. +/* The deepest directory containing every input, which is what the relative keys are + * relative to. One file has no shared prefix to find, so its own directory is the root. */ +function commonRoot(paths) { + const dirs = paths.map((p) => path.dirname(path.resolve(p)).split(path.sep)); + let shared = dirs[0]; + for (const parts of dirs.slice(1)) { + let i = 0; + while (i < shared.length && i < parts.length && shared[i] === parts[i]) i += 1; + shared = shared.slice(0, i); + } + return shared.join(path.sep) || path.sep; +} + +// Cube keys model files by their path *relative to the model root* -- its own +// FileRepository walks the tree and joins the directory back on, so a cube at +// `cubes/orders.yml` is keyed by exactly that. Using the basename instead let two files +// of one name collide, and the loser was dropped without a word: `cubes/orders.yml` plus +// an invalid `views/orders.yml` reported COMPILED OK, while the same two files under +// distinct names failed as they should. A gate that quietly drops half its input is +// worse than no gate, and the Databricks fixture emits that exact pair. +const root = commonRoot(files); const dataSchemaFiles = files.map((p) => ({ - fileName: path.basename(p), + fileName: path.relative(root, path.resolve(p)), content: fs.readFileSync(p, 'utf8'), })); +// Nothing may share a key, or Cube silently sees fewer files than we passed. +const seen = new Map(); +for (const f of dataSchemaFiles) { + if (seen.has(f.fileName)) { + console.log(`COMPILE FAILED\nduplicate model key '${f.fileName}'`); + process.exit(1); + } + seen.set(f.fileName, true); +} + const { compiler } = prepareCompiler( - { localPath: () => path.dirname(files[0]), dataSchemaFiles: () => Promise.resolve(dataSchemaFiles) }, + { localPath: () => root, dataSchemaFiles: () => Promise.resolve(dataSchemaFiles) }, { adapter: 'postgres' }); compiler.compile() diff --git a/converters/cube/tools/interop_matrix.py b/converters/cube/tools/interop_matrix.py index 89939abd..bb194087 100644 --- a/converters/cube/tools/interop_matrix.py +++ b/converters/cube/tools/interop_matrix.py @@ -257,7 +257,8 @@ def main(): tail = (r.stderr.strip().splitlines() or [""])[-1] result = "SKIP" if _is_environment_failure(r.stderr) else "FAIL" note = tail[:40] - failures += result == "FAIL" + if result == "FAIL": + failures += 1 else: result = "OK" if produced_output(dest, is_dir) else "EMPTY" print(f"{name:<12} {result:<7} {warns:>5} {foreign:>8} {note}") From c000d0136e695ad7b1d72575b3fe92c3392428be Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Wed, 5 Aug 2026 15:12:46 +0500 Subject: [PATCH 45/46] Preserve the model name whenever the view cannot carry it [P1] The record added for cube/view collisions was scoped to that one cause, and the ordinary causes went unrecorded. `Sales Model` is a legal Ossie name that cannot be a Cube identifier, so it exported as view `sales_model` with nothing parked and came back named `sales_model`. The same for `--name "Sales Model"` over a stashed view already called `sales_model`: the stashed branch compared the mapped name against the *sanitized* model name, the two matched, and the override was silently undone. Keyed on the difference now rather than on the reason for it -- the raw name is preserved whenever it differs from the name of the view that will carry it, whether that difference comes from sanitizing, an override, or a collision. A model whose name is already its view's name still parks nothing, so an ordinary Cube document stays clean. Verified stable over three cycles, since the value has to survive being read back out of the stash and not merely written once. Also non-blocking, from the same review: `test_two_export_cycles_produce_the_same_cube_model` was entirely behind the optional Cube gate, but comparing two exports needs no Cube installation -- so the regression it exists for was unchecked everywhere without a built checkout, CI included. Split, with only `assert_cube_compiles` gated. An audit of every `cube_gate` test found one more of mine with the same mistake (`test_a_renamed_view_still_compiles_and_stays_renamed`); split the same way. The rest are genuinely Cube-only. Checked the new checks can fail: restoring the collision-only rule breaks both name tests, and the two split tests now run (and pass) with no Cube checkout present where they were previously skipped. 571 tests with both gates, 404 with neither, 97% coverage. --- converters/cube/README.md | 2 +- converters/cube/src/ossie_cube/osi_to_cube.py | 59 +++++--- converters/cube/tests/test_roundtrip.py | 128 +++++++++++++++--- 3 files changed, 147 insertions(+), 42 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index 4473715e..a6ef7939 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -116,7 +116,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | Apache Ossie | Cube | Notes | |---|---|---| | `semantic_model` | a **view** | Cube users are view-first, and Cube's agent reads `meta.ai_context` only from views and members -- so the view, not any cube, is the model boundary. | -| `semantic_model.name` | view name | Import: the mapped view's name (override with `--name`). Cube keeps cubes and views in one namespace, so a model named after one of its own datasets — what a Databricks metric view over a same-named table produces — would emit two members of one name and Cube refuses the model. The generated view becomes `_view` and the model's own name is recorded in `meta.ossie.model_name`, so import hands back the original rather than the renamed view's. | +| `semantic_model.name` | view name | Import: the mapped view's name (override with `--name`). Export: whenever the emitted view cannot carry the name exactly, the original is recorded in `meta.ossie.model_name` and import hands that back instead of the view's. Three causes — the name is not a valid Cube identifier (`Sales Model` → view `sales_model`); a `--name` override the stashed view does not match; or the name is also a dataset's, since Cube keeps cubes and views in one namespace and would refuse a model with two members of one name (what a Databricks metric view over a same-named table produces), so the generated view becomes `_view`. | | `model.description` / `ai_context.instructions` | view `description` / `meta.ai_context` | Import: taken from the sole view, or `--view`. | | dataset | `cubes[]` entry in `model/cubes/.yml` | Import: a non-canonical original path is stashed and restored on export. | | `dataset.source` (dotted) | `sql_table` | Passed through verbatim; Cube interpolates it straight into `FROM`, so no catalog/schema split is needed. | diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index f6f3ecc5..c86066e1 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -1252,16 +1252,13 @@ def _balanced(s): # --- views ---------------------------------------------------------------------- -def _uncollided_view_name(vname, cube_names, model, parked, issues): +def _uncollided_view_name(vname, cube_names): """A generated view name that no cube already owns. - Returns `vname` untouched when it is free. Otherwise appends `_view` (then - `_view_2`, ...) and records the model's real name under `meta.ossie.model_name`, so - the next import restores it rather than adopting the renamed view's. - - Renaming rather than refusing: an Ossie model whose name matches one of its own - datasets is a perfectly ordinary document -- every Databricks metric view over a - same-named table produces one -- and it is the model most worth converting. + Returns `vname` untouched when it is free, otherwise appends `_view` (then + `_view_2`, ...). Renaming rather than refusing: an Ossie model whose name matches one + of its own datasets is a perfectly ordinary document -- every Databricks metric view + over a same-named table produces one -- and it is the model most worth converting. """ taken = {str(name).lower() for name in cube_names.values()} if vname.lower() not in taken: @@ -1269,15 +1266,32 @@ def _uncollided_view_name(vname, cube_names, model, parked, issues): candidate, suffix = f"{vname}_view", 2 while candidate.lower() in taken: candidate, suffix = f"{vname}_view_{suffix}", suffix + 1 - parked["model_name"] = model.get("name") - issues.add( - IssueType.PARKED_IN_META, f"view '{candidate}'", - f"the model name '{vname}' is also a cube name, and Cube keeps cubes and " - f"views in one namespace, so the view is emitted as '{candidate}'; the " - f"model's name is preserved under meta.ossie.model_name") return candidate +def _record_model_name(parked, model, view_name, issues, collided=False): + """Preserve the model's own name when the view carrying it is named something else. + + The mapped view's name *is* the model's name on re-import, so any difference at all + has to be recorded or the name silently changes. Keying this on the difference rather + than on the reason for it is the point: a cube/view collision is only one cause, and + scoping the record to that one let the ordinary ones through. `Sales Model` is a + legal Ossie name and can only be a Cube view named `sales_model`, so it came back + sanitized; likewise `--name 'Sales Model'` over a stashed view already called + `sales_model`, where the sanitized forms matched and the raw name did not. + """ + raw = model.get("name") + if raw is None or raw == view_name: + return + parked["model_name"] = raw + why = (f"the model name '{raw}' is also a cube name, and Cube keeps cubes and views " + f"in one namespace, so " if collided else "") + issues.add( + IssueType.PARKED_IN_META, f"view '{view_name}'", + f"{why}the view is emitted as '{view_name}' rather than '{raw}'; the model's " + f"name is preserved under meta.ossie.model_name") + + def _build_views(model, model_stash, cube_names, relationships, datasets, base_cube, emitted_members, issues): """Return {file path: [view dict, ...]}. @@ -1309,13 +1323,13 @@ def _build_views(model, model_stash, cube_names, relationships, datasets, if "views" in model_stash: mapped = model_stash.get("mapped_view") paths = model_stash.get("view_files") or {} - # The mapped view's name is the model's name on re-import, so a difference - # between them has to be recorded or the next import adopts the view's name. - # This is also what keeps a renamed view stable across a second cycle: the - # rename below is not re-derived from the stash, and `meta.ossie` does not - # survive stashing. - if mapped is not None and mapped != model_vname: - parked["model_name"] = model.get("name") + # Compared against the *raw* name, not the sanitized one: a stashed view named + # `sales_model` and a model named `Sales Model` have equal sanitized forms, so + # comparing those saw no difference and the name came back sanitized. This is + # also what keeps a renamed view stable across a second cycle -- the rename is + # not re-derived from the stash, and `meta.ossie` does not survive stashing. + if mapped is not None: + _record_model_name(parked, model, mapped, issues) if foreign and mapped is None: # The model's own metadata rides on the view that represents it, and # there isn't one: the source Cube model had several views and none was @@ -1346,7 +1360,8 @@ def _build_views(model, model_stash, cube_names, relationships, datasets, out.setdefault(path, []).append(view) return out - vname = _uncollided_view_name(model_vname, cube_names, model, parked, issues) + vname = _uncollided_view_name(model_vname, cube_names) + _record_model_name(parked, model, vname, issues, collided=vname != model_vname) view = {"name": vname} if model.get("description"): view["description"] = escape_braces_for_cube(model["description"]) diff --git a/converters/cube/tests/test_roundtrip.py b/converters/cube/tests/test_roundtrip.py index b31355b6..b06cf350 100644 --- a/converters/cube/tests/test_roundtrip.py +++ b/converters/cube/tests/test_roundtrip.py @@ -285,20 +285,94 @@ def test_a_model_named_after_one_of_its_datasets_does_not_collide_in_cube(): assert parse(back)["semantic_model"][0]["name"] == "orders" -@cube_gate -def test_a_renamed_view_still_compiles_and_stays_renamed(): +def test_a_renamed_view_stays_renamed_on_the_second_export(): """The rename has to survive a second export too. It is not re-derived from the stash -- `meta.ossie` is stripped when a view is stashed -- so the recorded model name is what keeps cycle two from emitting a colliding view again.""" files, _ = convert_ossie_to_cube(load_fixture("databricks_ossie.yaml")) - assert_cube_compiles(files, "model named after its own dataset") second, _ = convert_ossie_to_cube(convert_cube_to_ossie(files)[0]) assert set(second) == set(files) assert parse(second["model/views/orders_view.yml"])["views"][0][ "name"] == "orders_view" + + +@cube_gate +def test_a_renamed_view_compiles_on_both_cycles(): + files, _ = convert_ossie_to_cube(load_fixture("databricks_ossie.yaml")) + assert_cube_compiles(files, "model named after its own dataset") + second, _ = convert_ossie_to_cube(convert_cube_to_ossie(files)[0]) assert_cube_compiles(second, "model named after its own dataset (cycle 2)") +_SALES_MODEL = ( + f"version: {OSSIE_VERSION}\n" + "semantic_model:\n" + "- name: Sales Model\n" + " datasets:\n" + " - name: orders\n" + " source: shop.public.orders\n" + " primary_key:\n - id\n" + " fields:\n" + " - name: id\n dimension: {}\n datatype: Integer\n" + " expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: id\n" +) + + +def test_a_model_name_needing_sanitizing_is_preserved(): + """`Sales Model` is a legal Ossie name and cannot be a Cube identifier, so the view is + `sales_model` -- and the model came back named `sales_model` too. + + The record was scoped to cube/view collisions, which is the rarer cause; plain + sanitization is the common one and went unrecorded. Three cycles because the value has + to survive being read back out of the stash, not just written once. + """ + files, issues = convert_ossie_to_cube(_SALES_MODEL) + view = parse(files["model/views/sales_model.yml"])["views"][0] + assert view["name"] == "sales_model" + assert view["meta"]["ossie"]["model_name"] == "Sales Model" + assert any("preserved under meta.ossie.model_name" in i.detail + for i in issues.of_type(IssueType.PARKED_IN_META)) + + ossie = _SALES_MODEL + for cycle in range(3): + ossie, _ = convert_cube_to_ossie(convert_ossie_to_cube(ossie)[0]) + assert parse(ossie)["semantic_model"][0]["name"] == "Sales Model", ( + f"lost on cycle {cycle + 1}") + + +def test_a_name_override_that_sanitizes_to_the_view_name_is_preserved(): + """`--name 'Sales Model'` over a Cube model whose view is already `sales_model`. + + Both sides sanitize to `sales_model`, so comparing the *sanitized* forms saw no + difference and recorded nothing -- the override was silently undone on the way back. + The comparison is against the raw name for exactly this case. + """ + cube = { + "model/cubes/orders.yml": + "cubes:\n- name: orders\n sql_table: shop.public.orders\n" + " dimensions:\n - name: id\n sql: id\n type: number\n" + " primary_key: true\n", + "model/views/sales_model.yml": + "views:\n- name: sales_model\n cubes:\n - join_path: orders\n" + " includes: '*'\n", + } + ossie, _ = convert_cube_to_ossie(cube, model_name="Sales Model") + assert parse(ossie)["semantic_model"][0]["name"] == "Sales Model" + for cycle in range(3): + ossie, _ = convert_cube_to_ossie(convert_ossie_to_cube(ossie)[0]) + assert parse(ossie)["semantic_model"][0]["name"] == "Sales Model", ( + f"lost on cycle {cycle + 1}") + + +def test_a_model_name_already_matching_its_view_records_nothing(): + """The record only appears when the names actually differ, so an ordinary model keeps + a clean Cube document with no `meta.ossie` on its view at all.""" + files, _ = convert_ossie_to_cube(_SALES_MODEL.replace("Sales Model", "sales_model")) + view = parse(files["model/views/sales_model.yml"])["views"][0] + assert "meta" not in view + + def test_a_model_from_another_converter_is_stable_after_one_cycle(): """`Ossie -> Cube -> Ossie` twice, compared byte-for-byte. @@ -313,7 +387,27 @@ def test_a_model_from_another_converter_is_stable_after_one_cycle(): assert second == first -@cube_gate +_SHADOWED_KEY_COLUMN = ( + f"version: {OSSIE_VERSION}\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: shop.public.orders\n" + " primary_key:\n - id\n" + " fields:\n" + " - name: id\n expression:\n dialects:\n" + " - dialect: ANSI_SQL\n expression: LOWER(email)\n" + " datatype: String\n" +) + + +def _two_export_cycles(ossie): + first, _ = convert_ossie_to_cube(ossie) + second, _ = convert_ossie_to_cube(convert_cube_to_ossie(first)[0]) + return first, second + + def test_two_export_cycles_produce_the_same_cube_model(): """`Ossie -> Cube -> Ossie -> Cube`, on the collision that made the first differ from the second: a key column `id` alongside a computed field also named `id`. @@ -322,25 +416,21 @@ def test_two_export_cycles_produce_the_same_cube_model(): that the next export reads another leaves both Ossie documents identical while the Cube model changes -- here the key moved off the column and onto `LOWER(email)`, so Cube deduplicated on a different value and returned different counts. + + Ungated: comparing two exports needs no Cube installation, and putting the whole test + behind the optional gate meant the regression it exists for went unchecked on every + machine without a built Cube checkout -- including CI. """ - ossie = ( - f"version: {OSSIE_VERSION}\n" - "semantic_model:\n" - "- name: shop\n" - " datasets:\n" - " - name: orders\n" - " source: shop.public.orders\n" - " primary_key:\n - id\n" - " fields:\n" - " - name: id\n expression:\n dialects:\n" - " - dialect: ANSI_SQL\n expression: LOWER(email)\n" - " datatype: String\n" - ) - first, _ = convert_ossie_to_cube(ossie) - second, _ = convert_ossie_to_cube(convert_cube_to_ossie(first)[0]) + first, second = _two_export_cycles(_SHADOWED_KEY_COLUMN) assert parse_files(second) == parse_files(first) keys = [d for d in parse_files(first)["model/cubes/orders.yml"]["cubes"][0][ "dimensions"] if d.get("primary_key")] assert [(d["name"], d["sql"]) for d in keys] == [("id_pk", "id")] + + +@cube_gate +def test_the_second_export_cycle_still_compiles(): + """The half of the above that genuinely needs Cube.""" + _, second = _two_export_cycles(_SHADOWED_KEY_COLUMN) assert_cube_compiles(second, "second export cycle") From dab57c27c18eaaa3b75f6e61ec0994eeef454e49 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Wed, 5 Aug 2026 15:41:05 +0500 Subject: [PATCH 46/46] Carry model metadata on a cube when no view can hold it [P1] Model-level metadata has no Cube field of its own, so it rides on the view representing the model -- and export emitted no view at all when the stash recorded none mapped. A Cube model need not contain a view, and one with several need not say which is the model, so this is an ordinary input rather than an edge case. Both cases dropped the name, description and AI context in silence, with no issue reported: a cube-only model imported with `--name 'Sales Model'` came back as the synthesized `cube_model`. That contradicts the documented lossless Ossie -> Cube -> Ossie round trip. They ride on a cube now, under `meta.ossie.model`, and import reads them back when no view is mapped. The carrier is the alphabetically first cube -- deterministic, and independent of both dataset ordering and the relationship graph, so every export picks the same one. Import does not depend on the choice; it reads whichever cube carries the record, which cannot accumulate because the record is consumed and stripped from the stash. Only values import could not otherwise recover are parked, so a Cube model that never had model-level metadata still round-trips byte-identical rather than acquiring a `meta.ossie` key it never had. That matters beyond tidiness: every fixture in the feature matrix is cube-only, and their structural round trips would all have started failing. A name equal to the one import synthesizes is recoverable by definition, hence the shared DEFAULT_MODEL_NAME rather than a second copy of the literal. Foreign-vendor `custom_extensions` deliberately keep refusing export in this case, and the README now says why rather than leaving it looking inconsistent: import restores those only from the mapped view, so a cube carrier would not bring them home. That path fails loudly, which was never the complaint here. Checked the new checks can fail: removing the export half breaks 3 tests, removing the import half breaks 2, and both halves are exercised over three cycles because the value has to survive being read back out of a cube's stash rather than merely written once. The carrier's output is put through the Cube compile gate too, since it is new YAML in the emitted model and holds a literal brace -- Cube compiles every string as an f-string. 576 tests with both gates, 408 with neither, 97% coverage. Cross-converter matrix unchanged. --- converters/cube/README.md | 11 ++- converters/cube/src/ossie_cube/_common.py | 5 + converters/cube/src/ossie_cube/cube_to_osi.py | 30 +++++- converters/cube/src/ossie_cube/osi_to_cube.py | 64 ++++++++++++ converters/cube/tests/test_roundtrip.py | 99 +++++++++++++++++++ 5 files changed, 203 insertions(+), 6 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index a6ef7939..a06ed97c 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -115,9 +115,9 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | Apache Ossie | Cube | Notes | |---|---|---| -| `semantic_model` | a **view** | Cube users are view-first, and Cube's agent reads `meta.ai_context` only from views and members -- so the view, not any cube, is the model boundary. | +| `semantic_model` | a **view** | Cube users are view-first, and Cube's agent reads `meta.ai_context` only from views and members -- so the view, not any cube, is the model boundary. A Cube model need not contain a view, though, and one with several need not say which is the model; when no view can carry the model's metadata it is parked on the alphabetically first cube under `meta.ossie.model` and import reads it back from there. | | `semantic_model.name` | view name | Import: the mapped view's name (override with `--name`). Export: whenever the emitted view cannot carry the name exactly, the original is recorded in `meta.ossie.model_name` and import hands that back instead of the view's. Three causes — the name is not a valid Cube identifier (`Sales Model` → view `sales_model`); a `--name` override the stashed view does not match; or the name is also a dataset's, since Cube keeps cubes and views in one namespace and would refuse a model with two members of one name (what a Databricks metric view over a same-named table produces), so the generated view becomes `_view`. | -| `model.description` / `ai_context.instructions` | view `description` / `meta.ai_context` | Import: taken from the sole view, or `--view`. | +| `model.description` / `ai_context.instructions` | view `description` / `meta.ai_context` | Import: taken from the sole view, or `--view`; or from `meta.ossie.model` on a cube when no view is mapped. | | dataset | `cubes[]` entry in `model/cubes/.yml` | Import: a non-canonical original path is stashed and restored on export. | | `dataset.source` (dotted) | `sql_table` | Passed through verbatim; Cube interpolates it straight into `FROM`, so no catalog/schema split is needed. | | `dataset.source` (`SELECT ...`) | `sql` | Cube requires exactly one of `sql` / `sql_table`. | @@ -440,9 +440,10 @@ invalid) when an input breaks one of these: - a dimension has an unknown `type`, or a `geo` dimension is missing `latitude.sql` / `longitude.sql`; - the model carries foreign-vendor `custom_extensions` but no view is mapped, so - there is nowhere to park them (re-import with `--view `); model-level - metadata rides on the view representing the model, and picking one arbitrarily - would not survive a re-import; + there is nowhere to park them (re-import with `--view `). The model's own + name, description and AI context are carried on a cube in this case; foreign + extensions are not, because import restores those only from the mapped view -- + so this refuses loudly rather than dropping them; - there are no convertible cubes at all; the input YAML is malformed. ## Notes and limitations diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index a43ee3ac..e0a3fd49 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -37,6 +37,11 @@ # Vendor id used for the `custom_extensions` stash. VENDOR = "CUBE" +# The Ossie model name import synthesizes when the Cube model offers none -- no view is +# mapped and `--name` was not given. Shared so export can recognize a name it did *not* +# synthesize and therefore has to preserve; two copies of the literal would drift. +DEFAULT_MODEL_NAME = "cube_model" + # Cube SQL is the SQL of the model's data source, so there is no CUBE entry in # the Ossie dialect enum. Import emits ANSI_SQL; export prefers ANSI_SQL and lets # the caller prepend a warehouse dialect the actual data source would accept. diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 44bd3c34..73fbf7f9 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -39,6 +39,7 @@ AGG_TO_OSSIE_FUNC, AGG_TO_RESULT_DATATYPE, CALCULATED_MEASURE_TYPES, + DEFAULT_MODEL_NAME, DIALECT_ANSI, DATATYPE_TO_DIM_TYPE, DIM_TYPE_TO_DATATYPE, @@ -140,13 +141,23 @@ def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=False # its datasets has its view renamed. Without this the name would silently become # the renamed view's on the way back. parked_model_name = parked_of(mapped_view.get("meta")).get("model_name") - model = {"name": model_name or parked_model_name or mapped_name or "cube_model"} + # With no view mapped there is nothing to take an identity from, so a previous export + # parked the model's metadata on a cube instead. Read before the datasets are built, + # because building them strips `meta.ossie` from the cube. + carried = _model_carried_by_a_cube(cubes) if mapped_name is None else {} + + model = {"name": (model_name or parked_model_name or mapped_name + or carried.get("name") or DEFAULT_MODEL_NAME)} if mapped_view.get("description"): model["description"] = unescape_braces_from_cube( mapped_view["description"]) + elif carried.get("description"): + model["description"] = carried["description"] ai = _ai_context_from_meta(mapped_view.get("meta")) if ai: model["ai_context"] = ai + elif carried.get("ai_context"): + model["ai_context"] = carried["ai_context"] # Anything a cube's stash has to carry is worked out before the dataset is # built: joins with no Ossie form, and measures with no static Ossie @@ -427,6 +438,23 @@ def _ai_context_from_meta(meta): return None +def _model_carried_by_a_cube(cubes): + """Model-level metadata a previous export parked on a cube, or {}. + + Takes the `{name: cube}` mapping and reads the first cube that carries the record, so + the result does not depend on which cube export chose as the carrier. In practice there + is exactly one: the record is consumed here and stripped from the stash, so it cannot + accumulate across cycles. + """ + for cube in cubes.values(): + if not isinstance(cube, dict): + continue + carried = parked_of(cube.get("meta")).get("model") + if isinstance(carried, dict) and carried: + return carried + return {} + + def parked_of(meta): """The `meta.ossie` subtree, with Cube's brace escaping undone. diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index c86066e1..4b778e36 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -39,6 +39,7 @@ AGG_TO_RESULT_DATATYPE, DATATYPE_TO_DIM_TYPE, DEFAULT_DATATYPE_FOR_CUBE_TYPE, + DEFAULT_MODEL_NAME, DIALECT_ANSI, OSSIE_FUNC_TO_AGG, OSSIE_VERSION, @@ -168,11 +169,13 @@ def _convert_model(model, dialect, base_cube, issues): stashed_paths = model_stash.get("cube_files") or {} files_content = {} emitted_members = {} + cubes_by_name = {} for ds_name, ds in datasets.items(): cname = cube_names[ds_name] cube = _build_cube(ds, plan[cname], tables, joins_by_cube.get(cname), measures_by_cube.get(cname), join_parked_by_cube.get(cname), dialect, issues) + cubes_by_name[cname] = cube stashed = stashed_paths.get(cname) path = (safe_relative_path(stashed, f"cube '{cname}'") if stashed else cube_file(cname)) @@ -190,6 +193,9 @@ def _convert_model(model, dialect, base_cube, issues): issues).items(): files_content.setdefault(vpath, {}).setdefault("views", []).extend(views) + if not _a_view_carries_the_model(model_stash): + _carry_model_on_a_cube(model, cubes_by_name, issues) + files = {path: dump_yaml(content) for path, content in files_content.items()} # Files a prior import could not convert (`.js` models, Jinja-templated YAML, @@ -1252,6 +1258,64 @@ def _balanced(s): # --- views ---------------------------------------------------------------------- +def _a_view_carries_the_model(model_stash): + """Whether some view will hold the model's own name, description and AI context. + + Model-level metadata has no Cube field of its own, so it rides on the view that + represents the model. There are two ways that happens: a hand-authored Ossie document + has no stashed view set, so export generates a view; or a stashed set records which + view the model was mapped from. Neither holds for a Cube model that has no views at + all -- which Cube does not require -- or one with several where none was chosen. + """ + if "views" not in model_stash: + return True # export generates one + return model_stash.get("mapped_view") is not None + + +def _carry_model_on_a_cube(model, cubes_by_name, issues): + """Park model-level metadata on a cube when no view can hold it. + + Without this the metadata was dropped in silence: a cube-only Cube model imported with + `--name 'Sales Model'` exported to cubes alone, and re-importing had nothing to read + the name from, so it came back as the synthesized `cube_model`. A description or AI + context added on the Ossie side went the same way. That contradicts the documented + lossless `Ossie -> Cube -> Ossie` round trip, and Cube models without views are + ordinary. + + The carrier is the alphabetically first cube -- deterministic, and independent of both + dataset ordering and the relationship graph, so export picks the same one every time. + Import does not depend on the choice: it reads whichever cube carries the record. + + Only genuinely unrecoverable values are parked, so a Cube model that had no + model-level metadata to begin with still round-trips to a byte-identical document + rather than gaining a `meta.ossie` key it never had. A name equal to the one import + synthesizes is recoverable by definition. + """ + metadata = {} + if model.get("name") and model["name"] != DEFAULT_MODEL_NAME: + metadata["name"] = model["name"] + if model.get("description"): + metadata["description"] = model["description"] + if model.get("ai_context"): + metadata["ai_context"] = model["ai_context"] + if not metadata or not cubes_by_name: + return + + carrier = sorted(cubes_by_name)[0] + cube = cubes_by_name[carrier] + # Escaped here rather than by `_build_meta`, which has already run for this cube: + # Cube compiles every string in a model as a Python f-string, so an unescaped brace + # in a description or in AI instructions fails the whole compile. + cube.setdefault("meta", {}).setdefault("ossie", {})["model"] = ( + escape_braces_for_cube(metadata)) + issues.add( + IssueType.PARKED_IN_META, f"cube '{carrier}'", + f"the model has no view to carry its metadata (" + f"{', '.join(sorted(metadata))}), because the Cube model it came from has no " + f"view mapped to it; parked on this cube under meta.ossie.model so a re-import " + f"can recover it") + + def _uncollided_view_name(vname, cube_names): """A generated view name that no cube already owns. diff --git a/converters/cube/tests/test_roundtrip.py b/converters/cube/tests/test_roundtrip.py index b06cf350..600fb8b2 100644 --- a/converters/cube/tests/test_roundtrip.py +++ b/converters/cube/tests/test_roundtrip.py @@ -26,6 +26,7 @@ import json import pytest +import yaml from _cube_gate import ( assert_cube_compiles, assert_ossie_is_valid, @@ -365,6 +366,104 @@ def test_a_name_override_that_sanitizes_to_the_view_name_is_preserved(): f"lost on cycle {cycle + 1}") +_CUBE_ONLY = { + "model/cubes/orders.yml": + "cubes:\n- name: orders\n sql_table: shop.public.orders\n" + " dimensions:\n - name: id\n sql: id\n type: number\n" + " primary_key: true\n", +} +_TWO_VIEWS = { + **_CUBE_ONLY, + "model/views/a.yml": + "views:\n- name: view_a\n cubes:\n - join_path: orders\n" + " includes: '*'\n", + "model/views/b.yml": + "views:\n- name: view_b\n cubes:\n - join_path: orders\n" + " includes: '*'\n", +} + + +def _with_model_metadata(cube_files): + """Import, then add the model-level metadata a user would edit in on the Ossie side.""" + ossie, _ = convert_cube_to_ossie(cube_files, model_name="Sales Model") + doc = parse(ossie) + model = doc["semantic_model"][0] + model["description"] = "Sales overview with a {brace}" + model["ai_context"] = {"instructions": "Prefer completed orders"} + return json.loads(json.dumps(doc)), model + + +def _dump(doc): + return yaml.safe_dump(doc, sort_keys=False) + + +@pytest.mark.parametrize("label,cube_files", [ + ("no views at all", _CUBE_ONLY), + ("two views, none selected", _TWO_VIEWS), +]) +def test_model_metadata_survives_when_no_view_can_carry_it(label, cube_files): + """Model-level metadata has no Cube field; it rides on the view representing the model. + + A Cube model need not contain a view, and one with several views need not say which is + the model -- and in both cases export emitted no view at all, so the name, description + and AI context were dropped without a word. `--name 'Sales Model'` came back as the + synthesized `cube_model`. They ride on a deterministic cube instead now. + + Three cycles, since the value has to survive being read back out of a cube's stash; + and a literal brace, because Cube compiles every string in a model as an f-string. + """ + doc, _ = _with_model_metadata(cube_files) + ossie = _dump(doc) + for cycle in range(3): + files, issues = convert_ossie_to_cube(ossie) + ossie, _ = convert_cube_to_ossie(files) + model = parse(ossie)["semantic_model"][0] + assert model["name"] == "Sales Model", f"{label}: lost on cycle {cycle + 1}" + assert model["description"] == "Sales overview with a {brace}" + assert model["ai_context"]["instructions"] == "Prefer completed orders" + + # Reported, not silent -- the whole complaint about the old behaviour. + assert any(i.element_name == "cube 'orders'" and "no view to carry" in i.detail + for i in issues.of_type(IssueType.PARKED_IN_META)) + + +def test_the_carrier_is_the_alphabetically_first_cube(): + """Deterministic, and independent of dataset order and of the relationship graph, so + every export picks the same cube. Import does not depend on the choice.""" + cube_files = { + "model/cubes/zeta.yml": + "cubes:\n- name: zeta\n sql_table: s.p.zeta\n dimensions:\n" + " - name: id\n sql: id\n type: number\n primary_key: true\n", + "model/cubes/alpha.yml": + "cubes:\n- name: alpha\n sql_table: s.p.alpha\n dimensions:\n" + " - name: id\n sql: id\n type: number\n primary_key: true\n", + } + doc, _ = _with_model_metadata(cube_files) + files, _ = convert_ossie_to_cube(_dump(doc)) + carried = { + name: parse(text)["cubes"][0].get("meta", {}).get("ossie", {}).get("model") + for name, text in files.items()} + assert carried["model/cubes/alpha.yml"]["name"] == "Sales Model" + assert carried["model/cubes/zeta.yml"] is None + + +def test_a_cube_only_model_without_metadata_gains_nothing(): + """The record appears only when there is something unrecoverable to keep, so a Cube + model that never had model-level metadata still round-trips byte-identical instead of + acquiring a `meta.ossie` key it never had. Every feature fixture is cube-only, so this + is what keeps their structural round trips honest.""" + files, _ = convert_ossie_to_cube(convert_cube_to_ossie(_CUBE_ONLY)[0]) + assert parse_files(files) == parse_files(_CUBE_ONLY) + + +@cube_gate +def test_a_model_carried_on_a_cube_still_compiles(): + """The carrier is new YAML in the emitted model, and it holds a literal brace.""" + doc, _ = _with_model_metadata(_CUBE_ONLY) + files, _ = convert_ossie_to_cube(_dump(doc)) + assert_cube_compiles(files, "model metadata carried on a cube") + + def test_a_model_name_already_matching_its_view_records_nothing(): """The record only appears when the names actually differ, so an ordinary model keeps a clean Cube document with no `meta.ossie` on its view at all."""