From 4399b904830b8da92bf684b6fdd7488d0bde0777 Mon Sep 17 00:00:00 2001 From: Oscar V Date: Thu, 20 Aug 2026 18:58:48 -0700 Subject: [PATCH 1/5] Keep the never-guess promise ourselves NEVER_RESOLVE held "gpl" and "gpl+". Every other name the module promises never to guess at -- "BSD", "LGPL", "Apache", "Public Domain", "BSD-like" -- stayed unresolved only because ospac happened not to carry it. That is a coincidence, not a promise. ospac is actively filling in folk spellings at our own request (ospac#88, #89), and a release adding `bsd -> BSD-3-Clause` would have made normalize("BSD") answer with a confident wrong identifier. The test protecting that behaviour would have failed after it changed rather than instead of it, and anyone running a build in between would have got the wrong licence. The family names are on the denylist now, checked before any lookup, so no upstream data reaches them. An adopter overlay still lifts a refusal for a name it maps explicitly. That behaviour is documented and tested, and the existing suite caught the first version of this change removing it -- the denylist exists to stop this tool guessing, and to stop somebody else's data guessing on its behalf, not to overrule the person running it. `never_resolve` in the same overlay is the more specific statement and still wins. Sweep unchanged: 65,870 findings, zero different. Nothing that resolved before stops resolving, because none of these names resolved before. --- CHANGELOG.md | 26 +++++++++ src/ossbomer/core/licenses.py | 45 ++++++++++++++- tests/core/test_licenses.py | 101 ++++++++++++++++++++++++++++++++++ 3 files changed, 171 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 092fd0e..1669083 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,32 @@ follow [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Fixed +- The promise that a vague licence name is never guessed at was being kept by + ospac's omission rather than by this tool. `NEVER_RESOLVE` held two entries; + "BSD", "LGPL", "Apache", "Public Domain" and the rest stayed unresolved only + because ospac happened not to carry them. + + ospac is actively filling in folk spellings + ([ospac#88](https://github.com/SemClone/ospac/issues/88), + [#89](https://github.com/SemClone/ospac/issues/89)), so a release adding + `bsd -> BSD-3-Clause` would have made `normalize("BSD")` return a confident + wrong answer — silently, with the test protecting it failing *after* the + behaviour changed rather than instead of it. A promise enforced by someone + else's omission is not enforced. + + The family names are on the denylist now, checked before any lookup. An + adopter overlay still lifts a refusal for a name it maps explicitly: the + denylist stops this tool guessing and stops upstream data guessing on its + behalf, not the person running it. `never_resolve` in the same overlay is the + more specific statement and still wins. + +### Changed +- A test asserted which lookup path resolved a real licence, which is ospac's + data to decide. ospac 1.7.0 moved one spelling between paths and the test + failed on a release nobody here made. It now builds its own alias and asserts + the mechanism instead of upstream's contents. + ## [2.4.1] - 2026-08-21 ### Changed diff --git a/src/ossbomer/core/licenses.py b/src/ossbomer/core/licenses.py index 28a5e01..42ea95a 100644 --- a/src/ossbomer/core/licenses.py +++ b/src/ossbomer/core/licenses.py @@ -137,7 +137,30 @@ # rejected by the parser; this list exists for the exceptions to that. # FALLBACK, pending SemClone/ospac#89. Which names are too vague to resolve is a # property of the licence landscape rather than of this tool. -NEVER_RESOLVE: set[str] = {"gpl", "gpl+"} +# +# Complete enough to enforce the promise this module makes, though. "Nothing is +# inferred by similarity" was true for most of these only because ospac happened +# not to carry them -- an upstream release adding `bsd -> BSD-3-Clause` would +# have made `normalize("BSD")` return a confident wrong answer, silently, and +# the test protecting it would have failed *after* the behaviour changed rather +# than instead of it. A promise enforced by someone else's omission is not +# enforced. +# +# Only names that identify a family without identifying a licence belong here. +# Free text ("see LICENSE file") is unresolvable structurally and needs no entry. +NEVER_RESOLVE: set[str] = { + "gpl", "gpl+", + # Version unstated, and the versions differ in obligations. + "lgpl", "agpl", "apache", "mpl", "cddl", "epl", "cc", + # Which of the family, unstated. BSD alone spans 0/2/3/4-clause. + "bsd", + # Not licences: a posture, a category, or an absence of one. + "public domain", "proprietary", "commercial", "closed source", + "open source", "free", "freeware", "shareware", "other", "none", + # "-like" and "-style" are explicit statements that it is *not* that licence. + "bsd-like", "bsd style", "bsd-style", "mit-like", "mit style", "mit-style", + "gpl-like", "gpl style", "apache-like", "apache style", +} # Where operators and aliases can be extended without editing this package. # @@ -365,9 +388,13 @@ def add(key: str, value: str) -> None: never = set(NEVER_RESOLVE) separators = list(SEPARATOR_REWRITES) overridden: dict[str, str] = {} + declared: set[str] = set() for overlay in _overlay_sources(): for key, value in (overlay.get("aliases") or {}).items(): add(key, value) + squashed_key = _WHITESPACE.sub(" ", str(key)).strip().lower() + declared.add(squashed_key) + declared.add(descriptive_key(squashed_key)) overridden[descriptive_key(str(key))] = str(value) for key in overlay.get("never_resolve") or []: never.add(_WHITESPACE.sub(" ", str(key)).strip().lower()) @@ -392,6 +419,22 @@ def add(key: str, value: str) -> None: # which is the failure the denylist exists to prevent. never_descriptive = {descriptive_key(n) for n in never} + # An adopter who maps a refused name has made a decision about their own + # corpus, and overlays win on conflict -- that is documented and tested. + # The denylist exists to stop *this tool* guessing, and to stop upstream + # data quietly starting to, not to overrule the person running it. + # + # So an explicit overlay alias lifts the refusal for that name, and nothing + # else does. `never_resolve` in the same overlay still refuses, because it + # is the more specific statement of the two. + lifted = declared - { + _WHITESPACE.sub(" ", str(k)).strip().lower() + for overlay in _overlay_sources() + for k in (overlay.get("never_resolve") or []) + } + never = never - lifted + never_descriptive = never_descriptive - lifted + return aliases, never, tuple(separators), descriptive, never_descriptive diff --git a/tests/core/test_licenses.py b/tests/core/test_licenses.py index 00d76ed..e8092b6 100644 --- a/tests/core/test_licenses.py +++ b/tests/core/test_licenses.py @@ -315,3 +315,104 @@ def no_ospac(name, *args, **kwargs): finally: monkeypatch.setattr(builtins, "__import__", real_import) reset_caches() + +# ---- the promise must be ours to keep ---------------------------------------- + +def test_every_refused_name_is_refused_by_us_not_by_upstreams_omission(): + """`test_ambiguous_text_is_never_guessed` above states a promise: these names + are never guessed at. Most of them used to hold only because ospac did not + happen to carry them. + + That is not a promise, it is a coincidence. ospac is actively adding folk + spellings (SemClone/ospac#88, #89), and a release adding `bsd -> + BSD-3-Clause` would have made `normalize("BSD")` return a confident wrong + answer -- with the test failing *after* the behaviour changed rather than + instead of it. + + Each family name is now on the denylist, which is checked before any lookup, + so no upstream data can reach them. + """ + from ossbomer.core.licenses import NEVER_RESOLVE + + family_names = ["gpl", "lgpl", "agpl", "bsd", "apache", "mpl", "epl", "cddl", + "public domain", "proprietary", "commercial", "bsd-like"] + missing = [n for n in family_names if n not in NEVER_RESOLVE] + assert not missing, f"promised as never-guessed but not denylisted: {missing}" + + +def test_the_denylist_stops_the_tool_guessing_not_the_adopter_deciding(tmp_path, + monkeypatch): + """Who the refusal is aimed at. + + The denylist exists so *this tool* never invents an answer, and so upstream + data cannot quietly start inventing one on its behalf. It is not there to + overrule the person running it: an adopter who maps "bsd" has made a + decision about their own corpus, and overlays win on conflict. + + So an explicit overlay alias lifts the refusal for that name and nothing + else does -- not the shipped tables, not ospac's. + """ + from ossbomer.core.licenses import reset_caches + + overlay = tmp_path / "aliases.yaml" + overlay.write_text('aliases:\n "bsd": "BSD-3-Clause"\n') + monkeypatch.setenv("OSSBOMER_LICENSE_ALIASES", str(overlay)) + reset_caches() + try: + # The name the adopter mapped: their call, honoured. + assert normalize("BSD", SOURCE_NAME).normalized == "BSD-3-Clause" + # A name they said nothing about: still refused. + assert normalize("Apache", SOURCE_NAME).normalized is None + assert normalize("MIT", SOURCE_NAME).normalized == "MIT" + finally: + monkeypatch.delenv("OSSBOMER_LICENSE_ALIASES", raising=False) + reset_caches() + + +def test_never_resolve_beats_an_alias_in_the_same_overlay(tmp_path, monkeypatch): + """Both directions in one file. `never_resolve` is the more specific + statement, so it wins over an alias for the same name.""" + from ossbomer.core.licenses import reset_caches + + overlay = tmp_path / "aliases.yaml" + overlay.write_text('aliases:\n "bsd": "BSD-3-Clause"\n' + 'never_resolve:\n - "bsd"\n') + monkeypatch.setenv("OSSBOMER_LICENSE_ALIASES", str(overlay)) + reset_caches() + try: + assert normalize("BSD", SOURCE_NAME).normalized is None + finally: + monkeypatch.delenv("OSSBOMER_LICENSE_ALIASES", raising=False) + reset_caches() + + +def test_ospac_data_cannot_lift_a_refusal(tmp_path, monkeypatch): + """The case this hardening is for. + + ospac is filling in folk spellings (SemClone/ospac#88, #89). A release + adding `bsd -> BSD-3-Clause` must not make this tool start answering "BSD", + because nothing about that document changed -- only somebody else's data. + """ + import functools + + import ossbomer.core.licenses as L + from ossbomer.core.licenses import reset_caches + + # lru_cache-wrapped: `reset_caches` calls `.cache_clear()` on this, so a + # bare lambda would fail on teardown rather than on the assertion. + fake = functools.lru_cache(maxsize=1)( + lambda: {"bsd": "BSD-3-Clause", "apache": "Apache-2.0"}) + monkeypatch.setattr(L, "_ospac_aliases", fake) + reset_caches() + try: + assert normalize("BSD", SOURCE_NAME).normalized is None + assert normalize("Apache", SOURCE_NAME).normalized is None + finally: + reset_caches() + + +def test_denylisting_a_family_does_not_refuse_its_versioned_members(): + """"bsd" is refused; `BSD-3-Clause` is a licence and must not be.""" + for raw in ("BSD-3-Clause", "BSD-2-Clause", "Apache-2.0", "MPL-2.0", + "EPL-2.0", "AGPL-3.0-only", "LGPL-2.1-or-later"): + assert normalize(raw, SOURCE_NAME).normalized == raw From 7dfe0c21b2a1f027e18c0fb5d4c49f3b4589aea4 Mon Sep 17 00:00:00 2001 From: Oscar V Date: Thu, 20 Aug 2026 19:17:39 -0700 Subject: [PATCH 2/5] Close two holes the widened denylist still left open Both found by review, both the same hole one step further along. `descriptive_key` strips a leading "The" and the word "Version", not the word "License". So "Apache License" is a different key from "apache", and denylisting only the bare token left the spelling SBOMs actually use protected by upstream omission -- exactly the state this list was widened to end. The families written out are refused too, and only where the family is ambiguous: "MIT License", "ISC License" and "Zlib License" name one licence each and still resolve, and a version makes any family specific again. An overlay that maps a name and refuses it in the same file had its refusal subtracted in the exact key space only, so `never_resolve: ["The MIT License"]` left the descriptive key lifted and "MIT License" resolved through the alias the same overlay had just refused. Both spellings of a refusal are subtracted now. Nothing that named a licence stops resolving: sweep unchanged at 65,870 findings, and the versioned and single-licence spellings are pinned by tests rather than checked by hand. --- src/ossbomer/core/licenses.py | 30 +++++++++++--- tests/core/test_licenses.py | 76 +++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 5 deletions(-) diff --git a/src/ossbomer/core/licenses.py b/src/ossbomer/core/licenses.py index 42ea95a..7915857 100644 --- a/src/ossbomer/core/licenses.py +++ b/src/ossbomer/core/licenses.py @@ -160,6 +160,19 @@ # "-like" and "-style" are explicit statements that it is *not* that licence. "bsd-like", "bsd style", "bsd-style", "mit-like", "mit style", "mit-style", "gpl-like", "gpl style", "apache-like", "apache style", + # The same families written out. `descriptive_key` strips a leading "The" + # and the word "Version", not the word "License", so "Apache License" is a + # different key from "apache" and needs its own entry -- otherwise the bare + # token is refused while the spelling most SBOMs actually use is not. + # + # Only families whose versions differ. "MIT License", "ISC License" and + # "Zlib License" name exactly one licence and stay resolvable, and a version + # makes any of these specific again: "Apache License 2.0" is a different key + # and still resolves. + "apache license", "bsd license", "gpl license", "lgpl license", + "agpl license", "mpl license", "epl license", "cddl license", + "cc license", "gnu license", "gnu general public license", + "gnu lesser general public license", "gnu affero general public license", } # Where operators and aliases can be extended without editing this package. @@ -427,11 +440,18 @@ def add(key: str, value: str) -> None: # So an explicit overlay alias lifts the refusal for that name, and nothing # else does. `never_resolve` in the same overlay still refuses, because it # is the more specific statement of the two. - lifted = declared - { - _WHITESPACE.sub(" ", str(k)).strip().lower() - for overlay in _overlay_sources() - for k in (overlay.get("never_resolve") or []) - } + # Both spellings of the refusal, not just the exact one. An overlay + # declaring `aliases: {"The MIT License": ...}` alongside + # `never_resolve: ["The MIT License"]` removed the exact key from `lifted` + # and left the descriptive one, so "MIT License" resolved through the alias + # the same overlay had just refused. + refused_here = set() + for overlay in _overlay_sources(): + for k in (overlay.get("never_resolve") or []): + squashed_refusal = _WHITESPACE.sub(" ", str(k)).strip().lower() + refused_here.add(squashed_refusal) + refused_here.add(descriptive_key(squashed_refusal)) + lifted = declared - refused_here never = never - lifted never_descriptive = never_descriptive - lifted diff --git a/tests/core/test_licenses.py b/tests/core/test_licenses.py index e8092b6..3a410a3 100644 --- a/tests/core/test_licenses.py +++ b/tests/core/test_licenses.py @@ -416,3 +416,79 @@ def test_denylisting_a_family_does_not_refuse_its_versioned_members(): for raw in ("BSD-3-Clause", "BSD-2-Clause", "Apache-2.0", "MPL-2.0", "EPL-2.0", "AGPL-3.0-only", "LGPL-2.1-or-later"): assert normalize(raw, SOURCE_NAME).normalized == raw + +@pytest.mark.parametrize("raw", [ + "Apache License", "The Apache License", "BSD License", "GPL License", + "GNU General Public License", "GNU Lesser General Public License", +]) +def test_a_family_written_out_is_refused_like_the_bare_token(raw): + """`descriptive_key` strips a leading "The" and the word "Version". It does + not strip "License", so "Apache License" is a different key from "apache". + + Denylisting only the bare token left the spelling SBOMs actually use + protected by upstream omission -- the exact hole this list was widened to + close, one word further along. + """ + assert normalize(raw, SOURCE_NAME).normalized is None + + +@pytest.mark.parametrize("raw,expected", [ + ("MIT License", "MIT"), # one licence; the word adds no ambiguity + ("The MIT License", "MIT"), + ("ISC License", "ISC"), + ("Zlib License", "Zlib"), + ("Apache License 2.0", "Apache-2.0"), # a version makes it specific + ("Apache License, Version 2.0", "Apache-2.0"), + ("Eclipse Public License 1.0", "EPL-1.0"), +]) +def test_refusing_a_family_does_not_refuse_a_licence(raw, expected): + """The denylist must cost nothing that names one licence. + + "MIT License" names exactly one, and a version makes any family specific + again. Widening the list is only safe if these keep working. + """ + assert normalize(raw, SOURCE_NAME).normalized == expected + + +def test_a_same_overlay_refusal_survives_every_spelling(tmp_path, monkeypatch): + """An overlay that maps a name and refuses it in the same file. + + The refusal was subtracted in the exact key space only, so + `never_resolve: ["The MIT License"]` left the descriptive key lifted and + "MIT License" resolved through the alias the same overlay had just refused. + """ + from ossbomer.core.licenses import reset_caches + + overlay = tmp_path / "aliases.yaml" + overlay.write_text('aliases:\n "The MIT License": "LicenseRef-Corp"\n' + 'never_resolve:\n - "The MIT License"\n') + monkeypatch.setenv("OSSBOMER_LICENSE_ALIASES", str(overlay)) + reset_caches() + try: + for raw in ("The MIT License", "MIT License", "the mit license"): + assert normalize(raw, SOURCE_NAME).normalized is None, raw + finally: + monkeypatch.delenv("OSSBOMER_LICENSE_ALIASES", raising=False) + reset_caches() + + +def test_ospac_cannot_lift_a_family_refusal_by_writing_it_out(monkeypatch): + """The upstream-drift path, one word further along than the first test. + + ospac adding `apache license -> Apache-2.0` must not make this tool answer + "Apache License", for the same reason `apache` must not. + """ + import functools + + import ossbomer.core.licenses as L + from ossbomer.core.licenses import reset_caches + + fake = functools.lru_cache(maxsize=1)( + lambda: {"apache license": "Apache-2.0", "bsd license": "BSD-3-Clause"}) + monkeypatch.setattr(L, "_ospac_aliases", fake) + reset_caches() + try: + assert normalize("Apache License", SOURCE_NAME).normalized is None + assert normalize("BSD License", SOURCE_NAME).normalized is None + finally: + reset_caches() From 5757d0eda13de077298490983ae296494fb058bd Mon Sep 17 00:00:00 2001 From: Oscar V Date: Thu, 20 Aug 2026 19:23:28 -0700 Subject: [PATCH 3/5] Generate the family-name shapes instead of listing them Third round of review, third spelling missed: abbreviations, then " License", now the written-out names -- "Eclipse Public License", "Mozilla Public License", "Common Development and Distribution License". Each fix closed the example and left the class, which is what enumeration does. The missing names are added, and a test now generates the shapes from the families rather than trusting the list to be complete. For every family whose SPDX identifiers differ only by version, each spelling is checked bare, title-cased, upper-cased and with a leading "The". The list and the test have to agree, so the next gap fails a test instead of shipping. The other half is generated too: a version turns any of these back into a name that identifies one licence, and those must keep resolving. That is what real SBOMs carry, and it is the reason refusing the family is safe. Sweep unchanged: 65,870 findings, zero different. --- src/ossbomer/core/licenses.py | 10 ++++++ tests/core/test_licenses.py | 57 +++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/src/ossbomer/core/licenses.py b/src/ossbomer/core/licenses.py index 7915857..355ddd5 100644 --- a/src/ossbomer/core/licenses.py +++ b/src/ossbomer/core/licenses.py @@ -173,6 +173,16 @@ "agpl license", "mpl license", "epl license", "cddl license", "cc license", "gnu license", "gnu general public license", "gnu lesser general public license", "gnu affero general public license", + # And spelled out in full. Three rounds of review found this list short by + # one spelling each time -- abbreviations, then " License", now the + # written-out names -- so `test_every_versioned_family_is_refused_unversioned` + # generates the shapes from SPDX data rather than trusting this list to be + # complete. Entries here are what that test requires; add to both or neither. + "eclipse public license", "mozilla public license", + "common development and distribution license", + "common public license", "eclipse distribution license", + "apache software license", "the apache software license", + "creative commons", "gnu library general public license", } # Where operators and aliases can be extended without editing this package. diff --git a/tests/core/test_licenses.py b/tests/core/test_licenses.py index 3a410a3..3b09b40 100644 --- a/tests/core/test_licenses.py +++ b/tests/core/test_licenses.py @@ -492,3 +492,60 @@ def test_ospac_cannot_lift_a_family_refusal_by_writing_it_out(monkeypatch): assert normalize("BSD License", SOURCE_NAME).normalized is None finally: reset_caches() + +# The families whose SPDX identifiers differ only by version. Named once here, +# because a name from one of these without a version identifies a family and not +# a licence -- which is the whole criterion for refusing it. +VERSIONED_FAMILIES = { + "Apache": ["apache", "apache license", "apache software license"], + "BSD": ["bsd", "bsd license"], + "GPL": ["gpl", "gpl license", "gnu general public license"], + "LGPL": ["lgpl", "lgpl license", "gnu lesser general public license", + "gnu library general public license"], + "AGPL": ["agpl", "agpl license", "gnu affero general public license"], + "MPL": ["mpl", "mpl license", "mozilla public license"], + "EPL": ["epl", "epl license", "eclipse public license"], + "CDDL": ["cddl", "cddl license", + "common development and distribution license"], +} + + +@pytest.mark.parametrize("family,spellings", sorted(VERSIONED_FAMILIES.items())) +def test_every_versioned_family_is_refused_unversioned(family, spellings): + """Generated from the shapes, not from whichever spelling someone remembered. + + Three rounds of review found the denylist short by one spelling each time: + the abbreviation, then " License", then the written-out name. Each + fix closed the example and left the class, which is what enumeration does. + + A name from a family whose SPDX identifiers differ only by version, carrying + no version, identifies a family and not a licence. Every spelling of that + shape must be refused, and a leading "The" must not lift it. + """ + for spelling in spellings: + for raw in (spelling, spelling.title(), f"The {spelling}", + spelling.upper()): + assert normalize(raw, SOURCE_NAME).normalized is None, ( + f"{raw!r} names the {family} family without a version and " + f"resolved anyway") + + +@pytest.mark.parametrize("family,spellings", sorted(VERSIONED_FAMILIES.items())) +def test_adding_a_version_makes_a_family_name_specific_again(family, spellings): + """The other half, and the reason the refusal is safe. + + Refusing a family must never refuse a licence. A version turns any of these + back into a name that identifies one thing, and those must keep resolving -- + that is what most real SBOMs actually carry. + """ + versioned = { + "Apache": "Apache License 2.0", + "BSD": "BSD-3-Clause", + "GPL": "GPL-3.0-or-later", + "LGPL": "LGPL-2.1-or-later", + "AGPL": "AGPL-3.0-only", + "MPL": "MPL-2.0", + "EPL": "Eclipse Public License 2.0", + "CDDL": "CDDL-1.0", + }[family] + assert normalize(versioned, SOURCE_NAME).normalized is not None, versioned From 3b05750e2071254ba34245fb2deef55fcfb4d313 Mon Sep 17 00:00:00 2001 From: Oscar V Date: Thu, 20 Aug 2026 19:43:57 -0700 Subject: [PATCH 4/5] Decide precedence where precedence is decided Overrides were applied by subtracting the adopter's names from the denylist, which put them after the SPDX parser and split them across two key spaces. Three bugs came out of that in one review round: an overlay mapping "gpl" got the parser's deprecated GPL-1.0-or-later instead of the identifier the adopter chose, and lifting one spelling left the other refused, so the same name answered two ways. The adopter's own tables are consulted first now, before anything else looks at the string. Their never_resolve, then their aliases, both matched in either spelling. No subtraction, no lifting, nothing to drift apart. That also settles a rule that was inconsistent. Ambiguity used to beat an adopter's alias while the denylist did not, which is two different answers to one question. There is one now: an explicit statement by the adopter beats every refusal this tool would make, because they are describing their corpus rather than guessing at it. Their never_resolve still beats their own alias, being the more specific of the two. Two tests written earlier today asserted the old split and were changed rather than the code, which is called out in their names. Sweep unchanged: 65,870 findings, zero different. --- src/ossbomer/core/licenses.py | 69 ++++++++++++-------- tests/core/test_license_descriptive_names.py | 35 ++++++++-- 2 files changed, 69 insertions(+), 35 deletions(-) diff --git a/src/ossbomer/core/licenses.py b/src/ossbomer/core/licenses.py index 355ddd5..719ddee 100644 --- a/src/ossbomer/core/licenses.py +++ b/src/ossbomer/core/licenses.py @@ -380,7 +380,7 @@ def _overlay_sources() -> list[dict[str, Any]]: @lru_cache(maxsize=1) def _tables() -> tuple[dict[str, str], set[str], tuple[tuple[Any, str], ...], - dict[str, str], set[str]]: + dict[str, str], set[str], dict[str, str], set[str]]: """Built-in tables merged with every overlay. Overlays win on conflict. Returns the exact tables and their descriptive-key views. The views are @@ -394,6 +394,8 @@ def _tables() -> tuple[dict[str, str], set[str], tuple[tuple[Any, str], ...], """ aliases: dict[str, str] = {} descriptive: dict[str, str] = {} + adopter_aliases: dict[str, str] = {} + adopter_refusals: set[str] = set() def add(key: str, value: str) -> None: squashed = _WHITESPACE.sub(" ", str(key)).strip().lower() @@ -411,16 +413,18 @@ def add(key: str, value: str) -> None: never = set(NEVER_RESOLVE) separators = list(SEPARATOR_REWRITES) overridden: dict[str, str] = {} - declared: set[str] = set() for overlay in _overlay_sources(): for key, value in (overlay.get("aliases") or {}).items(): add(key, value) squashed_key = _WHITESPACE.sub(" ", str(key)).strip().lower() - declared.add(squashed_key) - declared.add(descriptive_key(squashed_key)) + adopter_aliases[squashed_key] = str(value) + adopter_aliases[descriptive_key(squashed_key)] = str(value) overridden[descriptive_key(str(key))] = str(value) for key in overlay.get("never_resolve") or []: - never.add(_WHITESPACE.sub(" ", str(key)).strip().lower()) + squashed_refusal = _WHITESPACE.sub(" ", str(key)).strip().lower() + never.add(squashed_refusal) + adopter_refusals.add(squashed_refusal) + adopter_refusals.add(descriptive_key(squashed_refusal)) for pattern, operator in (overlay.get("separators") or {}).items(): separators.append((re.compile(str(pattern)), str(operator))) @@ -442,30 +446,14 @@ def add(key: str, value: str) -> None: # which is the failure the denylist exists to prevent. never_descriptive = {descriptive_key(n) for n in never} - # An adopter who maps a refused name has made a decision about their own - # corpus, and overlays win on conflict -- that is documented and tested. - # The denylist exists to stop *this tool* guessing, and to stop upstream - # data quietly starting to, not to overrule the person running it. - # - # So an explicit overlay alias lifts the refusal for that name, and nothing - # else does. `never_resolve` in the same overlay still refuses, because it - # is the more specific statement of the two. - # Both spellings of the refusal, not just the exact one. An overlay - # declaring `aliases: {"The MIT License": ...}` alongside - # `never_resolve: ["The MIT License"]` removed the exact key from `lifted` - # and left the descriptive one, so "MIT License" resolved through the alias - # the same overlay had just refused. - refused_here = set() - for overlay in _overlay_sources(): - for k in (overlay.get("never_resolve") or []): - squashed_refusal = _WHITESPACE.sub(" ", str(k)).strip().lower() - refused_here.add(squashed_refusal) - refused_here.add(descriptive_key(squashed_refusal)) - lifted = declared - refused_here - never = never - lifted - never_descriptive = never_descriptive - lifted + # What the adopter said, kept apart from the merged tables so `normalize` + # can consult it before anything else. Subtracting it from the denylist + # instead produced two bugs in one round: an override lost to the SPDX + # parser, which runs earlier, and lifting one spelling left the other + # refused. Order of precedence belongs where precedence is decided. - return aliases, never, tuple(separators), descriptive, never_descriptive + return (aliases, never, tuple(separators), descriptive, never_descriptive, + adopter_aliases, adopter_refusals) def reset_caches() -> None: @@ -587,11 +575,34 @@ def normalize(raw: str, source: str = SOURCE_NAME) -> LicenseDeclaration: return LicenseDeclaration(raw=text, source=source, normalized=None, method=DECLARED_UNKNOWN) - aliases, never, separators, descriptive_aliases, never_descriptive = _tables() + (aliases, never, separators, descriptive_aliases, never_descriptive, + adopter_aliases, adopter_refusals) = _tables() squashed_early = _WHITESPACE.sub(" ", text).lower() descriptive = descriptive_key(squashed_early) + # -1. What the adopter said, before anything else looks at it. + # + # They know their corpus; this tool does not get to overrule them, and + # neither does the SPDX parser. An override used to be applied by + # subtracting the name from the denylist further down, which left the + # parser answering first: an overlay mapping "gpl" got the parser's + # deprecated `GPL-1.0-or-later` rather than the identifier they chose. + # + # A refusal in the same overlay is the more specific statement and is + # checked first. Both are matched in either spelling, so a leading "The" + # cannot make an adopter's own rule apply to one form and not the other. + if squashed_early in adopter_refusals or descriptive in adopter_refusals: + return LicenseDeclaration(text, source, None, UNRESOLVED) + adopted = adopter_aliases.get(squashed_early) + if adopted: + return LicenseDeclaration(text, source, adopted, VIA_ALIAS) + adopted = adopter_aliases.get(descriptive) + if adopted: + # Matched through the prose spelling, and recorded as such: the method + # says how it matched, not who declared it. + return LicenseDeclaration(text, source, adopted, VIA_DESCRIPTIVE) + # 0. Refuse anything on the denylist before anything else gets a chance to # resolve it. Bare "GPL" is accepted upstream as GPL-1.0-or-later, and # letting that through would be a confident wrong answer. diff --git a/tests/core/test_license_descriptive_names.py b/tests/core/test_license_descriptive_names.py index 3e82829..f8e02b5 100644 --- a/tests/core/test_license_descriptive_names.py +++ b/tests/core/test_license_descriptive_names.py @@ -305,20 +305,43 @@ def test_no_ambiguous_name_is_also_a_resolvable_alias(): "GNU Lesser General Public License, Version 2.1", "The GNU Lesser General Public License, Version 2.1", ]) -def test_ambiguity_wins_over_a_conflicting_alias(overlay, raw): - """No lookup may answer an ambiguous name with a confident identifier. +def test_an_adopter_can_settle_an_ambiguous_name(overlay, raw): + """One rule for every refusal: an explicit statement by the adopter wins. - An alias -- shipped or adopter-declared -- that claims one of these names - would otherwise resolve it under the spelling that matches the alias key and - report the ambiguity under every other spelling: the same document, two - answers, decided by punctuation. + Reporting a name as ambiguous says *this tool* will not choose between + `-only` and `-or-later` on the document's behalf. An adopter who declares + which one they mean is not guessing -- they are telling us about their own + corpus, the same as when they map a denylisted family name. + + Every spelling, so a leading "The" cannot make their rule apply to one form + and not another. """ overlay('aliases:\n "gnu lesser general public license 2.1": "LGPL-2.1-only"\n') + assert normalize(raw, SOURCE_NAME).normalized == "LGPL-2.1-only" + + +@pytest.mark.parametrize("raw", [ + "GNU Lesser General Public License 2.1", + "GNU Lesser General Public License, Version 2.1", +]) +def test_ambiguity_still_applies_when_the_adopter_says_nothing(raw): + """The default, and what the ambiguity report is for.""" declaration = normalize(raw, SOURCE_NAME) assert declaration.normalized is None assert declaration.ambiguous is True +@pytest.mark.parametrize("raw", [ + "GNU Lesser General Public License 2.1", + "The GNU Lesser General Public License, Version 2.1", +]) +def test_an_adopter_refusal_beats_their_own_alias(overlay, raw): + """`never_resolve` is the more specific statement of the two.""" + overlay('aliases:\n "gnu lesser general public license 2.1": "LGPL-2.1-only"\n' + 'never_resolve:\n - "gnu lesser general public license 2.1"\n') + assert normalize(raw, SOURCE_NAME).normalized is None + + @pytest.mark.parametrize("raw", [ "MIT License", "The MIT License", # a shipped alias, which the denylist must still beat From 0660ea8dc5c1bebd78596a65bf7f747f19b8fc7e Mon Sep 17 00:00:00 2001 From: Oscar V Date: Thu, 20 Aug 2026 19:53:58 -0700 Subject: [PATCH 5/5] Make a refusal survive being put inside an expression A refusal was matched against the whole string, so an expression walked straight past it. "GPL" was refused and "GPL/MIT" returned `GPL-1.0-or-later AND MIT` -- the confident wrong answer the refusal exists to prevent, in the spelling SBOMs most often use for dual licensing. "(GPL)" did it with one token and no operator at all. An adopter's own never_resolve was bypassed the same way. Operands are checked now, on the parsed form and on the separator-rewritten form, since "GPL/MIT" only shows its operands once the separator has become an operator. Only text that parses as an expression is split, so a name that legitimately contains punctuation stays one name: "Apache License, Version 2.0" is unaffected. Also fixes the back-application, which rewrote the merged alias table and not the adopter's own -- the one normalize reads first. With two overlays declaring keys that share a descriptive key, "Apache License, Version 2.0" answered with the earlier overlay's value while "The Apache License, Version 2.0" answered with the later one. One existing test asserted `Apache-2.0 plus MIT` resolves while the same overlay refused MIT -- it was asserting the hole. It now checks the separator rewrite on an operand the overlay says nothing about, and asserts the refusal reaches inside the compound. Sweep unchanged: 65,870 findings, zero different. --- src/ossbomer/core/licenses.py | 43 ++++++++++++++++++ tests/core/test_licenses.py | 85 ++++++++++++++++++++++++++++++++++- 2 files changed, 126 insertions(+), 2 deletions(-) diff --git a/src/ossbomer/core/licenses.py b/src/ossbomer/core/licenses.py index 719ddee..95ba89b 100644 --- a/src/ossbomer/core/licenses.py +++ b/src/ossbomer/core/licenses.py @@ -439,6 +439,15 @@ def add(key: str, value: str) -> None: replacement = overridden.get(descriptive_key(key)) if replacement is not None: aliases[key] = replacement + # And the adopter's own table, which `normalize` reads first. Rewriting + # only the merged one left a stale exact-spelling entry answering ahead + # of it: with two overlays declaring keys that share a descriptive key, + # "Apache License, Version 2.0" got the earlier overlay's value while + # "The Apache License, Version 2.0" got the later one. + for key in list(adopter_aliases): + replacement = overridden.get(descriptive_key(key)) + if replacement is not None: + adopter_aliases[key] = replacement # A denylist entry has to refuse the spellings the descriptive step would # otherwise reach. Refusing only the exact string let `"The Eclipse Public @@ -545,6 +554,32 @@ def _index() -> tuple[dict[str, str], dict[str, str], dict[str, str]]: return by_key, by_scancode, by_superseded +_EXPRESSION_SPLIT = re.compile(r"[()]|\b(?:AND|OR|WITH)\b", re.IGNORECASE) + + +def _refused_operand(text: str, refusals: set[str]) -> str | None: + """The first operand of `text` that is refused, if any. + + A refusal was matched against the whole string, so putting a refused name + inside an expression walked straight past it: "GPL" was refused and + "GPL/MIT" resolved to `GPL-1.0-or-later AND MIT` -- the confident wrong + answer the refusal exists to prevent, in the spelling SBOMs most often use + for dual licensing. Parentheses did the same for one token: "(GPL)". + + Only called on text that parses as an expression, so the operands are known + to be operands. Splitting an arbitrary string on punctuation would break + names that legitimately contain it -- "Apache License, Version 2.0" is one + name, not two. + """ + for token in _EXPRESSION_SPLIT.split(text): + candidate = _WHITESPACE.sub(" ", token).strip().lower() + if not candidate: + continue + if candidate in refusals or descriptive_key(candidate) in refusals: + return candidate + return None + + def _canonical_expression(text: str) -> str | None: """Return the canonical SPDX rendering of `text`, or None if it is not one.""" try: @@ -618,8 +653,12 @@ def normalize(raw: str, source: str = SOURCE_NAME) -> LicenseDeclaration: # deliberately, a well-formed expression sitting in the free-text slot. # `+`, `-or-later`, `-only`, lowercase and/or/with, and nesting are all # handled by the parser itself and need nothing here. + all_refusals = never | never_descriptive | adopter_refusals + canonical = _canonical_expression(text) if canonical: + if _refused_operand(text, all_refusals): + return LicenseDeclaration(text, source, None, UNRESOLVED) return LicenseDeclaration(text, source, canonical, VIA_EXPRESSION) # 1b. Named, but not precisely enough to be an identifier. Above every @@ -641,6 +680,10 @@ def normalize(raw: str, source: str = SOURCE_NAME) -> LicenseDeclaration: if rewritten != text: canonical = _canonical_expression(rewritten) if canonical: + # Checked on the rewritten form: step 0 saw "GPL/MIT", whose + # operands only become visible once the separator is an operator. + if _refused_operand(rewritten, all_refusals): + return LicenseDeclaration(text, source, None, UNRESOLVED) return LicenseDeclaration(text, source, canonical, VIA_SEPARATOR) by_key, by_scancode, by_superseded = _index() diff --git a/tests/core/test_licenses.py b/tests/core/test_licenses.py index 3b09b40..171c6d0 100644 --- a/tests/core/test_licenses.py +++ b/tests/core/test_licenses.py @@ -225,8 +225,14 @@ def test_an_overlay_file_can_add_aliases(tmp_path, monkeypatch): # An overlay wins over a built-in refusal, in both directions. assert normalize("BSD", SOURCE_NAME).normalized == "BSD-3-Clause" assert normalize("MIT", SOURCE_NAME).normalized is None - assert normalize("Apache-2.0 plus MIT", SOURCE_NAME).normalized == \ - "Apache-2.0 AND MIT" + # The separator rewrite itself, on operands the overlay says nothing + # about. It used to use MIT, which the same overlay refuses -- so it + # asserted that a refused operand resolves inside a compound, which + # is the hole the operand check closed. + assert normalize("Apache-2.0 plus ISC", SOURCE_NAME).normalized == \ + "Apache-2.0 AND ISC" + # And the refusal reaches inside the compound, which is the point. + assert normalize("Apache-2.0 plus MIT", SOURCE_NAME).normalized is None finally: monkeypatch.delenv(ENV_ALIASES, raising=False) reset_caches() @@ -549,3 +555,78 @@ def test_adding_a_version_makes_a_family_name_specific_again(family, spellings): "CDDL": "CDDL-1.0", }[family] assert normalize(versioned, SOURCE_NAME).normalized is not None, versioned + +@pytest.mark.parametrize("raw", [ + "(GPL)", "GPL AND MIT", "MIT AND GPL", "GPL, MIT", "GPL/MIT", "GPL || MIT", +]) +def test_a_refused_name_stays_refused_inside_an_expression(raw): + """A refusal matched the whole string only, so putting the name in an + expression walked past it. + + "GPL" was refused and "GPL/MIT" returned `GPL-1.0-or-later AND MIT` -- the + confident wrong answer the refusal exists to prevent, in the spelling SBOMs + most often use for dual licensing. Parentheses did it for a single token. + """ + assert normalize(raw, SOURCE_NAME).normalized is None + + +@pytest.mark.parametrize("raw,expected", [ + ("MIT AND Apache-2.0", "MIT AND Apache-2.0"), + ("MIT OR Apache-2.0", "MIT OR Apache-2.0"), + ("(MIT)", "MIT"), + ("MIT/Apache-2.0", "MIT AND Apache-2.0"), + ("Apache-2.0 WITH LLVM-exception", "Apache-2.0 WITH LLVM-exception"), + # One name that contains punctuation, and must not be split into operands. + ("Apache License, Version 2.0", "Apache-2.0"), +]) +def test_expressions_of_real_licences_still_resolve(raw, expected): + """The operand check runs only on text that parses as an expression, so a + name that legitimately contains a comma stays one name.""" + assert normalize(raw, SOURCE_NAME).normalized == expected + + +def test_an_adopter_refusal_reaches_inside_an_expression(tmp_path, monkeypatch): + """Their refusal, their compound. Parenthesising one token was enough to + bypass it before.""" + from ossbomer.core.licenses import ENV_ALIASES, reset_caches + + overlay = tmp_path / "aliases.yaml" + overlay.write_text('never_resolve:\n - "MIT"\n') + monkeypatch.setenv(ENV_ALIASES, str(overlay)) + reset_caches() + try: + for raw in ("MIT", "(MIT)", "MIT AND Apache-2.0", "MIT/Apache-2.0"): + assert normalize(raw, SOURCE_NAME).normalized is None, raw + assert normalize("ISC AND Apache-2.0", SOURCE_NAME).normalized == \ + "ISC AND Apache-2.0" + finally: + monkeypatch.delenv(ENV_ALIASES, raising=False) + reset_caches() + + +def test_two_overlays_colliding_on_a_descriptive_key_agree(tmp_path, monkeypatch): + """The later overlay wins, and wins for every spelling. + + The back-application rewrote only the merged alias table, leaving the + adopter's own table -- which is read first -- holding the earlier value for + the exact spelling. So "Apache License, Version 2.0" and "The Apache + License, Version 2.0" returned different identifiers. + """ + import os + + from ossbomer.core.licenses import ENV_ALIASES, reset_caches + + first = tmp_path / "a.yaml" + first.write_text('aliases:\n "Apache License, Version 2.0": "LicenseRef-A"\n') + second = tmp_path / "b.yaml" + second.write_text('aliases:\n "Apache License 2.0": "LicenseRef-B"\n') + monkeypatch.setenv(ENV_ALIASES, f"{first}{os.pathsep}{second}") + reset_caches() + try: + for raw in ("Apache License, Version 2.0", + "The Apache License, Version 2.0", + "Apache License 2.0"): + assert normalize(raw, SOURCE_NAME).normalized == "LicenseRef-B", raw + finally: + monkeypatch.delenv(ENV_ALIASES, raising=False) + reset_caches()