From ea6bbc7ad06584a3ffd063ad49dff9b7c190030c Mon Sep 17 00:00:00 2001 From: Craig Johnson Date: Fri, 10 Jul 2026 01:32:45 +0300 Subject: [PATCH 1/2] feat: optional netbox-dlm integration + provision table preflight (2.5.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Preflight: Django registers ContentTypes for every installed app even when its migrations never ran, so netbox_branching's provision runs CREATE TABLE branch.T (LIKE public.T) for a missing table and the sync dies mid-provision with an opaque ProgrammingError (field report: netbox_dlm_contract). missing_branch_table_report() diffs get_tables_to_replicate() against the database's real tables; the single-branch executor now fails in seconds with the app labels and remedy, and a new Database tables Health check surfaces the gap before anyone syncs. netbox-dlm integration: three opt-in NQE maps (disabled by default) sync Forward's end-of-life analysis into the optional netbox-dlm plugin — SoftwareVersion from device.platform.osSupport per (platform, osVersion), HardwareNotice from chassis part support per device type, and DeviceSoftware one row per device (the adapter ensures the referenced SoftwareVersion without overwriting announced dates). Wired end to end through the optional-plugin pattern: registry integration + adapter contract, choices (+migration 0032), query registry, decision/budget/contract/identity tables, runner dispatch, and delete ordering (children before the PROTECTed SoftwareVersion). Dates are parsed to date objects so unchanged rows do not churn. Also fixes a latent import-order bug the new models exposed: branch_budget's _fallback_bucket_key_family was defined after the module-level shard-contract build and NameError'd for any supported model without a structured contract. Live against ADP: 35 software versions, 48 hardware notices, 4960 device- software rows; queries lint clean. Full suite (1012) + lint + harness green; adversarial review findings (delete ranks, date churn, reporting filter) fixed. --- CHANGELOG.md | 4 + README.md | 7 +- docs/01_User_Guide/README.md | 9 +- ...-10-dlm-support-and-provision-preflight.md | 85 +++++++ docs/README.md | 3 +- forward_netbox/__init__.py | 2 +- forward_netbox/choices.py | 8 + .../0032_alter_forwardnqemap_netbox_model.py | 90 ++++++++ .../queries/forward_dlm_device_software.nqe | 25 ++ .../queries/forward_dlm_hardware_notices.nqe | 28 +++ .../queries/forward_dlm_software_versions.nqe | 28 +++ forward_netbox/tests/test_dlm_integration.py | 106 +++++++++ .../tests/test_plugin_integrations.py | 3 + .../tests/test_provision_preflight.py | 74 ++++++ .../utilities/apply_engine_decision.py | 24 ++ forward_netbox/utilities/branch_budget.py | 32 ++- forward_netbox/utilities/branching.py | 30 +++ forward_netbox/utilities/health.py | 33 +++ .../utilities/plugin_integrations/registry.py | 35 +++ .../utilities/query_fetch_execution.py | 1 + forward_netbox/utilities/query_registry.py | 18 ++ .../utilities/single_branch_executor.py | 14 ++ forward_netbox/utilities/sync.py | 12 + forward_netbox/utilities/sync_contracts.py | 15 ++ forward_netbox/utilities/sync_dlm.py | 217 ++++++++++++++++++ forward_netbox/utilities/sync_primitives.py | 5 + .../utilities/sync_runner_adapters.py | 24 ++ pyproject.toml | 2 +- 28 files changed, 915 insertions(+), 19 deletions(-) create mode 100644 docs/03_Plans/active/2026-07-10-dlm-support-and-provision-preflight.md create mode 100644 forward_netbox/migrations/0032_alter_forwardnqemap_netbox_model.py create mode 100644 forward_netbox/queries/forward_dlm_device_software.nqe create mode 100644 forward_netbox/queries/forward_dlm_hardware_notices.nqe create mode 100644 forward_netbox/queries/forward_dlm_software_versions.nqe create mode 100644 forward_netbox/tests/test_dlm_integration.py create mode 100644 forward_netbox/tests/test_provision_preflight.py create mode 100644 forward_netbox/utilities/sync_dlm.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e5cda75..a3b2fdf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ Generated from the README compatibility table by `scripts/gen_changelog.py`. Do not edit by hand. +## v2.5.0 + +Feature: optional **netbox-dlm** (Device Lifecycle Management) integration — three new opt-in NQE maps sync Forward's end-of-life analysis into the netbox-dlm plugin: OS software versions with vendor EOL dates per (platform, version), hardware end-of-life notices per device type (Cisco/Palo Alto/Fortinet part support), and each device's running software version. Disabled by default; requires the netbox-dlm plugin (run `makemigrations netbox_dlm && migrate` after installing it — it ships no migrations). Fix: syncs no longer crash mid-provision when an installed plugin's migrations were never applied (`relation ... does not exist`) — a preflight now fails in seconds with the app name and remedy, and a new **Database tables** Health check surfaces the gap before you sync. + ## v2.4.5 Fix: sync no longer crashes on netbox-branching **1.1.1** (`SquashMergeStrategy has no attribute '_split_bidirectional_cycles'` — 1.1.1 removed that internal helper; the bidirectional-cycle split is now built into the plugin and the dependency is bounded to `<1.2`). Also fixes SNMP-endpoint rows failing validation: the bundled endpoint query branches now clamp sysDescr-derived `device_type` to NetBox's 100-char limit (`substring`) and guard empty slugs — the fix lives in the NQE queries (the source of truth), so **Publish Bundled Queries** again after upgrading (fixes the `Ensure this value has at most 100 characters` rejects and the `At least one coalesce lookup must be provided` error). diff --git a/README.md b/README.md index f5abe314..d4607cf9 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,8 @@ On top of the import it adds scope control (sync only devices carrying chosen Fo - **It is not a source of truth for Forward configuration or intent** — it populates NetBox from Forward's collected snapshot, nothing more. - **It does not require** the optional `netbox-routing` / `netbox-peering-manager` - plugins unless you enable the beta BGP/OSPF maps. + / `netbox-dlm` plugins unless you enable the beta BGP/OSPF or device-lifecycle + maps. ## Screenshots @@ -66,7 +67,8 @@ Latest release requires NetBox `4.6.4` and `netbox-branching` `1.1.0+`. Expand f | Plugin Release | NetBox Version | Status | | --- | --- | --- | -| `v2.4.5` | `4.6.4` required; needs netbox-branching `1.1.0` – `1.1.x` | Current release; Fix: sync no longer crashes on netbox-branching **1.1.1** (`SquashMergeStrategy has no attribute '_split_bidirectional_cycles'` — 1.1.1 removed that internal helper; the bidirectional-cycle split is now built into the plugin and the dependency is bounded to `<1.2`). Also fixes SNMP-endpoint rows failing validation: the bundled endpoint query branches now clamp sysDescr-derived `device_type` to NetBox's 100-char limit (`substring`) and guard empty slugs — the fix lives in the NQE queries (the source of truth), so **Publish Bundled Queries** again after upgrading (fixes the `Ensure this value has at most 100 characters` rejects and the `At least one coalesce lookup must be provided` error). | +| `v2.5.0` | `4.6.4` required; needs netbox-branching `1.1.0` – `1.1.x` | Current release; Feature: optional **netbox-dlm** (Device Lifecycle Management) integration — three new opt-in NQE maps sync Forward's end-of-life analysis into the netbox-dlm plugin: OS software versions with vendor EOL dates per (platform, version), hardware end-of-life notices per device type (Cisco/Palo Alto/Fortinet part support), and each device's running software version. Disabled by default; requires the netbox-dlm plugin (run `makemigrations netbox_dlm && migrate` after installing it — it ships no migrations). Fix: syncs no longer crash mid-provision when an installed plugin's migrations were never applied (`relation ... does not exist`) — a preflight now fails in seconds with the app name and remedy, and a new **Database tables** Health check surfaces the gap before you sync. | +| `v2.4.5` | `4.6.4` required; needs netbox-branching `1.1.0` – `1.1.x` | Superseded by `v2.5.0`; Fix: sync no longer crashes on netbox-branching **1.1.1** (`SquashMergeStrategy has no attribute '_split_bidirectional_cycles'` — 1.1.1 removed that internal helper; the bidirectional-cycle split is now built into the plugin and the dependency is bounded to `<1.2`). Also fixes SNMP-endpoint rows failing validation: the bundled endpoint query branches now clamp sysDescr-derived `device_type` to NetBox's 100-char limit (`substring`) and guard empty slugs — the fix lives in the NQE queries (the source of truth), so **Publish Bundled Queries** again after upgrading (fixes the `Ensure this value has at most 100 characters` rejects and the `At least one coalesce lookup must be provided` error). | | `v2.4.4` | `4.6.4` required; needs netbox-branching `1.1.0+` | Superseded by `v2.4.5`; Fix: SNMP-endpoint import now works on **tag-scoped** syncs — the device-tag include scope silently excluded every endpoint, both query-side and in the plugin's local scope filter (whose scoped-device set was built from modeled devices only, so endpoint rows were always dropped; with prune enabled they would even be deleted). Endpoint import now ignores the include scope (exclude tags still apply) and endpoint names join the scoped set (validated live: 355 Avocent endpoints import under a tag-scoped sync). Also fixes the merge-phase `Tag with this Name already exists` issues: a same-named/same-slug tag already on main is now treated as merged instead of failing the branch's tag create. | | `v2.4.3` | `4.6.4` required; needs netbox-branching `1.1.0+` | Superseded by `v2.4.4`; Fix: the pinned-query opt-in Health warning (2.4.1) over-claimed failure — it reads "nothing new syncs" and fires on any pinned map with the feature on, even after the query is fixed, because the Health page can't read a pinned query's contents. Reworded to a "Pinned — can't verify locally" heads-up that points at **Export Live Query Drift** to confirm (`source_matches_bundled`), instead of asserting failure. No behavior change. | | `v2.4.2` | `4.6.4` required; needs netbox-branching `1.1.0+` | Superseded by `v2.4.3`; Fix: endpoint import (`sync_endpoints`) and device-tag sync (`sync_device_tags`) now work with the alias-aware and rules-aware query variants (`forward_devices_with_netbox_aliases`, `forward_device_feature_tags_with_rules`), not just the base queries — operators running the variants saw the toggles silently do nothing (validated live: 355 Avocent endpoints import; `Mgmt_*` tags sync). Adds a **Publish Bundled Queries** button on the sync Health page (beside Refresh Query IDs) and two Health warnings: when an opt-in feature is enabled but no enabled map provides it, and when a base query and its opt-in variant are both enabled (they double-apply rows for the same model and churn — enable one). The alias-aware device query now emits the clean role name (e.g. `ROUTER`) to match the base query — expect a one-time role update on alias-mapped devices. | @@ -201,6 +203,7 @@ use. - Optional disabled NQE maps for NetBox Device Type Library alias matching through a Forward JSON data file - Optional disabled NQE map for data-file-driven device feature tag rules - Feature-flagged beta BGP and OSPF maps for optional `netbox-routing` and `netbox-peering-manager` deployments +- Beta device-lifecycle maps for optional `netbox-dlm` deployments: OS/hardware end-of-life dates and per-device running software from Forward's support analysis - Snapshot-aware execution with `latestProcessed` or an explicit Forward snapshot per sync - Ingestion records that preserve the selected snapshot mode, resolved snapshot ID, and Forward snapshot metrics - Built-in coverage for: diff --git a/docs/01_User_Guide/README.md b/docs/01_User_Guide/README.md index 53295870..42e89fc5 100644 --- a/docs/01_User_Guide/README.md +++ b/docs/01_User_Guide/README.md @@ -20,14 +20,14 @@ pip install forward-netbox Alternatively, install a specific wheel or source archive from GitHub Releases: ```bash -pip install /path/to/forward_netbox-2.4.5-py3-none-any.whl -pip install /path/to/forward_netbox-2.4.5.tar.gz +pip install /path/to/forward_netbox-2.5.0-py3-none-any.whl +pip install /path/to/forward_netbox-2.5.0.tar.gz ``` If you mirror the package into a private Python index, pin the same release version: ```bash -pip install forward-netbox==2.4.5 +pip install forward-netbox==2.5.0 ``` ## Release Compatibility @@ -39,7 +39,8 @@ Latest release requires NetBox `4.6.4` and `netbox-branching` `1.1.0+`. Expand f | Plugin Release | NetBox Version | Status | | --- | --- | --- | -| `v2.4.5` | `4.6.4` required (4.5.x dropped); needs netbox-branching `1.1.0` – `1.1.x` | Current release; Fix: sync no longer crashes on netbox-branching **1.1.1** (`SquashMergeStrategy has no attribute '_split_bidirectional_cycles'` — 1.1.1 removed that internal helper; the bidirectional-cycle split is now built into the plugin and the dependency is bounded to `<1.2`). Also fixes SNMP-endpoint rows failing validation: the bundled endpoint query branches now clamp sysDescr-derived `device_type` to NetBox's 100-char limit (`substring`) and guard empty slugs — the fix lives in the NQE queries (the source of truth), so **Publish Bundled Queries** again after upgrading (fixes the `Ensure this value has at most 100 characters` rejects and the `At least one coalesce lookup must be provided` error). | +| `v2.5.0` | `4.6.4` required (4.5.x dropped); needs netbox-branching `1.1.0` – `1.1.x` | Current release; Feature: optional **netbox-dlm** (Device Lifecycle Management) integration — three new opt-in NQE maps sync Forward's end-of-life analysis into the netbox-dlm plugin: OS software versions with vendor EOL dates per (platform, version), hardware end-of-life notices per device type (Cisco/Palo Alto/Fortinet part support), and each device's running software version. Disabled by default; requires the netbox-dlm plugin (run `makemigrations netbox_dlm && migrate` after installing it — it ships no migrations). Fix: syncs no longer crash mid-provision when an installed plugin's migrations were never applied (`relation ... does not exist`) — a preflight now fails in seconds with the app name and remedy, and a new **Database tables** Health check surfaces the gap before you sync. | +| `v2.4.5` | `4.6.4` required (4.5.x dropped); needs netbox-branching `1.1.0` – `1.1.x` | Superseded by `v2.5.0`; Fix: sync no longer crashes on netbox-branching **1.1.1** (`SquashMergeStrategy has no attribute '_split_bidirectional_cycles'` — 1.1.1 removed that internal helper; the bidirectional-cycle split is now built into the plugin and the dependency is bounded to `<1.2`). Also fixes SNMP-endpoint rows failing validation: the bundled endpoint query branches now clamp sysDescr-derived `device_type` to NetBox's 100-char limit (`substring`) and guard empty slugs — the fix lives in the NQE queries (the source of truth), so **Publish Bundled Queries** again after upgrading (fixes the `Ensure this value has at most 100 characters` rejects and the `At least one coalesce lookup must be provided` error). | | `v2.4.4` | `4.6.4` required (4.5.x dropped); needs netbox-branching `1.1.0+` | Superseded by `v2.4.5`; Fix: SNMP-endpoint import now works on **tag-scoped** syncs — the device-tag include scope silently excluded every endpoint, both query-side and in the plugin's local scope filter (whose scoped-device set was built from modeled devices only, so endpoint rows were always dropped; with prune enabled they would even be deleted). Endpoint import now ignores the include scope (exclude tags still apply) and endpoint names join the scoped set. Also fixes the merge-phase `Tag with this Name already exists` issues: a same-named/same-slug tag already on main is now treated as merged instead of failing the branch's tag create. | | `v2.4.3` | `4.6.4` required (4.5.x dropped); needs netbox-branching `1.1.0+` | Superseded by `v2.4.4`; Fix: the pinned-query opt-in Health warning (2.4.1) over-claimed failure and fired even after the query was fixed (the Health page can't read a pinned query's contents). Reworded to a "Pinned — can't verify locally" heads-up pointing at Export Live Query Drift to confirm. No behavior change. | | `v2.4.2` | `4.6.4` required (4.5.x dropped); needs netbox-branching `1.1.0+` | Superseded by `v2.4.3`; Fix: endpoint import (`sync_endpoints`) and device-tag sync (`sync_device_tags`) now work with the alias-aware and rules-aware query variants (`forward_devices_with_netbox_aliases`, `forward_device_feature_tags_with_rules`), not just the base queries — operators running the variants saw the toggles silently do nothing. Adds a **Publish Bundled Queries** button on the sync Health page and two Health warnings: when an opt-in feature is enabled but no enabled map provides it, and when a base query and its opt-in variant are both enabled (duplicate-emission churn). The alias-aware device query now emits the clean role name (e.g. `ROUTER`) to match the base query — expect a one-time role update on alias-mapped devices. | diff --git a/docs/03_Plans/active/2026-07-10-dlm-support-and-provision-preflight.md b/docs/03_Plans/active/2026-07-10-dlm-support-and-provision-preflight.md new file mode 100644 index 00000000..43f3d928 --- /dev/null +++ b/docs/03_Plans/active/2026-07-10-dlm-support-and-provision-preflight.md @@ -0,0 +1,85 @@ +# Feature: optional netbox-dlm integration + provision table preflight (2.5.0) + +## Goal + +Stop syncs from crashing when an installed plugin's migrations were never +applied (field report: `relation "public.netbox_dlm_contract" does not exist` +mid-provision), and add an optional integration for the netbox-dlm Device +Lifecycle Management plugin backed by Forward's end-of-life analysis. + +## Constraints + +- netbox-dlm is NOT installed in CI or the dev runtime; every surface must + degrade gracefully when the plugin is absent (existing optional-plugin + pattern) and the new maps ship disabled. +- New model strings require a ForwardNQEMap choices migration (0032) — the + only schema change; no data migrations. +- NQE stays the source of truth: field-length clamping and identity shaping + live in the bundled queries, not the plugin. +- Never commit the ADP network id / token. + +## Touched Surfaces + +- `forward_netbox/utilities/branching.py` (`missing_branch_table_report`), + `single_branch_executor.py` (preflight), `health.py` (Database tables check) +- `forward_netbox/utilities/sync_dlm.py` (new adapter), + `sync_runner_adapters.py` + `sync.py` (dispatch wiring) +- `forward_netbox/choices.py` (+ migration 0032), `query_registry.py`, + `plugin_integrations/registry.py`, `apply_engine_decision.py`, + `branch_budget.py`, `sync_contracts.py`, `sync_primitives.py`, + `query_fetch_execution.py`, `sync_reporting.py` +- New queries: `forward_dlm_software_versions.nqe`, + `forward_dlm_hardware_notices.nqe`, `forward_dlm_device_software.nqe` +- Tests: `test_provision_preflight.py`, `test_dlm_integration.py` + +## Approach + +1. Preflight: Django registers ContentTypes for every installed app even when + its migrations never ran, so netbox_branching's provision runs + `CREATE TABLE branch.T (LIKE public.T)` for a missing table and the sync + dies with an opaque ProgrammingError. `missing_branch_table_report()` diffs + `get_tables_to_replicate()` against `connection.introspection.table_names()` + (the exact list provision replicates), maps missing tables to app labels, + and the executor raises an actionable SyncError before the fetch. A + fail-severity **Database tables** Health check surfaces the same report. +2. netbox-dlm integration (three opt-in maps, following the + netbox_routing/ACI optional-plugin pattern end to end): SoftwareVersion + from `device.platform.osSupport` per (platform, osVersion); HardwareNotice + from chassis `device.platform.components[].support` per device type; + DeviceSoftware one row per device (the adapter ensures the referenced + SoftwareVersion exists without overwriting announced dates). Registry + integration (`DLM_INTEGRATION`, adapter contract on + `forward_netbox.utilities.sync_dlm`), decision/budget/contract/identity + tables, and runner dispatch are wired for all three model strings. +3. Fixed a latent import-order bug the new models exposed: + `_fallback_bucket_key_family` was defined after the module-level shard + contract build in `branch_budget.py` and NameError'd for any supported + model without a structured shard contract. + +## Validation + +Live against the ADP demo network: 35 software-version rows (ISO dates, +announcement URLs), 48 hardware-notice rows, 4960 device-software rows; all +three bundled queries lint clean and run via raw NQE. Unit tests cover the +preflight (report, executor SyncError, Health check pass/fail/soft-fail) and +the DLM wiring (choices/registry/maps disabled-by-default/adapter contract/ +runner dispatch/query structure). Full suite + lint + harness green. + +## Rollback + +Revert the branch and roll back migration 0032 (choices-only alter). No data +migrations; the new maps are seeded disabled and can be deleted. + +## Decision Log + +- Ship the DLM maps disabled: the plugin is alpha (0.1.0), ships no + migrations, and the integration is a beta surface — operators opt in per map. +- DeviceSoftware ensures a bare SoftwareVersion when the vendor has no EOL + announcement (create-only, never overwriting dates from the versions map), + so every device links even when Forward lacks announcement data. +- Forward exposes no end-of-sale date, so HardwareNotice.end_of_sale stays + null; end_of_support/lastSupportDate, end_of_security_patches/ + lastVulnerabilityDate, end_of_sw_releases/lastMaintenanceDate. +- The preflight lives in the executor (not provision monkeypatching) and the + Health check never raises — diagnostics failures return None rather than + breaking the page. diff --git a/docs/README.md b/docs/README.md index f9d1c4d8..e4017a61 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,7 +13,8 @@ Latest release requires NetBox `4.6.4` and `netbox-branching` `1.1.0+`. Expand f | Plugin Release | NetBox Version | Status | | --- | --- | --- | -| `v2.4.5` | `4.6.4` required; needs netbox-branching `1.1.0` – `1.1.x` | Current release; Fix: sync no longer crashes on netbox-branching **1.1.1** (`SquashMergeStrategy has no attribute '_split_bidirectional_cycles'` — 1.1.1 removed that internal helper; the bidirectional-cycle split is now built into the plugin and the dependency is bounded to `<1.2`). Also fixes SNMP-endpoint rows failing validation: the bundled endpoint query branches now clamp sysDescr-derived `device_type` to NetBox's 100-char limit (`substring`) and guard empty slugs — the fix lives in the NQE queries (the source of truth), so **Publish Bundled Queries** again after upgrading (fixes the `Ensure this value has at most 100 characters` rejects and the `At least one coalesce lookup must be provided` error). | +| `v2.5.0` | `4.6.4` required; needs netbox-branching `1.1.0` – `1.1.x` | Current release; Feature: optional **netbox-dlm** (Device Lifecycle Management) integration — three new opt-in NQE maps sync Forward's end-of-life analysis into the netbox-dlm plugin: OS software versions with vendor EOL dates per (platform, version), hardware end-of-life notices per device type (Cisco/Palo Alto/Fortinet part support), and each device's running software version. Disabled by default; requires the netbox-dlm plugin (run `makemigrations netbox_dlm && migrate` after installing it — it ships no migrations). Fix: syncs no longer crash mid-provision when an installed plugin's migrations were never applied (`relation ... does not exist`) — a preflight now fails in seconds with the app name and remedy, and a new **Database tables** Health check surfaces the gap before you sync. | +| `v2.4.5` | `4.6.4` required; needs netbox-branching `1.1.0` – `1.1.x` | Superseded by `v2.5.0`; Fix: sync no longer crashes on netbox-branching **1.1.1** (`SquashMergeStrategy has no attribute '_split_bidirectional_cycles'` — 1.1.1 removed that internal helper; the bidirectional-cycle split is now built into the plugin and the dependency is bounded to `<1.2`). Also fixes SNMP-endpoint rows failing validation: the bundled endpoint query branches now clamp sysDescr-derived `device_type` to NetBox's 100-char limit (`substring`) and guard empty slugs — the fix lives in the NQE queries (the source of truth), so **Publish Bundled Queries** again after upgrading (fixes the `Ensure this value has at most 100 characters` rejects and the `At least one coalesce lookup must be provided` error). | | `v2.4.4` | `4.6.4` required; needs netbox-branching `1.1.0+` | Superseded by `v2.4.5`; Fix: SNMP-endpoint import now works on **tag-scoped** syncs — the device-tag include scope silently excluded every endpoint, both query-side and in the plugin's local scope filter (whose scoped-device set was built from modeled devices only, so endpoint rows were always dropped; with prune enabled they would even be deleted). Endpoint import now ignores the include scope (exclude tags still apply) and endpoint names join the scoped set. Also fixes the merge-phase `Tag with this Name already exists` issues: a same-named/same-slug tag already on main is now treated as merged instead of failing the branch's tag create. | | `v2.4.3` | `4.6.4` required; needs netbox-branching `1.1.0+` | Superseded by `v2.4.4`; Fix: the pinned-query opt-in Health warning (2.4.1) over-claimed failure and fired even after the query was fixed (the Health page can't read a pinned query's contents). Reworded to a "Pinned — can't verify locally" heads-up pointing at Export Live Query Drift to confirm. No behavior change. | | `v2.4.2` | `4.6.4` required; needs netbox-branching `1.1.0+` | Superseded by `v2.4.3`; Fix: endpoint import (`sync_endpoints`) and device-tag sync (`sync_device_tags`) now work with the alias-aware and rules-aware query variants (`forward_devices_with_netbox_aliases`, `forward_device_feature_tags_with_rules`), not just the base queries — operators running the variants saw the toggles silently do nothing. Adds a **Publish Bundled Queries** button on the sync Health page and two Health warnings: when an opt-in feature is enabled but no enabled map provides it, and when a base query and its opt-in variant are both enabled (duplicate-emission churn). The alias-aware device query now emits the clean role name (e.g. `ROUTER`) to match the base query — expect a one-time role update on alias-mapped devices. | diff --git a/forward_netbox/__init__.py b/forward_netbox/__init__.py index fd2479dc..07ce1c32 100644 --- a/forward_netbox/__init__.py +++ b/forward_netbox/__init__.py @@ -8,7 +8,7 @@ class NetboxForwardConfig(PluginConfig): name = "forward_netbox" verbose_name = "Forward" description = "Sync Forward data into NetBox using built-in NQE queries." - version = "2.4.5" + version = "2.5.0" base_url = "forward" min_version = "4.6.4" diff --git a/forward_netbox/choices.py b/forward_netbox/choices.py index 9988efcb..7e6f1ef7 100644 --- a/forward_netbox/choices.py +++ b/forward_netbox/choices.py @@ -13,6 +13,12 @@ "netbox_peering_manager.peeringsession", ) +FORWARD_DLM_MODELS = ( + "netbox_dlm.softwareversion", + "netbox_dlm.hardwarenotice", + "netbox_dlm.devicesoftware", +) + FORWARD_ACI_MODELS = ( "netbox_cisco_aci.acifabric", "netbox_cisco_aci.acipod", @@ -49,6 +55,7 @@ "dcim.module", *FORWARD_BGP_MODELS, *FORWARD_ACI_MODELS, + *FORWARD_DLM_MODELS, ) FORWARD_OPTIONAL_MODELS = { @@ -56,6 +63,7 @@ "dcim.module", *FORWARD_BGP_MODELS, *FORWARD_ACI_MODELS, + *FORWARD_DLM_MODELS, } diff --git a/forward_netbox/migrations/0032_alter_forwardnqemap_netbox_model.py b/forward_netbox/migrations/0032_alter_forwardnqemap_netbox_model.py new file mode 100644 index 00000000..d49e4b86 --- /dev/null +++ b/forward_netbox/migrations/0032_alter_forwardnqemap_netbox_model.py @@ -0,0 +1,90 @@ +# Generated by Django 6.0.6 on 2026-07-09 21:57 +import django.db.models.deletion +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("contenttypes", "0002_remove_content_type_name"), + ("forward_netbox", "0031_encrypt_source_password"), + ] + + operations = [ + migrations.AlterField( + model_name="forwardnqemap", + name="netbox_model", + field=models.ForeignKey( + limit_choices_to=models.Q( + models.Q(("app_label", "dcim"), ("model", "site")), + models.Q(("app_label", "dcim"), ("model", "manufacturer")), + models.Q(("app_label", "dcim"), ("model", "devicerole")), + models.Q(("app_label", "dcim"), ("model", "platform")), + models.Q(("app_label", "dcim"), ("model", "devicetype")), + models.Q(("app_label", "dcim"), ("model", "device")), + models.Q(("app_label", "dcim"), ("model", "virtualchassis")), + models.Q(("app_label", "extras"), ("model", "taggeditem")), + models.Q(("app_label", "dcim"), ("model", "interface")), + models.Q(("app_label", "dcim"), ("model", "cable")), + models.Q(("app_label", "dcim"), ("model", "macaddress")), + models.Q(("app_label", "ipam"), ("model", "vlan")), + models.Q(("app_label", "ipam"), ("model", "vrf")), + models.Q(("app_label", "ipam"), ("model", "prefix")), + models.Q(("app_label", "ipam"), ("model", "ipaddress")), + models.Q(("app_label", "ipam"), ("model", "fhrpgroup")), + models.Q(("app_label", "dcim"), ("model", "inventoryitem")), + models.Q(("app_label", "dcim"), ("model", "module")), + models.Q(("app_label", "netbox_routing"), ("model", "bgppeer")), + models.Q( + ("app_label", "netbox_routing"), ("model", "bgpaddressfamily") + ), + models.Q( + ("app_label", "netbox_routing"), + ("model", "bgppeeraddressfamily"), + ), + models.Q( + ("app_label", "netbox_routing"), ("model", "ospfinstance") + ), + models.Q(("app_label", "netbox_routing"), ("model", "ospfarea")), + models.Q( + ("app_label", "netbox_routing"), ("model", "ospfinterface") + ), + models.Q( + ("app_label", "netbox_peering_manager"), + ("model", "peeringsession"), + ), + models.Q(("app_label", "netbox_cisco_aci"), ("model", "acifabric")), + models.Q(("app_label", "netbox_cisco_aci"), ("model", "acipod")), + models.Q(("app_label", "netbox_cisco_aci"), ("model", "acinode")), + models.Q(("app_label", "netbox_cisco_aci"), ("model", "acitenant")), + models.Q(("app_label", "netbox_cisco_aci"), ("model", "acivrf")), + models.Q( + ("app_label", "netbox_cisco_aci"), ("model", "acibridgedomain") + ), + models.Q( + ("app_label", "netbox_cisco_aci"), ("model", "aciappprofile") + ), + models.Q( + ("app_label", "netbox_cisco_aci"), ("model", "aciendpointgroup") + ), + models.Q( + ("app_label", "netbox_cisco_aci"), ("model", "acicontract") + ), + models.Q(("app_label", "netbox_cisco_aci"), ("model", "acifilter")), + models.Q(("app_label", "netbox_cisco_aci"), ("model", "acil3out")), + models.Q( + ("app_label", "netbox_cisco_aci"), + ("model", "acistaticportbinding"), + ), + models.Q(("app_label", "netbox_dlm"), ("model", "softwareversion")), + models.Q(("app_label", "netbox_dlm"), ("model", "hardwarenotice")), + models.Q(("app_label", "netbox_dlm"), ("model", "devicesoftware")), + _connector="OR", + ), + on_delete=django.db.models.deletion.PROTECT, + related_name="+", + to="contenttypes.contenttype", + ), + ), + ] diff --git a/forward_netbox/queries/forward_dlm_device_software.nqe b/forward_netbox/queries/forward_dlm_device_software.nqe new file mode 100644 index 00000000..f8395703 --- /dev/null +++ b/forward_netbox/queries/forward_dlm_device_software.nqe @@ -0,0 +1,25 @@ +/** + * @intent Forward per-device running software version (netbox-dlm) + * @description The OS version each Forward-collected device is running, for + * the optional netbox-dlm plugin's DeviceSoftware model (one row per device). + * The apply adapter ensures the matching SoftwareVersion exists (end-of-life + * dates come from the Forward DLM Software Versions map when the vendor + * publishes them). + */ + +import "netbox_utilities"; + +@query +f(forward_netbox_shard_keys: List) = +foreach device in network.devices +where device.snapshotInfo.result == DeviceSnapshotResult.completed +where device.platform.vendor != Vendor.FORWARD_CUSTOM +where isEmpty(forward_netbox_shard_keys) || device.name in forward_netbox_shard_keys +where isPresent(device.platform.osVersion) +let platform_name = normalizeDevicePlatformName(device) +select { + name: device.name, + platform: platform_name, + platform_slug: slugify(platform_name), + version: substring(device.platform.osVersion, 0, 50) +}; diff --git a/forward_netbox/queries/forward_dlm_hardware_notices.nqe b/forward_netbox/queries/forward_dlm_hardware_notices.nqe new file mode 100644 index 00000000..7e00c163 --- /dev/null +++ b/forward_netbox/queries/forward_dlm_hardware_notices.nqe @@ -0,0 +1,28 @@ +/** + * @intent Forward hardware end-of-life notices per device type (netbox-dlm) + * @description Distinct device models with vendor hardware end-of-life dates + * from Forward's per-part support analysis (chassis parts), for the optional + * netbox-dlm plugin's HardwareNotice model. Part support data is only present + * for vendors Forward tracks (Cisco, Palo Alto Networks, Fortinet). + */ + +import "netbox_utilities"; + +@query +f(forward_netbox_shard_keys: List) = +foreach device in network.devices +where device.snapshotInfo.result == DeviceSnapshotResult.completed +where device.platform.vendor != Vendor.FORWARD_CUSTOM +where isEmpty(forward_netbox_shard_keys) || device.name in forward_netbox_shard_keys +let model = device.platform.model +foreach part in device.platform.components +where part.partType == DevicePartType.CHASSIS +where isPresent(part.support) +select distinct { + device_type: model, + device_type_slug: slugifyNetboxModel(model), + end_of_support: part.support.lastSupportDate, + end_of_security_patches: part.support.lastVulnerabilityDate, + end_of_sw_releases: part.support.lastMaintenanceDate, + documentation_url: substring(part.support.announcementUrl, 0, 200) +}; diff --git a/forward_netbox/queries/forward_dlm_software_versions.nqe b/forward_netbox/queries/forward_dlm_software_versions.nqe new file mode 100644 index 00000000..ef92ec29 --- /dev/null +++ b/forward_netbox/queries/forward_dlm_software_versions.nqe @@ -0,0 +1,28 @@ +/** + * @intent Forward OS software versions with end-of-life dates (netbox-dlm) + * @description Distinct (platform, OS version) pairs with vendor end-of-life + * dates from Forward's OS support analysis, for the optional netbox-dlm + * plugin's SoftwareVersion model. Only devices whose OS has a known vendor + * end-of-life announcement are emitted (device.platform.osSupport is absent + * otherwise). + */ + +import "netbox_utilities"; + +@query +f(forward_netbox_shard_keys: List) = +foreach device in network.devices +where device.snapshotInfo.result == DeviceSnapshotResult.completed +where device.platform.vendor != Vendor.FORWARD_CUSTOM +where isEmpty(forward_netbox_shard_keys) || device.name in forward_netbox_shard_keys +where isPresent(device.platform.osSupport) +where isPresent(device.platform.osVersion) +let support = device.platform.osSupport +let platform_name = normalizeDevicePlatformName(device) +select distinct { + platform: platform_name, + platform_slug: slugify(platform_name), + version: substring(device.platform.osVersion, 0, 50), + end_of_support: support.lastSupportDate, + documentation_url: substring(support.announcementUrl, 0, 200) +}; diff --git a/forward_netbox/tests/test_dlm_integration.py b/forward_netbox/tests/test_dlm_integration.py new file mode 100644 index 00000000..3c84dd00 --- /dev/null +++ b/forward_netbox/tests/test_dlm_integration.py @@ -0,0 +1,106 @@ +# Optional netbox-dlm (Device Lifecycle Management) integration wiring. +# +# The plugin is NOT installed in CI; these tests cover the registry surface, +# bundled queries, and adapter contract so the integration is exercised without +# it. Field origin: the design partner authoring netbox-dlm runs this plugin. +import importlib + +from django.test import SimpleTestCase + +from forward_netbox.choices import FORWARD_DLM_MODELS +from forward_netbox.choices import FORWARD_OPTIONAL_MODELS +from forward_netbox.choices import FORWARD_SUPPORTED_MODELS +from forward_netbox.utilities.plugin_integrations.registry import DLM_INTEGRATION +from forward_netbox.utilities.plugin_integrations.registry import ( + optional_integration_for_model, +) +from forward_netbox.utilities.query_registry import _default_query_parameters +from forward_netbox.utilities.query_registry import _read_query +from forward_netbox.utilities.query_registry import builtin_nqe_map_rows + +DLM_MODEL_STRINGS = ( + "netbox_dlm.softwareversion", + "netbox_dlm.hardwarenotice", + "netbox_dlm.devicesoftware", +) + + +class DlmRegistryWiringTest(SimpleTestCase): + def test_models_registered_as_supported_and_optional(self): + for model_string in DLM_MODEL_STRINGS: + self.assertIn(model_string, FORWARD_DLM_MODELS) + self.assertIn(model_string, FORWARD_SUPPORTED_MODELS) + self.assertIn(model_string, FORWARD_OPTIONAL_MODELS) + + def test_integration_matches_models(self): + for model_string in DLM_MODEL_STRINGS: + integration = optional_integration_for_model(model_string) + self.assertIsNotNone(integration, model_string) + self.assertEqual(integration.app_label, "netbox_dlm") + + def test_builtin_maps_seeded_disabled_by_default(self): + rows = {row["name"]: row for row in builtin_nqe_map_rows()} + for name in DLM_INTEGRATION.query_maps: + self.assertIn(name, rows) + self.assertFalse( + rows[name].get("enabled", True), + f"{name} must ship disabled (opt-in plugin surface)", + ) + + def test_adapter_module_exposes_apply_and_delete_per_model(self): + adapter = importlib.import_module(DLM_INTEGRATION.adapter_module) + for model_string in DLM_INTEGRATION.supported_models: + slug = model_string.replace(".", "_") + self.assertTrue(callable(getattr(adapter, f"apply_{slug}", None)), slug) + self.assertTrue(callable(getattr(adapter, f"delete_{slug}", None)), slug) + + +class DlmQueryStructureTest(SimpleTestCase): + def test_software_versions_query_shape(self): + src = _read_query("forward_dlm_software_versions.nqe") + self.assertIn("device.platform.osSupport", src) + self.assertIn("lastSupportDate", src) + # dlm SoftwareVersion.version is Char(50); announcement URL is URLField. + self.assertIn("substring(device.platform.osVersion, 0, 50)", src) + self.assertIn("substring(support.announcementUrl, 0, 200)", src) + self.assertIn("select distinct", src) + + def test_hardware_notices_query_shape(self): + src = _read_query("forward_dlm_hardware_notices.nqe") + self.assertIn("DevicePartType.CHASSIS", src) + for field in ( + "end_of_support:", + "end_of_security_patches:", + "end_of_sw_releases:", + "device_type_slug:", + ): + self.assertIn(field, src) + + def test_device_software_query_shape(self): + src = _read_query("forward_dlm_device_software.nqe") + self.assertIn("device.platform.osVersion", src) + for field in ("name:", "platform_slug:", "version:"): + self.assertIn(field, src) + + def test_queries_seed_shard_parameter(self): + for filename in ( + "forward_dlm_software_versions.nqe", + "forward_dlm_hardware_notices.nqe", + "forward_dlm_device_software.nqe", + ): + params = _default_query_parameters(filename) + self.assertIn("forward_netbox_shard_keys", params, filename) + + +class DlmRunnerDispatchTest(SimpleTestCase): + def test_runner_has_apply_and_delete_methods(self): + from forward_netbox.utilities.sync import ForwardSyncRunner + + for model_string in DLM_MODEL_STRINGS: + slug = model_string.replace(".", "_") + self.assertTrue( + callable(getattr(ForwardSyncRunner, f"_apply_{slug}", None)), slug + ) + self.assertTrue( + callable(getattr(ForwardSyncRunner, f"_delete_{slug}", None)), slug + ) diff --git a/forward_netbox/tests/test_plugin_integrations.py b/forward_netbox/tests/test_plugin_integrations.py index 51b79f8a..6be68ede 100644 --- a/forward_netbox/tests/test_plugin_integrations.py +++ b/forward_netbox/tests/test_plugin_integrations.py @@ -19,6 +19,7 @@ def test_registry_exposes_generic_optional_integrations(self): "routing.netbox_routing", "peering.netbox_peering_manager", "aci.netbox_cisco_aci", + "lifecycle.netbox_dlm", ], ) self.assertEqual( @@ -42,6 +43,7 @@ def test_registry_exposes_generic_optional_integrations(self): "routing.netbox_routing", "peering.netbox_peering_manager", "aci.netbox_cisco_aci", + "lifecycle.netbox_dlm", }, ) self.assertEqual( @@ -60,6 +62,7 @@ def fake_version(package_name): "netbox-routing": "1.0.0", "netbox-peering-manager": "1.0.0", "netbox-cisco-aci": "0.2.1", + "netbox-dlm": "0.1.0", } return versions[package_name] diff --git a/forward_netbox/tests/test_provision_preflight.py b/forward_netbox/tests/test_provision_preflight.py new file mode 100644 index 00000000..0de65a6e --- /dev/null +++ b/forward_netbox/tests/test_provision_preflight.py @@ -0,0 +1,74 @@ +# A plugin in PLUGINS whose migrations were never applied leaves ContentTypes +# registered but tables missing; branch provisioning then dies mid-CREATE TABLE +# with an opaque ProgrammingError (field report: netbox_dlm_contract). The +# preflight must catch it before provisioning, with the remedy in the message. +from unittest.mock import Mock +from unittest.mock import patch + +from core.exceptions import SyncError +from django.test import TestCase + +from forward_netbox.utilities.branching import missing_branch_table_report +from forward_netbox.utilities.health import _database_tables_check +from forward_netbox.utilities.single_branch_executor import ( + ForwardSingleBranchExecutor, +) + + +class MissingBranchTableReportTest(TestCase): + def test_healthy_database_reports_nothing(self): + self.assertEqual(missing_branch_table_report(), {}) + + def test_unmigrated_table_is_reported(self): + real_tables = ["dcim_device", "netbox_dlm_contract"] + with patch( + "netbox_branching.utilities.get_tables_to_replicate", + return_value=real_tables, + ): + report = missing_branch_table_report() + # dcim_device exists; the dlm table does not (no model registered for + # it in this environment either, so it lands in the unknown bucket). + self.assertEqual(report, {"unknown": ["netbox_dlm_contract"]}) + + +class ExecutorTablePreflightTest(TestCase): + def test_run_raises_actionable_sync_error_before_fetch(self): + executor = object.__new__(ForwardSingleBranchExecutor) + with ( + patch.object(ForwardSingleBranchExecutor, "_set_runtime_phase", Mock()), + patch( + "forward_netbox.utilities.single_branch_executor." + "missing_branch_table_report", + return_value={"netbox_dlm": ["netbox_dlm_contract"]}, + ), + ): + with self.assertRaises(SyncError) as ctx: + executor.run() + message = str(ctx.exception) + self.assertIn("netbox_dlm", message) + self.assertIn("netbox_dlm_contract", message) + self.assertIn("migrate", message) + + +class DatabaseTablesHealthCheckTest(TestCase): + def test_healthy_database_passes(self): + check = _database_tables_check() + self.assertIsNotNone(check) + self.assertEqual(check["status"], "pass") + + def test_missing_tables_fail_with_remedy(self): + with patch( + "forward_netbox.utilities.health.missing_branch_table_report", + return_value={"netbox_dlm": ["netbox_dlm_contract"]}, + ): + check = _database_tables_check() + self.assertEqual(check["status"], "fail") + self.assertIn("netbox_dlm_contract", check["message"]) + self.assertIn("migrate", check["message"]) + + def test_diagnostics_failure_returns_none_not_crash(self): + with patch( + "forward_netbox.utilities.health.missing_branch_table_report", + side_effect=RuntimeError("introspection unavailable"), + ): + self.assertIsNone(_database_tables_check()) diff --git a/forward_netbox/utilities/apply_engine_decision.py b/forward_netbox/utilities/apply_engine_decision.py index 157dacf1..d69d79c2 100644 --- a/forward_netbox/utilities/apply_engine_decision.py +++ b/forward_netbox/utilities/apply_engine_decision.py @@ -262,9 +262,33 @@ "netbox_cisco_aci.acifilter", "netbox_cisco_aci.acil3out", "netbox_cisco_aci.acistaticportbinding", + "netbox_dlm.softwareversion", + "netbox_dlm.hardwarenotice", + "netbox_dlm.devicesoftware", } ADAPTER_MODEL_BLOCKERS = { + "netbox_dlm.softwareversion": { + "blocker_code": "plugin_model_dependencies", + "blocker_reason": ( + "DLM software-version writes require plugin model and platform " + "dependency resolution handled by the adapter." + ), + }, + "netbox_dlm.hardwarenotice": { + "blocker_code": "plugin_model_dependencies", + "blocker_reason": ( + "DLM hardware-notice writes require plugin model and device-type " + "dependency resolution handled by the adapter." + ), + }, + "netbox_dlm.devicesoftware": { + "blocker_code": "plugin_model_dependencies", + "blocker_reason": ( + "DLM device-software writes require device and software-version " + "dependency resolution handled by the adapter." + ), + }, "dcim.cable": { "blocker_code": "relationship_identity_directionality", "blocker_reason": ( diff --git a/forward_netbox/utilities/branch_budget.py b/forward_netbox/utilities/branch_budget.py index 75aec13a..ee428219 100644 --- a/forward_netbox/utilities/branch_budget.py +++ b/forward_netbox/utilities/branch_budget.py @@ -21,6 +21,9 @@ DEFAULT_MODEL_CHANGE_DENSITY = { "dcim.cable": 3.0, "dcim.module": 2.0, + "netbox_dlm.softwareversion": 1.0, + "netbox_dlm.hardwarenotice": 1.0, + "netbox_dlm.devicesoftware": 1.0, "netbox_routing.bgppeer": 5.0, "netbox_routing.bgpaddressfamily": 2.0, "netbox_routing.bgppeeraddressfamily": 2.0, @@ -70,6 +73,9 @@ "netbox_routing.bgpaddressfamily", "netbox_routing.bgppeeraddressfamily", "netbox_peering_manager.peeringsession", + "netbox_dlm.softwareversion", + "netbox_dlm.hardwarenotice", + "netbox_dlm.devicesoftware", "netbox_cisco_aci.acifabric", "netbox_cisco_aci.acipod", "netbox_cisco_aci.acinode", @@ -110,6 +116,9 @@ "dcim.interface", "netbox_routing.ospfinstance", ), + "netbox_dlm.softwareversion": ("dcim.platform",), + "netbox_dlm.hardwarenotice": ("dcim.devicetype",), + "netbox_dlm.devicesoftware": ("dcim.device", "netbox_dlm.softwareversion"), "netbox_routing.bgppeer": ("dcim.device", "ipam.vrf"), "netbox_routing.bgpaddressfamily": ("dcim.device", "ipam.vrf"), "netbox_routing.bgppeeraddressfamily": ( @@ -141,6 +150,9 @@ ), } DELETE_DEPENDENCY_MODEL_ORDER = ( + "netbox_dlm.devicesoftware", + "netbox_dlm.hardwarenotice", + "netbox_dlm.softwareversion", "dcim.cable", "ipam.fhrpgroup", "ipam.ipaddress", @@ -312,6 +324,16 @@ def apply_dependency_dry_run(enabled_models) -> dict: SHARD_FETCH_PARAMETER_BUCKET_COUNT = "forward_netbox_shard_bucket_count" +def _fallback_bucket_key_family(model_string): + field_sets = default_coalesce_fields_for_model(model_string) + if not field_sets: + return "" + first = field_sets[0] + if not first: + return "" + return ",".join(str(field_name) for field_name in first if str(field_name)) + + def _model_fetch_fallback_contract(model_string): bucket_key_family = _fallback_bucket_key_family(model_string) return { @@ -689,16 +711,6 @@ def _structured_parameter_contract(model_string, shard_keys): return {} -def _fallback_bucket_key_family(model_string): - field_sets = default_coalesce_fields_for_model(model_string) - if not field_sets: - return "" - first = field_sets[0] - if not first: - return "" - return ",".join(str(field_name) for field_name in first if str(field_name)) - - def _parse_structured_shard_key(shard_key): parsed = {} for part in str(shard_key).split("|"): diff --git a/forward_netbox/utilities/branching.py b/forward_netbox/utilities/branching.py index 2204bb43..9acdc681 100644 --- a/forward_netbox/utilities/branching.py +++ b/forward_netbox/utilities/branching.py @@ -3,6 +3,36 @@ from utilities.request import NetBoxFakeRequest +def missing_branch_table_report(): + """Map app_label -> branch-replicated tables missing from the database. + + Django registers ContentTypes for every installed app even when that app's + migrations were never applied, so netbox_branching's provision happily runs + ``CREATE TABLE branch.T (LIKE public.T)`` for a table that does not exist + and the sync dies mid-provision with an opaque ProgrammingError (field + report: a plugin installed without running its migrations). Detect the gap + up front using the exact table list provision will replicate. + """ + from django.apps import apps + from django.db import connection + from netbox_branching.utilities import get_tables_to_replicate + + missing = set(get_tables_to_replicate()) - set( + connection.introspection.table_names() + ) + if not missing: + return {} + by_app: dict[str, set[str]] = {} + for model in apps.get_models(include_auto_created=True): + table = model._meta.db_table + if table in missing: + by_app.setdefault(model._meta.app_label, set()).add(table) + mapped = set().union(*by_app.values()) if by_app else set() + for table in missing - mapped: + by_app.setdefault("unknown", set()).add(table) + return {app: sorted(tables) for app, tables in sorted(by_app.items())} + + def build_branch_name(*, sync, ingestion, item): sync_id = getattr(sync, "pk", None) ingestion_id = getattr(ingestion, "pk", None) diff --git a/forward_netbox/utilities/health.py b/forward_netbox/utilities/health.py index a50f34e4..97a9c414 100644 --- a/forward_netbox/utilities/health.py +++ b/forward_netbox/utilities/health.py @@ -1,5 +1,6 @@ from collections import Counter +from .branching import missing_branch_table_report from .execution_ledger import api_usage_support_summary from .execution_ledger import dependency_lookup_cache_support_summary from .execution_ledger import ( @@ -271,6 +272,35 @@ def _base_variant_conflict_check(sync): ) +def _database_tables_check(): + """Fail loudly when an installed app's tables were never migrated. + + Branch provisioning replicates every branchable table with CREATE TABLE + LIKE; a plugin in PLUGINS whose migrations were never applied kills the + sync mid-provision. Surface it on the Health page with the remedy. + """ + try: + report = missing_branch_table_report() + except Exception: # noqa: BLE001 - diagnostics must never break Health + return None + if not report: + return _check( + name="Database tables", + status="pass", + message="Every branch-replicated table exists in the database.", + ) + detail = "; ".join(f"{app}: {', '.join(tables)}" for app, tables in report.items()) + return _check( + name="Database tables", + status="fail", + message=( + f"Installed apps have missing database tables ({detail}) — branch " + "provisioning will fail. Apply their migrations " + "(python manage.py migrate) or remove the plugin from PLUGINS." + ), + ) + + def sync_health_summary(sync): optional_plugin_capabilities = integration_capability_summary() maps = [ @@ -364,6 +394,9 @@ def sync_health_summary(sync): variant_conflict_check = _base_variant_conflict_check(sync) if variant_conflict_check is not None: checks.append(variant_conflict_check) + database_tables_check = _database_tables_check() + if database_tables_check is not None: + checks.append(database_tables_check) return { "source": _source_summary(sync), diff --git a/forward_netbox/utilities/plugin_integrations/registry.py b/forward_netbox/utilities/plugin_integrations/registry.py index 70939547..9ba89fd6 100644 --- a/forward_netbox/utilities/plugin_integrations/registry.py +++ b/forward_netbox/utilities/plugin_integrations/registry.py @@ -297,10 +297,45 @@ def as_dict(self): ) +DLM_INTEGRATION = OptionalPluginIntegration( + key="lifecycle.netbox_dlm", + app_label="netbox_dlm", + display_name="NetBox Device Lifecycle Management", + required_models=( + "netbox_dlm.softwareversion", + "netbox_dlm.hardwarenotice", + "netbox_dlm.devicesoftware", + ), + supported_models=( + "netbox_dlm.softwareversion", + "netbox_dlm.hardwarenotice", + "netbox_dlm.devicesoftware", + ), + query_maps=( + "Forward DLM Software Versions", + "Forward DLM Hardware Notices", + "Forward DLM Device Software", + ), + package_names=("netbox-dlm", "netbox_dlm"), + adapter_module="forward_netbox.utilities.sync_dlm", + enabled_by_default=False, + status="beta_surface", + notes=( + "Beta lifecycle import surface backed by the optional netbox-dlm " + "plugin: OS end-of-life dates per (platform, version), hardware " + "end-of-life notices per device type, and each device's running " + "software version from Forward's support analysis.", + "netbox-dlm ships no migrations; run `manage.py makemigrations " + "netbox_dlm && manage.py migrate` after installing it.", + ), +) + + OPTIONAL_PLUGIN_INTEGRATIONS = ( ROUTING_INTEGRATION, PEERING_INTEGRATION, ACI_INTEGRATION, + DLM_INTEGRATION, ) diff --git a/forward_netbox/utilities/query_fetch_execution.py b/forward_netbox/utilities/query_fetch_execution.py index 3272b297..f84cb289 100644 --- a/forward_netbox/utilities/query_fetch_execution.py +++ b/forward_netbox/utilities/query_fetch_execution.py @@ -250,6 +250,7 @@ def _extract_device_names(value: Any) -> set[str]: "netbox_routing.ospfarea": "device", "netbox_routing.ospfinterface": "device", "netbox_peering_manager.peeringsession": "device", + "netbox_dlm.devicesoftware": "name", } diff --git a/forward_netbox/utilities/query_registry.py b/forward_netbox/utilities/query_registry.py index 0c1c94c8..91f79b62 100644 --- a/forward_netbox/utilities/query_registry.py +++ b/forward_netbox/utilities/query_registry.py @@ -390,6 +390,24 @@ def routing_import_diagnostic_query() -> str: "filename": "forward_device_feature_tags_with_rules.nqe", "enabled": False, }, + { + "model_string": "netbox_dlm.softwareversion", + "name": "Forward DLM Software Versions", + "filename": "forward_dlm_software_versions.nqe", + "enabled": False, + }, + { + "model_string": "netbox_dlm.hardwarenotice", + "name": "Forward DLM Hardware Notices", + "filename": "forward_dlm_hardware_notices.nqe", + "enabled": False, + }, + { + "model_string": "netbox_dlm.devicesoftware", + "name": "Forward DLM Device Software", + "filename": "forward_dlm_device_software.nqe", + "enabled": False, + }, { "model_string": "dcim.module", "name": "Forward Modules", diff --git a/forward_netbox/utilities/single_branch_executor.py b/forward_netbox/utilities/single_branch_executor.py index 60b8ec36..7c410cd7 100644 --- a/forward_netbox/utilities/single_branch_executor.py +++ b/forward_netbox/utilities/single_branch_executor.py @@ -15,6 +15,7 @@ from ..choices import ForwardSyncStatusChoices from .branch_budget import build_branch_plan from .branching import build_branch_request +from .branching import missing_branch_table_report from .fast_bootstrap_executor import ForwardFastBootstrapExecutor from .multi_branch_lifecycle import create_noop_ingestion from .multi_branch_lifecycle import run_item_in_branch @@ -29,6 +30,19 @@ class ForwardSingleBranchExecutor(ForwardFastBootstrapExecutor): def run(self): self._set_runtime_phase("initializing", "Starting single-branch preflight.") + # Fail in seconds — before the expensive fetch and before provisioning + # would die mid-CREATE TABLE on a table that was never migrated. + missing_tables = missing_branch_table_report() + if missing_tables: + detail = "; ".join( + f"{app}: {', '.join(tables)}" for app, tables in missing_tables.items() + ) + raise SyncError( + "Cannot provision a branch: database tables are missing for " + f"installed apps ({detail}). Apply their migrations " + "(python manage.py migrate) or remove the plugin from PLUGINS, " + "then re-run the sync." + ) fetcher = ForwardQueryFetcher(self.sync, self.client, self.logger) context = fetcher.resolve_context() self._set_runtime_phase( diff --git a/forward_netbox/utilities/sync.py b/forward_netbox/utilities/sync.py index a0a765ee..50000cd2 100644 --- a/forward_netbox/utilities/sync.py +++ b/forward_netbox/utilities/sync.py @@ -20,6 +20,12 @@ from .sync_device import apply_dcim_virtualchassis from .sync_device import delete_dcim_device from .sync_device import delete_dcim_virtualchassis +from .sync_dlm import apply_netbox_dlm_devicesoftware +from .sync_dlm import apply_netbox_dlm_hardwarenotice +from .sync_dlm import apply_netbox_dlm_softwareversion +from .sync_dlm import delete_netbox_dlm_devicesoftware +from .sync_dlm import delete_netbox_dlm_hardwarenotice +from .sync_dlm import delete_netbox_dlm_softwareversion from .sync_events import EventsClearer from .sync_execution import run_sync_stage from .sync_interface import apply_dcim_interface @@ -202,6 +208,12 @@ sync_record_aggregated_conflict_warning, sync_record_aggregated_skip_warning, sync_record_issue, + apply_netbox_dlm_devicesoftware, + apply_netbox_dlm_hardwarenotice, + apply_netbox_dlm_softwareversion, + delete_netbox_dlm_devicesoftware, + delete_netbox_dlm_hardwarenotice, + delete_netbox_dlm_softwareversion, apply_netbox_peering_manager_peeringsession, apply_netbox_routing_bgpaddressfamily, apply_netbox_routing_bgppeer, diff --git a/forward_netbox/utilities/sync_contracts.py b/forward_netbox/utilities/sync_contracts.py index 7a32c8ba..c42b9535 100644 --- a/forward_netbox/utilities/sync_contracts.py +++ b/forward_netbox/utilities/sync_contracts.py @@ -273,6 +273,21 @@ class ModelSyncContract: ("device", "neighbor_address"), ), ), + "netbox_dlm.softwareversion": ModelSyncContract( + required_fields=("platform", "platform_slug", "version"), + allowed_coalesce_fields=("platform", "platform_slug", "version"), + default_coalesce_fields=(("platform_slug", "version"),), + ), + "netbox_dlm.hardwarenotice": ModelSyncContract( + required_fields=("device_type", "device_type_slug"), + allowed_coalesce_fields=("device_type", "device_type_slug"), + default_coalesce_fields=(("device_type_slug",),), + ), + "netbox_dlm.devicesoftware": ModelSyncContract( + required_fields=("name", "platform", "platform_slug", "version"), + allowed_coalesce_fields=("name",), + default_coalesce_fields=(("name",),), + ), "netbox_cisco_aci.acifabric": ModelSyncContract( required_fields=("name", "fabric_id"), allowed_coalesce_fields=("name",), diff --git a/forward_netbox/utilities/sync_dlm.py b/forward_netbox/utilities/sync_dlm.py new file mode 100644 index 00000000..629d8ac6 --- /dev/null +++ b/forward_netbox/utilities/sync_dlm.py @@ -0,0 +1,217 @@ +# Optional netbox-dlm (Device Lifecycle Management) integration. +# +# Maps Forward's end-of-life analysis to the netbox-dlm plugin's models: +# - SoftwareVersion <- device.platform.osSupport (per platform + OS version) +# - HardwareNotice <- device.platform.components[].support (per DeviceType) +# - DeviceSoftware <- device.platform.osVersion (one row per device) +# +# netbox-dlm's README expects DeviceSoftware to be populated by external sync +# tooling; this adapter is that tooling. All writes go through the standard +# runner upsert/delete primitives so branch staging, diffs, and prune behave +# exactly like every other model. +from datetime import date + +from django.core.exceptions import ObjectDoesNotExist + +from ..exceptions import ForwardDependencySkipError +from ..exceptions import ForwardSearchError + + +def _parse_date(value): + """ISO date string -> date, so values compare equal to stored DateFields + (a str value never equals a date and would update the row every sync).""" + if isinstance(value, date) or value in (None, ""): + return value or None + try: + return date.fromisoformat(str(value)) + except ValueError: + return None + + +def _dlm_model(runner, model_name, model_string): + return runner._optional_model("netbox_dlm", model_name, model_string) + + +def _lookup_platform(runner, row, model_string, object_label): + from dcim.models import Platform + + slug = str(row.get("platform_slug") or "").strip() + name = str(row.get("platform") or "").strip() + platform = None + if slug: + platform = runner._get_unique_or_raise(Platform, {"slug": slug}) + if platform is None and name: + platform = runner._get_unique_or_raise(Platform, {"name": name}) + if platform is None: + raise ForwardDependencySkipError( + f"Skipping {object_label} because platform `{name or slug}` is not " + "in NetBox yet.", + model_string=model_string, + context={"platform": name or slug}, + data=row, + ) + return platform + + +def _lookup_device_type(runner, row, model_string, object_label): + from dcim.models import DeviceType + + slug = str(row.get("device_type_slug") or "").strip() + model = str(row.get("device_type") or "").strip() + device_type = None + if slug: + device_type = runner._get_unique_or_raise(DeviceType, {"slug": slug}) + if device_type is None and model: + device_type = runner._get_unique_or_raise(DeviceType, {"model": model}) + if device_type is None: + raise ForwardDependencySkipError( + f"Skipping {object_label} because device type `{model or slug}` is " + "not in NetBox yet.", + model_string=model_string, + context={"device_type": model or slug}, + data=row, + ) + return device_type + + +def _lookup_device(runner, row, model_string, object_label): + try: + return runner._get_device_by_name(row["name"]) + except ObjectDoesNotExist as exc: + key = (row["name"],) + if runner._dependency_failed("dcim.device", key): + raise ForwardDependencySkipError( + f"Skipping {object_label} because dependency `dcim.device` " + f"failed for {key}.", + model_string=model_string, + context={"device": row["name"]}, + data=row, + ) from exc + raise ForwardSearchError( + f"Unable to find device `{row['name']}` for {object_label}.", + model_string=model_string, + context={"device": row["name"]}, + data=row, + ) from exc + + +def ensure_dlm_software_version(runner, row, *, with_dates=True): + SoftwareVersion = _dlm_model( + runner, "SoftwareVersion", "netbox_dlm.softwareversion" + ) + platform = _lookup_platform( + runner, row, "netbox_dlm.softwareversion", "DLM software version" + ) + values = { + "platform": platform, + "version": str(row.get("version") or "").strip(), + } + if with_dates: + end_of_support = _parse_date(row.get("end_of_support")) + if end_of_support: + values["end_of_support"] = end_of_support + if row.get("documentation_url"): + values["documentation_url"] = row["documentation_url"] + values = runner._model_field_values(SoftwareVersion, values) + if with_dates: + software_version, _ = runner._upsert_values_from_defaults( + "netbox_dlm.softwareversion", + SoftwareVersion, + values=values, + coalesce_sets=[("platform", "version")], + ) + else: + # Create-if-missing only: never overwrite end-of-life dates the + # versions map already applied. + software_version, _ = runner._coalesce_update_or_create( + SoftwareVersion, + coalesce_lookups=[{"platform": platform, "version": values.get("version")}], + create_values=values, + update_values={}, + ) + return software_version + + +def apply_netbox_dlm_softwareversion(runner, row): + return ensure_dlm_software_version(runner, row, with_dates=True) + + +def apply_netbox_dlm_hardwarenotice(runner, row): + HardwareNotice = _dlm_model(runner, "HardwareNotice", "netbox_dlm.hardwarenotice") + device_type = _lookup_device_type( + runner, row, "netbox_dlm.hardwarenotice", "DLM hardware notice" + ) + values = {"device_type": device_type} + for field in ("end_of_support", "end_of_security_patches", "end_of_sw_releases"): + parsed = _parse_date(row.get(field)) + if parsed: + values[field] = parsed + if row.get("documentation_url"): + values["documentation_url"] = row["documentation_url"] + values = runner._model_field_values(HardwareNotice, values) + notice, _ = runner._upsert_values_from_defaults( + "netbox_dlm.hardwarenotice", + HardwareNotice, + values=values, + coalesce_sets=[("device_type",)], + ) + return notice + + +def apply_netbox_dlm_devicesoftware(runner, row): + DeviceSoftware = _dlm_model(runner, "DeviceSoftware", "netbox_dlm.devicesoftware") + device = _lookup_device( + runner, row, "netbox_dlm.devicesoftware", "DLM device software" + ) + # Ensure the (platform, version) row exists even when the vendor publishes + # no end-of-life announcement (the versions map only emits announced ones); + # never overwrite dates the versions map already applied. + software_version = ensure_dlm_software_version(runner, row, with_dates=False) + values = runner._model_field_values( + DeviceSoftware, + {"device": device, "software_version": software_version}, + ) + device_software, _ = runner._upsert_values_from_defaults( + "netbox_dlm.devicesoftware", + DeviceSoftware, + values=values, + coalesce_sets=[("device",)], + ) + return device_software + + +def delete_netbox_dlm_softwareversion(runner, row): + SoftwareVersion = _dlm_model( + runner, "SoftwareVersion", "netbox_dlm.softwareversion" + ) + from dcim.models import Platform + + platform = runner._get_unique_or_raise( + Platform, {"slug": str(row.get("platform_slug") or "").strip()} + ) + if platform is None: + return False + return runner._delete_by_coalesce( + SoftwareVersion, + [{"platform": platform, "version": str(row.get("version") or "").strip()}], + ) + + +def delete_netbox_dlm_hardwarenotice(runner, row): + HardwareNotice = _dlm_model(runner, "HardwareNotice", "netbox_dlm.hardwarenotice") + from dcim.models import DeviceType + + device_type = runner._get_unique_or_raise( + DeviceType, {"slug": str(row.get("device_type_slug") or "").strip()} + ) + if device_type is None: + return False + return runner._delete_by_coalesce(HardwareNotice, [{"device_type": device_type}]) + + +def delete_netbox_dlm_devicesoftware(runner, row): + DeviceSoftware = _dlm_model(runner, "DeviceSoftware", "netbox_dlm.devicesoftware") + device = runner._lookup_device_by_name(row.get("name")) + if device is None: + return False + return runner._delete_by_coalesce(DeviceSoftware, [{"device": device}]) diff --git a/forward_netbox/utilities/sync_primitives.py b/forward_netbox/utilities/sync_primitives.py index 8da4065e..7b63898d 100644 --- a/forward_netbox/utilities/sync_primitives.py +++ b/forward_netbox/utilities/sync_primitives.py @@ -31,6 +31,9 @@ "ipam.vlan": (("site", "vid"),), "ipam.prefix": (("prefix", "vrf"), ("prefix", "vrf__isnull")), "netbox_peering_manager.peeringsession": (("bgp_peer",),), + "netbox_dlm.softwareversion": (("platform", "version"),), + "netbox_dlm.hardwarenotice": (("device_type",),), + "netbox_dlm.devicesoftware": (("device",),), "netbox_routing.bgprouter": ( ("assigned_object_type", "assigned_object_id", "asn"), ), @@ -742,6 +745,7 @@ def prime_dependency_lookup_caches(runner, model_string, rows): "netbox_routing.bgppeeraddressfamily": ("device",), "netbox_routing.ospfinstance": ("device",), "netbox_routing.ospfinterface": ("device",), + "netbox_dlm.devicesoftware": ("name",), } DEPENDENCY_PARENT_DEVICE_MODELS = tuple(DEPENDENCY_PARENT_DEVICE_FIELDS) @@ -856,6 +860,7 @@ def _dependency_device_names(model_string, rows): "netbox_routing.bgppeeraddressfamily": ("device",), "netbox_routing.ospfinstance": ("device",), "netbox_routing.ospfinterface": ("device",), + "netbox_dlm.devicesoftware": ("name",), }.get(model_string, ()) return { str(row.get(field)).strip() diff --git a/forward_netbox/utilities/sync_runner_adapters.py b/forward_netbox/utilities/sync_runner_adapters.py index 0e6c3ba5..cbe9e271 100644 --- a/forward_netbox/utilities/sync_runner_adapters.py +++ b/forward_netbox/utilities/sync_runner_adapters.py @@ -42,6 +42,12 @@ from .sync_device import apply_dcim_virtualchassis from .sync_device import delete_dcim_device from .sync_device import delete_dcim_virtualchassis +from .sync_dlm import apply_netbox_dlm_devicesoftware +from .sync_dlm import apply_netbox_dlm_hardwarenotice +from .sync_dlm import apply_netbox_dlm_softwareversion +from .sync_dlm import delete_netbox_dlm_devicesoftware +from .sync_dlm import delete_netbox_dlm_hardwarenotice +from .sync_dlm import delete_netbox_dlm_softwareversion from .sync_interface import apply_dcim_interface from .sync_interface import apply_dcim_macaddress from .sync_interface import apply_extras_taggeditem @@ -916,6 +922,24 @@ def _apply_netbox_routing_ospfinterface(self, row): def _apply_netbox_peering_manager_peeringsession(self, row): return apply_netbox_peering_manager_peeringsession(self, row) + def _apply_netbox_dlm_softwareversion(self, row): + return apply_netbox_dlm_softwareversion(self, row) + + def _apply_netbox_dlm_hardwarenotice(self, row): + return apply_netbox_dlm_hardwarenotice(self, row) + + def _apply_netbox_dlm_devicesoftware(self, row): + return apply_netbox_dlm_devicesoftware(self, row) + + def _delete_netbox_dlm_softwareversion(self, row): + return delete_netbox_dlm_softwareversion(self, row) + + def _delete_netbox_dlm_hardwarenotice(self, row): + return delete_netbox_dlm_hardwarenotice(self, row) + + def _delete_netbox_dlm_devicesoftware(self, row): + return delete_netbox_dlm_devicesoftware(self, row) + def _apply_netbox_cisco_aci_acifabric(self, row): return apply_netbox_cisco_aci_acifabric(self, row) diff --git a/pyproject.toml b/pyproject.toml index 1f23d5a4..c19199ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "forward-netbox" -version = "2.4.5" +version = "2.5.0" description = "NetBox plugin to sync Forward data into NetBox via built-in NQE queries" authors = ["Craig Johnson "] license = "MIT" From 0c693b376c50875f243a152aff8212f92f49dd7c Mon Sep 17 00:00:00 2001 From: Craig Johnson Date: Fri, 10 Jul 2026 01:43:52 +0300 Subject: [PATCH 2/2] docs: add netbox_dlm rows to apply-engine model matrix (harness gate) --- docs/02_Reference/apply-engine-model-matrix.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/02_Reference/apply-engine-model-matrix.md b/docs/02_Reference/apply-engine-model-matrix.md index 57e9e790..5d2d589f 100644 --- a/docs/02_Reference/apply-engine-model-matrix.md +++ b/docs/02_Reference/apply-engine-model-matrix.md @@ -51,6 +51,9 @@ These models remain adapter-required with explicit blocker codes: - `netbox_cisco_aci.acitenant` | `plugin_model_dependencies` - `netbox_cisco_aci.acivrf` | `plugin_model_dependencies` - `netbox_peering_manager.peeringsession` | `plugin_model_dependencies` +- `netbox_dlm.devicesoftware` | `plugin_model_dependencies` +- `netbox_dlm.hardwarenotice` | `plugin_model_dependencies` +- `netbox_dlm.softwareversion` | `plugin_model_dependencies` - `netbox_routing.bgpaddressfamily` | `plugin_model_dependencies` - `netbox_routing.bgppeer` | `plugin_model_dependencies` - `netbox_routing.bgppeeraddressfamily` | `plugin_model_dependencies`