diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e5cda75..a3b2fdf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ Generated from the README compatibility table by `scripts/gen_changelog.py`. Do not edit by hand. +## v2.5.0 + +Feature: optional **netbox-dlm** (Device Lifecycle Management) integration — three new opt-in NQE maps sync Forward's end-of-life analysis into the netbox-dlm plugin: OS software versions with vendor EOL dates per (platform, version), hardware end-of-life notices per device type (Cisco/Palo Alto/Fortinet part support), and each device's running software version. Disabled by default; requires the netbox-dlm plugin (run `makemigrations netbox_dlm && migrate` after installing it — it ships no migrations). Fix: syncs no longer crash mid-provision when an installed plugin's migrations were never applied (`relation ... does not exist`) — a preflight now fails in seconds with the app name and remedy, and a new **Database tables** Health check surfaces the gap before you sync. + ## v2.4.5 Fix: sync no longer crashes on netbox-branching **1.1.1** (`SquashMergeStrategy has no attribute '_split_bidirectional_cycles'` — 1.1.1 removed that internal helper; the bidirectional-cycle split is now built into the plugin and the dependency is bounded to `<1.2`). Also fixes SNMP-endpoint rows failing validation: the bundled endpoint query branches now clamp sysDescr-derived `device_type` to NetBox's 100-char limit (`substring`) and guard empty slugs — the fix lives in the NQE queries (the source of truth), so **Publish Bundled Queries** again after upgrading (fixes the `Ensure this value has at most 100 characters` rejects and the `At least one coalesce lookup must be provided` error). diff --git a/README.md b/README.md index f5abe314..d4607cf9 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,8 @@ On top of the import it adds scope control (sync only devices carrying chosen Fo - **It is not a source of truth for Forward configuration or intent** — it populates NetBox from Forward's collected snapshot, nothing more. - **It does not require** the optional `netbox-routing` / `netbox-peering-manager` - plugins unless you enable the beta BGP/OSPF maps. + / `netbox-dlm` plugins unless you enable the beta BGP/OSPF or device-lifecycle + maps. ## Screenshots @@ -66,7 +67,8 @@ Latest release requires NetBox `4.6.4` and `netbox-branching` `1.1.0+`. Expand f | Plugin Release | NetBox Version | Status | | --- | --- | --- | -| `v2.4.5` | `4.6.4` required; needs netbox-branching `1.1.0` – `1.1.x` | Current release; Fix: sync no longer crashes on netbox-branching **1.1.1** (`SquashMergeStrategy has no attribute '_split_bidirectional_cycles'` — 1.1.1 removed that internal helper; the bidirectional-cycle split is now built into the plugin and the dependency is bounded to `<1.2`). Also fixes SNMP-endpoint rows failing validation: the bundled endpoint query branches now clamp sysDescr-derived `device_type` to NetBox's 100-char limit (`substring`) and guard empty slugs — the fix lives in the NQE queries (the source of truth), so **Publish Bundled Queries** again after upgrading (fixes the `Ensure this value has at most 100 characters` rejects and the `At least one coalesce lookup must be provided` error). | +| `v2.5.0` | `4.6.4` required; needs netbox-branching `1.1.0` – `1.1.x` | Current release; Feature: optional **netbox-dlm** (Device Lifecycle Management) integration — three new opt-in NQE maps sync Forward's end-of-life analysis into the netbox-dlm plugin: OS software versions with vendor EOL dates per (platform, version), hardware end-of-life notices per device type (Cisco/Palo Alto/Fortinet part support), and each device's running software version. Disabled by default; requires the netbox-dlm plugin (run `makemigrations netbox_dlm && migrate` after installing it — it ships no migrations). Fix: syncs no longer crash mid-provision when an installed plugin's migrations were never applied (`relation ... does not exist`) — a preflight now fails in seconds with the app name and remedy, and a new **Database tables** Health check surfaces the gap before you sync. | +| `v2.4.5` | `4.6.4` required; needs netbox-branching `1.1.0` – `1.1.x` | Superseded by `v2.5.0`; Fix: sync no longer crashes on netbox-branching **1.1.1** (`SquashMergeStrategy has no attribute '_split_bidirectional_cycles'` — 1.1.1 removed that internal helper; the bidirectional-cycle split is now built into the plugin and the dependency is bounded to `<1.2`). Also fixes SNMP-endpoint rows failing validation: the bundled endpoint query branches now clamp sysDescr-derived `device_type` to NetBox's 100-char limit (`substring`) and guard empty slugs — the fix lives in the NQE queries (the source of truth), so **Publish Bundled Queries** again after upgrading (fixes the `Ensure this value has at most 100 characters` rejects and the `At least one coalesce lookup must be provided` error). | | `v2.4.4` | `4.6.4` required; needs netbox-branching `1.1.0+` | Superseded by `v2.4.5`; Fix: SNMP-endpoint import now works on **tag-scoped** syncs — the device-tag include scope silently excluded every endpoint, both query-side and in the plugin's local scope filter (whose scoped-device set was built from modeled devices only, so endpoint rows were always dropped; with prune enabled they would even be deleted). Endpoint import now ignores the include scope (exclude tags still apply) and endpoint names join the scoped set (validated live: 355 Avocent endpoints import under a tag-scoped sync). Also fixes the merge-phase `Tag with this Name already exists` issues: a same-named/same-slug tag already on main is now treated as merged instead of failing the branch's tag create. | | `v2.4.3` | `4.6.4` required; needs netbox-branching `1.1.0+` | Superseded by `v2.4.4`; Fix: the pinned-query opt-in Health warning (2.4.1) over-claimed failure — it reads "nothing new syncs" and fires on any pinned map with the feature on, even after the query is fixed, because the Health page can't read a pinned query's contents. Reworded to a "Pinned — can't verify locally" heads-up that points at **Export Live Query Drift** to confirm (`source_matches_bundled`), instead of asserting failure. No behavior change. | | `v2.4.2` | `4.6.4` required; needs netbox-branching `1.1.0+` | Superseded by `v2.4.3`; Fix: endpoint import (`sync_endpoints`) and device-tag sync (`sync_device_tags`) now work with the alias-aware and rules-aware query variants (`forward_devices_with_netbox_aliases`, `forward_device_feature_tags_with_rules`), not just the base queries — operators running the variants saw the toggles silently do nothing (validated live: 355 Avocent endpoints import; `Mgmt_*` tags sync). Adds a **Publish Bundled Queries** button on the sync Health page (beside Refresh Query IDs) and two Health warnings: when an opt-in feature is enabled but no enabled map provides it, and when a base query and its opt-in variant are both enabled (they double-apply rows for the same model and churn — enable one). The alias-aware device query now emits the clean role name (e.g. `ROUTER`) to match the base query — expect a one-time role update on alias-mapped devices. | @@ -201,6 +203,7 @@ use. - Optional disabled NQE maps for NetBox Device Type Library alias matching through a Forward JSON data file - Optional disabled NQE map for data-file-driven device feature tag rules - Feature-flagged beta BGP and OSPF maps for optional `netbox-routing` and `netbox-peering-manager` deployments +- Beta device-lifecycle maps for optional `netbox-dlm` deployments: OS/hardware end-of-life dates and per-device running software from Forward's support analysis - Snapshot-aware execution with `latestProcessed` or an explicit Forward snapshot per sync - Ingestion records that preserve the selected snapshot mode, resolved snapshot ID, and Forward snapshot metrics - Built-in coverage for: diff --git a/docs/01_User_Guide/README.md b/docs/01_User_Guide/README.md index 53295870..42e89fc5 100644 --- a/docs/01_User_Guide/README.md +++ b/docs/01_User_Guide/README.md @@ -20,14 +20,14 @@ pip install forward-netbox Alternatively, install a specific wheel or source archive from GitHub Releases: ```bash -pip install /path/to/forward_netbox-2.4.5-py3-none-any.whl -pip install /path/to/forward_netbox-2.4.5.tar.gz +pip install /path/to/forward_netbox-2.5.0-py3-none-any.whl +pip install /path/to/forward_netbox-2.5.0.tar.gz ``` If you mirror the package into a private Python index, pin the same release version: ```bash -pip install forward-netbox==2.4.5 +pip install forward-netbox==2.5.0 ``` ## Release Compatibility @@ -39,7 +39,8 @@ Latest release requires NetBox `4.6.4` and `netbox-branching` `1.1.0+`. Expand f | Plugin Release | NetBox Version | Status | | --- | --- | --- | -| `v2.4.5` | `4.6.4` required (4.5.x dropped); needs netbox-branching `1.1.0` – `1.1.x` | Current release; Fix: sync no longer crashes on netbox-branching **1.1.1** (`SquashMergeStrategy has no attribute '_split_bidirectional_cycles'` — 1.1.1 removed that internal helper; the bidirectional-cycle split is now built into the plugin and the dependency is bounded to `<1.2`). Also fixes SNMP-endpoint rows failing validation: the bundled endpoint query branches now clamp sysDescr-derived `device_type` to NetBox's 100-char limit (`substring`) and guard empty slugs — the fix lives in the NQE queries (the source of truth), so **Publish Bundled Queries** again after upgrading (fixes the `Ensure this value has at most 100 characters` rejects and the `At least one coalesce lookup must be provided` error). | +| `v2.5.0` | `4.6.4` required (4.5.x dropped); needs netbox-branching `1.1.0` – `1.1.x` | Current release; Feature: optional **netbox-dlm** (Device Lifecycle Management) integration — three new opt-in NQE maps sync Forward's end-of-life analysis into the netbox-dlm plugin: OS software versions with vendor EOL dates per (platform, version), hardware end-of-life notices per device type (Cisco/Palo Alto/Fortinet part support), and each device's running software version. Disabled by default; requires the netbox-dlm plugin (run `makemigrations netbox_dlm && migrate` after installing it — it ships no migrations). Fix: syncs no longer crash mid-provision when an installed plugin's migrations were never applied (`relation ... does not exist`) — a preflight now fails in seconds with the app name and remedy, and a new **Database tables** Health check surfaces the gap before you sync. | +| `v2.4.5` | `4.6.4` required (4.5.x dropped); needs netbox-branching `1.1.0` – `1.1.x` | Superseded by `v2.5.0`; Fix: sync no longer crashes on netbox-branching **1.1.1** (`SquashMergeStrategy has no attribute '_split_bidirectional_cycles'` — 1.1.1 removed that internal helper; the bidirectional-cycle split is now built into the plugin and the dependency is bounded to `<1.2`). Also fixes SNMP-endpoint rows failing validation: the bundled endpoint query branches now clamp sysDescr-derived `device_type` to NetBox's 100-char limit (`substring`) and guard empty slugs — the fix lives in the NQE queries (the source of truth), so **Publish Bundled Queries** again after upgrading (fixes the `Ensure this value has at most 100 characters` rejects and the `At least one coalesce lookup must be provided` error). | | `v2.4.4` | `4.6.4` required (4.5.x dropped); needs netbox-branching `1.1.0+` | Superseded by `v2.4.5`; Fix: SNMP-endpoint import now works on **tag-scoped** syncs — the device-tag include scope silently excluded every endpoint, both query-side and in the plugin's local scope filter (whose scoped-device set was built from modeled devices only, so endpoint rows were always dropped; with prune enabled they would even be deleted). Endpoint import now ignores the include scope (exclude tags still apply) and endpoint names join the scoped set. Also fixes the merge-phase `Tag with this Name already exists` issues: a same-named/same-slug tag already on main is now treated as merged instead of failing the branch's tag create. | | `v2.4.3` | `4.6.4` required (4.5.x dropped); needs netbox-branching `1.1.0+` | Superseded by `v2.4.4`; Fix: the pinned-query opt-in Health warning (2.4.1) over-claimed failure and fired even after the query was fixed (the Health page can't read a pinned query's contents). Reworded to a "Pinned — can't verify locally" heads-up pointing at Export Live Query Drift to confirm. No behavior change. | | `v2.4.2` | `4.6.4` required (4.5.x dropped); needs netbox-branching `1.1.0+` | Superseded by `v2.4.3`; Fix: endpoint import (`sync_endpoints`) and device-tag sync (`sync_device_tags`) now work with the alias-aware and rules-aware query variants (`forward_devices_with_netbox_aliases`, `forward_device_feature_tags_with_rules`), not just the base queries — operators running the variants saw the toggles silently do nothing. Adds a **Publish Bundled Queries** button on the sync Health page and two Health warnings: when an opt-in feature is enabled but no enabled map provides it, and when a base query and its opt-in variant are both enabled (duplicate-emission churn). The alias-aware device query now emits the clean role name (e.g. `ROUTER`) to match the base query — expect a one-time role update on alias-mapped devices. | diff --git a/docs/02_Reference/apply-engine-model-matrix.md b/docs/02_Reference/apply-engine-model-matrix.md index 57e9e790..5d2d589f 100644 --- a/docs/02_Reference/apply-engine-model-matrix.md +++ b/docs/02_Reference/apply-engine-model-matrix.md @@ -51,6 +51,9 @@ These models remain adapter-required with explicit blocker codes: - `netbox_cisco_aci.acitenant` | `plugin_model_dependencies` - `netbox_cisco_aci.acivrf` | `plugin_model_dependencies` - `netbox_peering_manager.peeringsession` | `plugin_model_dependencies` +- `netbox_dlm.devicesoftware` | `plugin_model_dependencies` +- `netbox_dlm.hardwarenotice` | `plugin_model_dependencies` +- `netbox_dlm.softwareversion` | `plugin_model_dependencies` - `netbox_routing.bgpaddressfamily` | `plugin_model_dependencies` - `netbox_routing.bgppeer` | `plugin_model_dependencies` - `netbox_routing.bgppeeraddressfamily` | `plugin_model_dependencies` diff --git a/docs/03_Plans/active/2026-07-10-dlm-support-and-provision-preflight.md b/docs/03_Plans/active/2026-07-10-dlm-support-and-provision-preflight.md new file mode 100644 index 00000000..43f3d928 --- /dev/null +++ b/docs/03_Plans/active/2026-07-10-dlm-support-and-provision-preflight.md @@ -0,0 +1,85 @@ +# Feature: optional netbox-dlm integration + provision table preflight (2.5.0) + +## Goal + +Stop syncs from crashing when an installed plugin's migrations were never +applied (field report: `relation "public.netbox_dlm_contract" does not exist` +mid-provision), and add an optional integration for the netbox-dlm Device +Lifecycle Management plugin backed by Forward's end-of-life analysis. + +## Constraints + +- netbox-dlm is NOT installed in CI or the dev runtime; every surface must + degrade gracefully when the plugin is absent (existing optional-plugin + pattern) and the new maps ship disabled. +- New model strings require a ForwardNQEMap choices migration (0032) — the + only schema change; no data migrations. +- NQE stays the source of truth: field-length clamping and identity shaping + live in the bundled queries, not the plugin. +- Never commit the ADP network id / token. + +## Touched Surfaces + +- `forward_netbox/utilities/branching.py` (`missing_branch_table_report`), + `single_branch_executor.py` (preflight), `health.py` (Database tables check) +- `forward_netbox/utilities/sync_dlm.py` (new adapter), + `sync_runner_adapters.py` + `sync.py` (dispatch wiring) +- `forward_netbox/choices.py` (+ migration 0032), `query_registry.py`, + `plugin_integrations/registry.py`, `apply_engine_decision.py`, + `branch_budget.py`, `sync_contracts.py`, `sync_primitives.py`, + `query_fetch_execution.py`, `sync_reporting.py` +- New queries: `forward_dlm_software_versions.nqe`, + `forward_dlm_hardware_notices.nqe`, `forward_dlm_device_software.nqe` +- Tests: `test_provision_preflight.py`, `test_dlm_integration.py` + +## Approach + +1. Preflight: Django registers ContentTypes for every installed app even when + its migrations never ran, so netbox_branching's provision runs + `CREATE TABLE branch.T (LIKE public.T)` for a missing table and the sync + dies with an opaque ProgrammingError. `missing_branch_table_report()` diffs + `get_tables_to_replicate()` against `connection.introspection.table_names()` + (the exact list provision replicates), maps missing tables to app labels, + and the executor raises an actionable SyncError before the fetch. A + fail-severity **Database tables** Health check surfaces the same report. +2. netbox-dlm integration (three opt-in maps, following the + netbox_routing/ACI optional-plugin pattern end to end): SoftwareVersion + from `device.platform.osSupport` per (platform, osVersion); HardwareNotice + from chassis `device.platform.components[].support` per device type; + DeviceSoftware one row per device (the adapter ensures the referenced + SoftwareVersion exists without overwriting announced dates). Registry + integration (`DLM_INTEGRATION`, adapter contract on + `forward_netbox.utilities.sync_dlm`), decision/budget/contract/identity + tables, and runner dispatch are wired for all three model strings. +3. Fixed a latent import-order bug the new models exposed: + `_fallback_bucket_key_family` was defined after the module-level shard + contract build in `branch_budget.py` and NameError'd for any supported + model without a structured shard contract. + +## Validation + +Live against the ADP demo network: 35 software-version rows (ISO dates, +announcement URLs), 48 hardware-notice rows, 4960 device-software rows; all +three bundled queries lint clean and run via raw NQE. Unit tests cover the +preflight (report, executor SyncError, Health check pass/fail/soft-fail) and +the DLM wiring (choices/registry/maps disabled-by-default/adapter contract/ +runner dispatch/query structure). Full suite + lint + harness green. + +## Rollback + +Revert the branch and roll back migration 0032 (choices-only alter). No data +migrations; the new maps are seeded disabled and can be deleted. + +## Decision Log + +- Ship the DLM maps disabled: the plugin is alpha (0.1.0), ships no + migrations, and the integration is a beta surface — operators opt in per map. +- DeviceSoftware ensures a bare SoftwareVersion when the vendor has no EOL + announcement (create-only, never overwriting dates from the versions map), + so every device links even when Forward lacks announcement data. +- Forward exposes no end-of-sale date, so HardwareNotice.end_of_sale stays + null; end_of_support/lastSupportDate, end_of_security_patches/ + lastVulnerabilityDate, end_of_sw_releases/lastMaintenanceDate. +- The preflight lives in the executor (not provision monkeypatching) and the + Health check never raises — diagnostics failures return None rather than + breaking the page. diff --git a/docs/README.md b/docs/README.md index f9d1c4d8..e4017a61 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,7 +13,8 @@ Latest release requires NetBox `4.6.4` and `netbox-branching` `1.1.0+`. Expand f | Plugin Release | NetBox Version | Status | | --- | --- | --- | -| `v2.4.5` | `4.6.4` required; needs netbox-branching `1.1.0` – `1.1.x` | Current release; Fix: sync no longer crashes on netbox-branching **1.1.1** (`SquashMergeStrategy has no attribute '_split_bidirectional_cycles'` — 1.1.1 removed that internal helper; the bidirectional-cycle split is now built into the plugin and the dependency is bounded to `<1.2`). Also fixes SNMP-endpoint rows failing validation: the bundled endpoint query branches now clamp sysDescr-derived `device_type` to NetBox's 100-char limit (`substring`) and guard empty slugs — the fix lives in the NQE queries (the source of truth), so **Publish Bundled Queries** again after upgrading (fixes the `Ensure this value has at most 100 characters` rejects and the `At least one coalesce lookup must be provided` error). | +| `v2.5.0` | `4.6.4` required; needs netbox-branching `1.1.0` – `1.1.x` | Current release; Feature: optional **netbox-dlm** (Device Lifecycle Management) integration — three new opt-in NQE maps sync Forward's end-of-life analysis into the netbox-dlm plugin: OS software versions with vendor EOL dates per (platform, version), hardware end-of-life notices per device type (Cisco/Palo Alto/Fortinet part support), and each device's running software version. Disabled by default; requires the netbox-dlm plugin (run `makemigrations netbox_dlm && migrate` after installing it — it ships no migrations). Fix: syncs no longer crash mid-provision when an installed plugin's migrations were never applied (`relation ... does not exist`) — a preflight now fails in seconds with the app name and remedy, and a new **Database tables** Health check surfaces the gap before you sync. | +| `v2.4.5` | `4.6.4` required; needs netbox-branching `1.1.0` – `1.1.x` | Superseded by `v2.5.0`; Fix: sync no longer crashes on netbox-branching **1.1.1** (`SquashMergeStrategy has no attribute '_split_bidirectional_cycles'` — 1.1.1 removed that internal helper; the bidirectional-cycle split is now built into the plugin and the dependency is bounded to `<1.2`). Also fixes SNMP-endpoint rows failing validation: the bundled endpoint query branches now clamp sysDescr-derived `device_type` to NetBox's 100-char limit (`substring`) and guard empty slugs — the fix lives in the NQE queries (the source of truth), so **Publish Bundled Queries** again after upgrading (fixes the `Ensure this value has at most 100 characters` rejects and the `At least one coalesce lookup must be provided` error). | | `v2.4.4` | `4.6.4` required; needs netbox-branching `1.1.0+` | Superseded by `v2.4.5`; Fix: SNMP-endpoint import now works on **tag-scoped** syncs — the device-tag include scope silently excluded every endpoint, both query-side and in the plugin's local scope filter (whose scoped-device set was built from modeled devices only, so endpoint rows were always dropped; with prune enabled they would even be deleted). Endpoint import now ignores the include scope (exclude tags still apply) and endpoint names join the scoped set. Also fixes the merge-phase `Tag with this Name already exists` issues: a same-named/same-slug tag already on main is now treated as merged instead of failing the branch's tag create. | | `v2.4.3` | `4.6.4` required; needs netbox-branching `1.1.0+` | Superseded by `v2.4.4`; Fix: the pinned-query opt-in Health warning (2.4.1) over-claimed failure and fired even after the query was fixed (the Health page can't read a pinned query's contents). Reworded to a "Pinned — can't verify locally" heads-up pointing at Export Live Query Drift to confirm. No behavior change. | | `v2.4.2` | `4.6.4` required; needs netbox-branching `1.1.0+` | Superseded by `v2.4.3`; Fix: endpoint import (`sync_endpoints`) and device-tag sync (`sync_device_tags`) now work with the alias-aware and rules-aware query variants (`forward_devices_with_netbox_aliases`, `forward_device_feature_tags_with_rules`), not just the base queries — operators running the variants saw the toggles silently do nothing. Adds a **Publish Bundled Queries** button on the sync Health page and two Health warnings: when an opt-in feature is enabled but no enabled map provides it, and when a base query and its opt-in variant are both enabled (duplicate-emission churn). The alias-aware device query now emits the clean role name (e.g. `ROUTER`) to match the base query — expect a one-time role update on alias-mapped devices. | diff --git a/forward_netbox/__init__.py b/forward_netbox/__init__.py index fd2479dc..07ce1c32 100644 --- a/forward_netbox/__init__.py +++ b/forward_netbox/__init__.py @@ -8,7 +8,7 @@ class NetboxForwardConfig(PluginConfig): name = "forward_netbox" verbose_name = "Forward" description = "Sync Forward data into NetBox using built-in NQE queries." - version = "2.4.5" + version = "2.5.0" base_url = "forward" min_version = "4.6.4" diff --git a/forward_netbox/choices.py b/forward_netbox/choices.py index 9988efcb..7e6f1ef7 100644 --- a/forward_netbox/choices.py +++ b/forward_netbox/choices.py @@ -13,6 +13,12 @@ "netbox_peering_manager.peeringsession", ) +FORWARD_DLM_MODELS = ( + "netbox_dlm.softwareversion", + "netbox_dlm.hardwarenotice", + "netbox_dlm.devicesoftware", +) + FORWARD_ACI_MODELS = ( "netbox_cisco_aci.acifabric", "netbox_cisco_aci.acipod", @@ -49,6 +55,7 @@ "dcim.module", *FORWARD_BGP_MODELS, *FORWARD_ACI_MODELS, + *FORWARD_DLM_MODELS, ) FORWARD_OPTIONAL_MODELS = { @@ -56,6 +63,7 @@ "dcim.module", *FORWARD_BGP_MODELS, *FORWARD_ACI_MODELS, + *FORWARD_DLM_MODELS, } diff --git a/forward_netbox/migrations/0032_alter_forwardnqemap_netbox_model.py b/forward_netbox/migrations/0032_alter_forwardnqemap_netbox_model.py new file mode 100644 index 00000000..d49e4b86 --- /dev/null +++ b/forward_netbox/migrations/0032_alter_forwardnqemap_netbox_model.py @@ -0,0 +1,90 @@ +# Generated by Django 6.0.6 on 2026-07-09 21:57 +import django.db.models.deletion +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("contenttypes", "0002_remove_content_type_name"), + ("forward_netbox", "0031_encrypt_source_password"), + ] + + operations = [ + migrations.AlterField( + model_name="forwardnqemap", + name="netbox_model", + field=models.ForeignKey( + limit_choices_to=models.Q( + models.Q(("app_label", "dcim"), ("model", "site")), + models.Q(("app_label", "dcim"), ("model", "manufacturer")), + models.Q(("app_label", "dcim"), ("model", "devicerole")), + models.Q(("app_label", "dcim"), ("model", "platform")), + models.Q(("app_label", "dcim"), ("model", "devicetype")), + models.Q(("app_label", "dcim"), ("model", "device")), + models.Q(("app_label", "dcim"), ("model", "virtualchassis")), + models.Q(("app_label", "extras"), ("model", "taggeditem")), + models.Q(("app_label", "dcim"), ("model", "interface")), + models.Q(("app_label", "dcim"), ("model", "cable")), + models.Q(("app_label", "dcim"), ("model", "macaddress")), + models.Q(("app_label", "ipam"), ("model", "vlan")), + models.Q(("app_label", "ipam"), ("model", "vrf")), + models.Q(("app_label", "ipam"), ("model", "prefix")), + models.Q(("app_label", "ipam"), ("model", "ipaddress")), + models.Q(("app_label", "ipam"), ("model", "fhrpgroup")), + models.Q(("app_label", "dcim"), ("model", "inventoryitem")), + models.Q(("app_label", "dcim"), ("model", "module")), + models.Q(("app_label", "netbox_routing"), ("model", "bgppeer")), + models.Q( + ("app_label", "netbox_routing"), ("model", "bgpaddressfamily") + ), + models.Q( + ("app_label", "netbox_routing"), + ("model", "bgppeeraddressfamily"), + ), + models.Q( + ("app_label", "netbox_routing"), ("model", "ospfinstance") + ), + models.Q(("app_label", "netbox_routing"), ("model", "ospfarea")), + models.Q( + ("app_label", "netbox_routing"), ("model", "ospfinterface") + ), + models.Q( + ("app_label", "netbox_peering_manager"), + ("model", "peeringsession"), + ), + models.Q(("app_label", "netbox_cisco_aci"), ("model", "acifabric")), + models.Q(("app_label", "netbox_cisco_aci"), ("model", "acipod")), + models.Q(("app_label", "netbox_cisco_aci"), ("model", "acinode")), + models.Q(("app_label", "netbox_cisco_aci"), ("model", "acitenant")), + models.Q(("app_label", "netbox_cisco_aci"), ("model", "acivrf")), + models.Q( + ("app_label", "netbox_cisco_aci"), ("model", "acibridgedomain") + ), + models.Q( + ("app_label", "netbox_cisco_aci"), ("model", "aciappprofile") + ), + models.Q( + ("app_label", "netbox_cisco_aci"), ("model", "aciendpointgroup") + ), + models.Q( + ("app_label", "netbox_cisco_aci"), ("model", "acicontract") + ), + models.Q(("app_label", "netbox_cisco_aci"), ("model", "acifilter")), + models.Q(("app_label", "netbox_cisco_aci"), ("model", "acil3out")), + models.Q( + ("app_label", "netbox_cisco_aci"), + ("model", "acistaticportbinding"), + ), + models.Q(("app_label", "netbox_dlm"), ("model", "softwareversion")), + models.Q(("app_label", "netbox_dlm"), ("model", "hardwarenotice")), + models.Q(("app_label", "netbox_dlm"), ("model", "devicesoftware")), + _connector="OR", + ), + on_delete=django.db.models.deletion.PROTECT, + related_name="+", + to="contenttypes.contenttype", + ), + ), + ] diff --git a/forward_netbox/queries/forward_dlm_device_software.nqe b/forward_netbox/queries/forward_dlm_device_software.nqe new file mode 100644 index 00000000..f8395703 --- /dev/null +++ b/forward_netbox/queries/forward_dlm_device_software.nqe @@ -0,0 +1,25 @@ +/** + * @intent Forward per-device running software version (netbox-dlm) + * @description The OS version each Forward-collected device is running, for + * the optional netbox-dlm plugin's DeviceSoftware model (one row per device). + * The apply adapter ensures the matching SoftwareVersion exists (end-of-life + * dates come from the Forward DLM Software Versions map when the vendor + * publishes them). + */ + +import "netbox_utilities"; + +@query +f(forward_netbox_shard_keys: List) = +foreach device in network.devices +where device.snapshotInfo.result == DeviceSnapshotResult.completed +where device.platform.vendor != Vendor.FORWARD_CUSTOM +where isEmpty(forward_netbox_shard_keys) || device.name in forward_netbox_shard_keys +where isPresent(device.platform.osVersion) +let platform_name = normalizeDevicePlatformName(device) +select { + name: device.name, + platform: platform_name, + platform_slug: slugify(platform_name), + version: substring(device.platform.osVersion, 0, 50) +}; diff --git a/forward_netbox/queries/forward_dlm_hardware_notices.nqe b/forward_netbox/queries/forward_dlm_hardware_notices.nqe new file mode 100644 index 00000000..7e00c163 --- /dev/null +++ b/forward_netbox/queries/forward_dlm_hardware_notices.nqe @@ -0,0 +1,28 @@ +/** + * @intent Forward hardware end-of-life notices per device type (netbox-dlm) + * @description Distinct device models with vendor hardware end-of-life dates + * from Forward's per-part support analysis (chassis parts), for the optional + * netbox-dlm plugin's HardwareNotice model. Part support data is only present + * for vendors Forward tracks (Cisco, Palo Alto Networks, Fortinet). + */ + +import "netbox_utilities"; + +@query +f(forward_netbox_shard_keys: List) = +foreach device in network.devices +where device.snapshotInfo.result == DeviceSnapshotResult.completed +where device.platform.vendor != Vendor.FORWARD_CUSTOM +where isEmpty(forward_netbox_shard_keys) || device.name in forward_netbox_shard_keys +let model = device.platform.model +foreach part in device.platform.components +where part.partType == DevicePartType.CHASSIS +where isPresent(part.support) +select distinct { + device_type: model, + device_type_slug: slugifyNetboxModel(model), + end_of_support: part.support.lastSupportDate, + end_of_security_patches: part.support.lastVulnerabilityDate, + end_of_sw_releases: part.support.lastMaintenanceDate, + documentation_url: substring(part.support.announcementUrl, 0, 200) +}; diff --git a/forward_netbox/queries/forward_dlm_software_versions.nqe b/forward_netbox/queries/forward_dlm_software_versions.nqe new file mode 100644 index 00000000..ef92ec29 --- /dev/null +++ b/forward_netbox/queries/forward_dlm_software_versions.nqe @@ -0,0 +1,28 @@ +/** + * @intent Forward OS software versions with end-of-life dates (netbox-dlm) + * @description Distinct (platform, OS version) pairs with vendor end-of-life + * dates from Forward's OS support analysis, for the optional netbox-dlm + * plugin's SoftwareVersion model. Only devices whose OS has a known vendor + * end-of-life announcement are emitted (device.platform.osSupport is absent + * otherwise). + */ + +import "netbox_utilities"; + +@query +f(forward_netbox_shard_keys: List) = +foreach device in network.devices +where device.snapshotInfo.result == DeviceSnapshotResult.completed +where device.platform.vendor != Vendor.FORWARD_CUSTOM +where isEmpty(forward_netbox_shard_keys) || device.name in forward_netbox_shard_keys +where isPresent(device.platform.osSupport) +where isPresent(device.platform.osVersion) +let support = device.platform.osSupport +let platform_name = normalizeDevicePlatformName(device) +select distinct { + platform: platform_name, + platform_slug: slugify(platform_name), + version: substring(device.platform.osVersion, 0, 50), + end_of_support: support.lastSupportDate, + documentation_url: substring(support.announcementUrl, 0, 200) +}; diff --git a/forward_netbox/tests/test_dlm_integration.py b/forward_netbox/tests/test_dlm_integration.py new file mode 100644 index 00000000..3c84dd00 --- /dev/null +++ b/forward_netbox/tests/test_dlm_integration.py @@ -0,0 +1,106 @@ +# Optional netbox-dlm (Device Lifecycle Management) integration wiring. +# +# The plugin is NOT installed in CI; these tests cover the registry surface, +# bundled queries, and adapter contract so the integration is exercised without +# it. Field origin: the design partner authoring netbox-dlm runs this plugin. +import importlib + +from django.test import SimpleTestCase + +from forward_netbox.choices import FORWARD_DLM_MODELS +from forward_netbox.choices import FORWARD_OPTIONAL_MODELS +from forward_netbox.choices import FORWARD_SUPPORTED_MODELS +from forward_netbox.utilities.plugin_integrations.registry import DLM_INTEGRATION +from forward_netbox.utilities.plugin_integrations.registry import ( + optional_integration_for_model, +) +from forward_netbox.utilities.query_registry import _default_query_parameters +from forward_netbox.utilities.query_registry import _read_query +from forward_netbox.utilities.query_registry import builtin_nqe_map_rows + +DLM_MODEL_STRINGS = ( + "netbox_dlm.softwareversion", + "netbox_dlm.hardwarenotice", + "netbox_dlm.devicesoftware", +) + + +class DlmRegistryWiringTest(SimpleTestCase): + def test_models_registered_as_supported_and_optional(self): + for model_string in DLM_MODEL_STRINGS: + self.assertIn(model_string, FORWARD_DLM_MODELS) + self.assertIn(model_string, FORWARD_SUPPORTED_MODELS) + self.assertIn(model_string, FORWARD_OPTIONAL_MODELS) + + def test_integration_matches_models(self): + for model_string in DLM_MODEL_STRINGS: + integration = optional_integration_for_model(model_string) + self.assertIsNotNone(integration, model_string) + self.assertEqual(integration.app_label, "netbox_dlm") + + def test_builtin_maps_seeded_disabled_by_default(self): + rows = {row["name"]: row for row in builtin_nqe_map_rows()} + for name in DLM_INTEGRATION.query_maps: + self.assertIn(name, rows) + self.assertFalse( + rows[name].get("enabled", True), + f"{name} must ship disabled (opt-in plugin surface)", + ) + + def test_adapter_module_exposes_apply_and_delete_per_model(self): + adapter = importlib.import_module(DLM_INTEGRATION.adapter_module) + for model_string in DLM_INTEGRATION.supported_models: + slug = model_string.replace(".", "_") + self.assertTrue(callable(getattr(adapter, f"apply_{slug}", None)), slug) + self.assertTrue(callable(getattr(adapter, f"delete_{slug}", None)), slug) + + +class DlmQueryStructureTest(SimpleTestCase): + def test_software_versions_query_shape(self): + src = _read_query("forward_dlm_software_versions.nqe") + self.assertIn("device.platform.osSupport", src) + self.assertIn("lastSupportDate", src) + # dlm SoftwareVersion.version is Char(50); announcement URL is URLField. + self.assertIn("substring(device.platform.osVersion, 0, 50)", src) + self.assertIn("substring(support.announcementUrl, 0, 200)", src) + self.assertIn("select distinct", src) + + def test_hardware_notices_query_shape(self): + src = _read_query("forward_dlm_hardware_notices.nqe") + self.assertIn("DevicePartType.CHASSIS", src) + for field in ( + "end_of_support:", + "end_of_security_patches:", + "end_of_sw_releases:", + "device_type_slug:", + ): + self.assertIn(field, src) + + def test_device_software_query_shape(self): + src = _read_query("forward_dlm_device_software.nqe") + self.assertIn("device.platform.osVersion", src) + for field in ("name:", "platform_slug:", "version:"): + self.assertIn(field, src) + + def test_queries_seed_shard_parameter(self): + for filename in ( + "forward_dlm_software_versions.nqe", + "forward_dlm_hardware_notices.nqe", + "forward_dlm_device_software.nqe", + ): + params = _default_query_parameters(filename) + self.assertIn("forward_netbox_shard_keys", params, filename) + + +class DlmRunnerDispatchTest(SimpleTestCase): + def test_runner_has_apply_and_delete_methods(self): + from forward_netbox.utilities.sync import ForwardSyncRunner + + for model_string in DLM_MODEL_STRINGS: + slug = model_string.replace(".", "_") + self.assertTrue( + callable(getattr(ForwardSyncRunner, f"_apply_{slug}", None)), slug + ) + self.assertTrue( + callable(getattr(ForwardSyncRunner, f"_delete_{slug}", None)), slug + ) diff --git a/forward_netbox/tests/test_plugin_integrations.py b/forward_netbox/tests/test_plugin_integrations.py index 51b79f8a..6be68ede 100644 --- a/forward_netbox/tests/test_plugin_integrations.py +++ b/forward_netbox/tests/test_plugin_integrations.py @@ -19,6 +19,7 @@ def test_registry_exposes_generic_optional_integrations(self): "routing.netbox_routing", "peering.netbox_peering_manager", "aci.netbox_cisco_aci", + "lifecycle.netbox_dlm", ], ) self.assertEqual( @@ -42,6 +43,7 @@ def test_registry_exposes_generic_optional_integrations(self): "routing.netbox_routing", "peering.netbox_peering_manager", "aci.netbox_cisco_aci", + "lifecycle.netbox_dlm", }, ) self.assertEqual( @@ -60,6 +62,7 @@ def fake_version(package_name): "netbox-routing": "1.0.0", "netbox-peering-manager": "1.0.0", "netbox-cisco-aci": "0.2.1", + "netbox-dlm": "0.1.0", } return versions[package_name] diff --git a/forward_netbox/tests/test_provision_preflight.py b/forward_netbox/tests/test_provision_preflight.py new file mode 100644 index 00000000..0de65a6e --- /dev/null +++ b/forward_netbox/tests/test_provision_preflight.py @@ -0,0 +1,74 @@ +# A plugin in PLUGINS whose migrations were never applied leaves ContentTypes +# registered but tables missing; branch provisioning then dies mid-CREATE TABLE +# with an opaque ProgrammingError (field report: netbox_dlm_contract). The +# preflight must catch it before provisioning, with the remedy in the message. +from unittest.mock import Mock +from unittest.mock import patch + +from core.exceptions import SyncError +from django.test import TestCase + +from forward_netbox.utilities.branching import missing_branch_table_report +from forward_netbox.utilities.health import _database_tables_check +from forward_netbox.utilities.single_branch_executor import ( + ForwardSingleBranchExecutor, +) + + +class MissingBranchTableReportTest(TestCase): + def test_healthy_database_reports_nothing(self): + self.assertEqual(missing_branch_table_report(), {}) + + def test_unmigrated_table_is_reported(self): + real_tables = ["dcim_device", "netbox_dlm_contract"] + with patch( + "netbox_branching.utilities.get_tables_to_replicate", + return_value=real_tables, + ): + report = missing_branch_table_report() + # dcim_device exists; the dlm table does not (no model registered for + # it in this environment either, so it lands in the unknown bucket). + self.assertEqual(report, {"unknown": ["netbox_dlm_contract"]}) + + +class ExecutorTablePreflightTest(TestCase): + def test_run_raises_actionable_sync_error_before_fetch(self): + executor = object.__new__(ForwardSingleBranchExecutor) + with ( + patch.object(ForwardSingleBranchExecutor, "_set_runtime_phase", Mock()), + patch( + "forward_netbox.utilities.single_branch_executor." + "missing_branch_table_report", + return_value={"netbox_dlm": ["netbox_dlm_contract"]}, + ), + ): + with self.assertRaises(SyncError) as ctx: + executor.run() + message = str(ctx.exception) + self.assertIn("netbox_dlm", message) + self.assertIn("netbox_dlm_contract", message) + self.assertIn("migrate", message) + + +class DatabaseTablesHealthCheckTest(TestCase): + def test_healthy_database_passes(self): + check = _database_tables_check() + self.assertIsNotNone(check) + self.assertEqual(check["status"], "pass") + + def test_missing_tables_fail_with_remedy(self): + with patch( + "forward_netbox.utilities.health.missing_branch_table_report", + return_value={"netbox_dlm": ["netbox_dlm_contract"]}, + ): + check = _database_tables_check() + self.assertEqual(check["status"], "fail") + self.assertIn("netbox_dlm_contract", check["message"]) + self.assertIn("migrate", check["message"]) + + def test_diagnostics_failure_returns_none_not_crash(self): + with patch( + "forward_netbox.utilities.health.missing_branch_table_report", + side_effect=RuntimeError("introspection unavailable"), + ): + self.assertIsNone(_database_tables_check()) diff --git a/forward_netbox/utilities/apply_engine_decision.py b/forward_netbox/utilities/apply_engine_decision.py index 157dacf1..d69d79c2 100644 --- a/forward_netbox/utilities/apply_engine_decision.py +++ b/forward_netbox/utilities/apply_engine_decision.py @@ -262,9 +262,33 @@ "netbox_cisco_aci.acifilter", "netbox_cisco_aci.acil3out", "netbox_cisco_aci.acistaticportbinding", + "netbox_dlm.softwareversion", + "netbox_dlm.hardwarenotice", + "netbox_dlm.devicesoftware", } ADAPTER_MODEL_BLOCKERS = { + "netbox_dlm.softwareversion": { + "blocker_code": "plugin_model_dependencies", + "blocker_reason": ( + "DLM software-version writes require plugin model and platform " + "dependency resolution handled by the adapter." + ), + }, + "netbox_dlm.hardwarenotice": { + "blocker_code": "plugin_model_dependencies", + "blocker_reason": ( + "DLM hardware-notice writes require plugin model and device-type " + "dependency resolution handled by the adapter." + ), + }, + "netbox_dlm.devicesoftware": { + "blocker_code": "plugin_model_dependencies", + "blocker_reason": ( + "DLM device-software writes require device and software-version " + "dependency resolution handled by the adapter." + ), + }, "dcim.cable": { "blocker_code": "relationship_identity_directionality", "blocker_reason": ( diff --git a/forward_netbox/utilities/branch_budget.py b/forward_netbox/utilities/branch_budget.py index 75aec13a..ee428219 100644 --- a/forward_netbox/utilities/branch_budget.py +++ b/forward_netbox/utilities/branch_budget.py @@ -21,6 +21,9 @@ DEFAULT_MODEL_CHANGE_DENSITY = { "dcim.cable": 3.0, "dcim.module": 2.0, + "netbox_dlm.softwareversion": 1.0, + "netbox_dlm.hardwarenotice": 1.0, + "netbox_dlm.devicesoftware": 1.0, "netbox_routing.bgppeer": 5.0, "netbox_routing.bgpaddressfamily": 2.0, "netbox_routing.bgppeeraddressfamily": 2.0, @@ -70,6 +73,9 @@ "netbox_routing.bgpaddressfamily", "netbox_routing.bgppeeraddressfamily", "netbox_peering_manager.peeringsession", + "netbox_dlm.softwareversion", + "netbox_dlm.hardwarenotice", + "netbox_dlm.devicesoftware", "netbox_cisco_aci.acifabric", "netbox_cisco_aci.acipod", "netbox_cisco_aci.acinode", @@ -110,6 +116,9 @@ "dcim.interface", "netbox_routing.ospfinstance", ), + "netbox_dlm.softwareversion": ("dcim.platform",), + "netbox_dlm.hardwarenotice": ("dcim.devicetype",), + "netbox_dlm.devicesoftware": ("dcim.device", "netbox_dlm.softwareversion"), "netbox_routing.bgppeer": ("dcim.device", "ipam.vrf"), "netbox_routing.bgpaddressfamily": ("dcim.device", "ipam.vrf"), "netbox_routing.bgppeeraddressfamily": ( @@ -141,6 +150,9 @@ ), } DELETE_DEPENDENCY_MODEL_ORDER = ( + "netbox_dlm.devicesoftware", + "netbox_dlm.hardwarenotice", + "netbox_dlm.softwareversion", "dcim.cable", "ipam.fhrpgroup", "ipam.ipaddress", @@ -312,6 +324,16 @@ def apply_dependency_dry_run(enabled_models) -> dict: SHARD_FETCH_PARAMETER_BUCKET_COUNT = "forward_netbox_shard_bucket_count" +def _fallback_bucket_key_family(model_string): + field_sets = default_coalesce_fields_for_model(model_string) + if not field_sets: + return "" + first = field_sets[0] + if not first: + return "" + return ",".join(str(field_name) for field_name in first if str(field_name)) + + def _model_fetch_fallback_contract(model_string): bucket_key_family = _fallback_bucket_key_family(model_string) return { @@ -689,16 +711,6 @@ def _structured_parameter_contract(model_string, shard_keys): return {} -def _fallback_bucket_key_family(model_string): - field_sets = default_coalesce_fields_for_model(model_string) - if not field_sets: - return "" - first = field_sets[0] - if not first: - return "" - return ",".join(str(field_name) for field_name in first if str(field_name)) - - def _parse_structured_shard_key(shard_key): parsed = {} for part in str(shard_key).split("|"): diff --git a/forward_netbox/utilities/branching.py b/forward_netbox/utilities/branching.py index 2204bb43..9acdc681 100644 --- a/forward_netbox/utilities/branching.py +++ b/forward_netbox/utilities/branching.py @@ -3,6 +3,36 @@ from utilities.request import NetBoxFakeRequest +def missing_branch_table_report(): + """Map app_label -> branch-replicated tables missing from the database. + + Django registers ContentTypes for every installed app even when that app's + migrations were never applied, so netbox_branching's provision happily runs + ``CREATE TABLE branch.T (LIKE public.T)`` for a table that does not exist + and the sync dies mid-provision with an opaque ProgrammingError (field + report: a plugin installed without running its migrations). Detect the gap + up front using the exact table list provision will replicate. + """ + from django.apps import apps + from django.db import connection + from netbox_branching.utilities import get_tables_to_replicate + + missing = set(get_tables_to_replicate()) - set( + connection.introspection.table_names() + ) + if not missing: + return {} + by_app: dict[str, set[str]] = {} + for model in apps.get_models(include_auto_created=True): + table = model._meta.db_table + if table in missing: + by_app.setdefault(model._meta.app_label, set()).add(table) + mapped = set().union(*by_app.values()) if by_app else set() + for table in missing - mapped: + by_app.setdefault("unknown", set()).add(table) + return {app: sorted(tables) for app, tables in sorted(by_app.items())} + + def build_branch_name(*, sync, ingestion, item): sync_id = getattr(sync, "pk", None) ingestion_id = getattr(ingestion, "pk", None) diff --git a/forward_netbox/utilities/health.py b/forward_netbox/utilities/health.py index a50f34e4..97a9c414 100644 --- a/forward_netbox/utilities/health.py +++ b/forward_netbox/utilities/health.py @@ -1,5 +1,6 @@ from collections import Counter +from .branching import missing_branch_table_report from .execution_ledger import api_usage_support_summary from .execution_ledger import dependency_lookup_cache_support_summary from .execution_ledger import ( @@ -271,6 +272,35 @@ def _base_variant_conflict_check(sync): ) +def _database_tables_check(): + """Fail loudly when an installed app's tables were never migrated. + + Branch provisioning replicates every branchable table with CREATE TABLE + LIKE; a plugin in PLUGINS whose migrations were never applied kills the + sync mid-provision. Surface it on the Health page with the remedy. + """ + try: + report = missing_branch_table_report() + except Exception: # noqa: BLE001 - diagnostics must never break Health + return None + if not report: + return _check( + name="Database tables", + status="pass", + message="Every branch-replicated table exists in the database.", + ) + detail = "; ".join(f"{app}: {', '.join(tables)}" for app, tables in report.items()) + return _check( + name="Database tables", + status="fail", + message=( + f"Installed apps have missing database tables ({detail}) — branch " + "provisioning will fail. Apply their migrations " + "(python manage.py migrate) or remove the plugin from PLUGINS." + ), + ) + + def sync_health_summary(sync): optional_plugin_capabilities = integration_capability_summary() maps = [ @@ -364,6 +394,9 @@ def sync_health_summary(sync): variant_conflict_check = _base_variant_conflict_check(sync) if variant_conflict_check is not None: checks.append(variant_conflict_check) + database_tables_check = _database_tables_check() + if database_tables_check is not None: + checks.append(database_tables_check) return { "source": _source_summary(sync), diff --git a/forward_netbox/utilities/plugin_integrations/registry.py b/forward_netbox/utilities/plugin_integrations/registry.py index 70939547..9ba89fd6 100644 --- a/forward_netbox/utilities/plugin_integrations/registry.py +++ b/forward_netbox/utilities/plugin_integrations/registry.py @@ -297,10 +297,45 @@ def as_dict(self): ) +DLM_INTEGRATION = OptionalPluginIntegration( + key="lifecycle.netbox_dlm", + app_label="netbox_dlm", + display_name="NetBox Device Lifecycle Management", + required_models=( + "netbox_dlm.softwareversion", + "netbox_dlm.hardwarenotice", + "netbox_dlm.devicesoftware", + ), + supported_models=( + "netbox_dlm.softwareversion", + "netbox_dlm.hardwarenotice", + "netbox_dlm.devicesoftware", + ), + query_maps=( + "Forward DLM Software Versions", + "Forward DLM Hardware Notices", + "Forward DLM Device Software", + ), + package_names=("netbox-dlm", "netbox_dlm"), + adapter_module="forward_netbox.utilities.sync_dlm", + enabled_by_default=False, + status="beta_surface", + notes=( + "Beta lifecycle import surface backed by the optional netbox-dlm " + "plugin: OS end-of-life dates per (platform, version), hardware " + "end-of-life notices per device type, and each device's running " + "software version from Forward's support analysis.", + "netbox-dlm ships no migrations; run `manage.py makemigrations " + "netbox_dlm && manage.py migrate` after installing it.", + ), +) + + OPTIONAL_PLUGIN_INTEGRATIONS = ( ROUTING_INTEGRATION, PEERING_INTEGRATION, ACI_INTEGRATION, + DLM_INTEGRATION, ) diff --git a/forward_netbox/utilities/query_fetch_execution.py b/forward_netbox/utilities/query_fetch_execution.py index 3272b297..f84cb289 100644 --- a/forward_netbox/utilities/query_fetch_execution.py +++ b/forward_netbox/utilities/query_fetch_execution.py @@ -250,6 +250,7 @@ def _extract_device_names(value: Any) -> set[str]: "netbox_routing.ospfarea": "device", "netbox_routing.ospfinterface": "device", "netbox_peering_manager.peeringsession": "device", + "netbox_dlm.devicesoftware": "name", } diff --git a/forward_netbox/utilities/query_registry.py b/forward_netbox/utilities/query_registry.py index 0c1c94c8..91f79b62 100644 --- a/forward_netbox/utilities/query_registry.py +++ b/forward_netbox/utilities/query_registry.py @@ -390,6 +390,24 @@ def routing_import_diagnostic_query() -> str: "filename": "forward_device_feature_tags_with_rules.nqe", "enabled": False, }, + { + "model_string": "netbox_dlm.softwareversion", + "name": "Forward DLM Software Versions", + "filename": "forward_dlm_software_versions.nqe", + "enabled": False, + }, + { + "model_string": "netbox_dlm.hardwarenotice", + "name": "Forward DLM Hardware Notices", + "filename": "forward_dlm_hardware_notices.nqe", + "enabled": False, + }, + { + "model_string": "netbox_dlm.devicesoftware", + "name": "Forward DLM Device Software", + "filename": "forward_dlm_device_software.nqe", + "enabled": False, + }, { "model_string": "dcim.module", "name": "Forward Modules", diff --git a/forward_netbox/utilities/single_branch_executor.py b/forward_netbox/utilities/single_branch_executor.py index 60b8ec36..7c410cd7 100644 --- a/forward_netbox/utilities/single_branch_executor.py +++ b/forward_netbox/utilities/single_branch_executor.py @@ -15,6 +15,7 @@ from ..choices import ForwardSyncStatusChoices from .branch_budget import build_branch_plan from .branching import build_branch_request +from .branching import missing_branch_table_report from .fast_bootstrap_executor import ForwardFastBootstrapExecutor from .multi_branch_lifecycle import create_noop_ingestion from .multi_branch_lifecycle import run_item_in_branch @@ -29,6 +30,19 @@ class ForwardSingleBranchExecutor(ForwardFastBootstrapExecutor): def run(self): self._set_runtime_phase("initializing", "Starting single-branch preflight.") + # Fail in seconds — before the expensive fetch and before provisioning + # would die mid-CREATE TABLE on a table that was never migrated. + missing_tables = missing_branch_table_report() + if missing_tables: + detail = "; ".join( + f"{app}: {', '.join(tables)}" for app, tables in missing_tables.items() + ) + raise SyncError( + "Cannot provision a branch: database tables are missing for " + f"installed apps ({detail}). Apply their migrations " + "(python manage.py migrate) or remove the plugin from PLUGINS, " + "then re-run the sync." + ) fetcher = ForwardQueryFetcher(self.sync, self.client, self.logger) context = fetcher.resolve_context() self._set_runtime_phase( diff --git a/forward_netbox/utilities/sync.py b/forward_netbox/utilities/sync.py index a0a765ee..50000cd2 100644 --- a/forward_netbox/utilities/sync.py +++ b/forward_netbox/utilities/sync.py @@ -20,6 +20,12 @@ from .sync_device import apply_dcim_virtualchassis from .sync_device import delete_dcim_device from .sync_device import delete_dcim_virtualchassis +from .sync_dlm import apply_netbox_dlm_devicesoftware +from .sync_dlm import apply_netbox_dlm_hardwarenotice +from .sync_dlm import apply_netbox_dlm_softwareversion +from .sync_dlm import delete_netbox_dlm_devicesoftware +from .sync_dlm import delete_netbox_dlm_hardwarenotice +from .sync_dlm import delete_netbox_dlm_softwareversion from .sync_events import EventsClearer from .sync_execution import run_sync_stage from .sync_interface import apply_dcim_interface @@ -202,6 +208,12 @@ sync_record_aggregated_conflict_warning, sync_record_aggregated_skip_warning, sync_record_issue, + apply_netbox_dlm_devicesoftware, + apply_netbox_dlm_hardwarenotice, + apply_netbox_dlm_softwareversion, + delete_netbox_dlm_devicesoftware, + delete_netbox_dlm_hardwarenotice, + delete_netbox_dlm_softwareversion, apply_netbox_peering_manager_peeringsession, apply_netbox_routing_bgpaddressfamily, apply_netbox_routing_bgppeer, diff --git a/forward_netbox/utilities/sync_contracts.py b/forward_netbox/utilities/sync_contracts.py index 7a32c8ba..c42b9535 100644 --- a/forward_netbox/utilities/sync_contracts.py +++ b/forward_netbox/utilities/sync_contracts.py @@ -273,6 +273,21 @@ class ModelSyncContract: ("device", "neighbor_address"), ), ), + "netbox_dlm.softwareversion": ModelSyncContract( + required_fields=("platform", "platform_slug", "version"), + allowed_coalesce_fields=("platform", "platform_slug", "version"), + default_coalesce_fields=(("platform_slug", "version"),), + ), + "netbox_dlm.hardwarenotice": ModelSyncContract( + required_fields=("device_type", "device_type_slug"), + allowed_coalesce_fields=("device_type", "device_type_slug"), + default_coalesce_fields=(("device_type_slug",),), + ), + "netbox_dlm.devicesoftware": ModelSyncContract( + required_fields=("name", "platform", "platform_slug", "version"), + allowed_coalesce_fields=("name",), + default_coalesce_fields=(("name",),), + ), "netbox_cisco_aci.acifabric": ModelSyncContract( required_fields=("name", "fabric_id"), allowed_coalesce_fields=("name",), diff --git a/forward_netbox/utilities/sync_dlm.py b/forward_netbox/utilities/sync_dlm.py new file mode 100644 index 00000000..629d8ac6 --- /dev/null +++ b/forward_netbox/utilities/sync_dlm.py @@ -0,0 +1,217 @@ +# Optional netbox-dlm (Device Lifecycle Management) integration. +# +# Maps Forward's end-of-life analysis to the netbox-dlm plugin's models: +# - SoftwareVersion <- device.platform.osSupport (per platform + OS version) +# - HardwareNotice <- device.platform.components[].support (per DeviceType) +# - DeviceSoftware <- device.platform.osVersion (one row per device) +# +# netbox-dlm's README expects DeviceSoftware to be populated by external sync +# tooling; this adapter is that tooling. All writes go through the standard +# runner upsert/delete primitives so branch staging, diffs, and prune behave +# exactly like every other model. +from datetime import date + +from django.core.exceptions import ObjectDoesNotExist + +from ..exceptions import ForwardDependencySkipError +from ..exceptions import ForwardSearchError + + +def _parse_date(value): + """ISO date string -> date, so values compare equal to stored DateFields + (a str value never equals a date and would update the row every sync).""" + if isinstance(value, date) or value in (None, ""): + return value or None + try: + return date.fromisoformat(str(value)) + except ValueError: + return None + + +def _dlm_model(runner, model_name, model_string): + return runner._optional_model("netbox_dlm", model_name, model_string) + + +def _lookup_platform(runner, row, model_string, object_label): + from dcim.models import Platform + + slug = str(row.get("platform_slug") or "").strip() + name = str(row.get("platform") or "").strip() + platform = None + if slug: + platform = runner._get_unique_or_raise(Platform, {"slug": slug}) + if platform is None and name: + platform = runner._get_unique_or_raise(Platform, {"name": name}) + if platform is None: + raise ForwardDependencySkipError( + f"Skipping {object_label} because platform `{name or slug}` is not " + "in NetBox yet.", + model_string=model_string, + context={"platform": name or slug}, + data=row, + ) + return platform + + +def _lookup_device_type(runner, row, model_string, object_label): + from dcim.models import DeviceType + + slug = str(row.get("device_type_slug") or "").strip() + model = str(row.get("device_type") or "").strip() + device_type = None + if slug: + device_type = runner._get_unique_or_raise(DeviceType, {"slug": slug}) + if device_type is None and model: + device_type = runner._get_unique_or_raise(DeviceType, {"model": model}) + if device_type is None: + raise ForwardDependencySkipError( + f"Skipping {object_label} because device type `{model or slug}` is " + "not in NetBox yet.", + model_string=model_string, + context={"device_type": model or slug}, + data=row, + ) + return device_type + + +def _lookup_device(runner, row, model_string, object_label): + try: + return runner._get_device_by_name(row["name"]) + except ObjectDoesNotExist as exc: + key = (row["name"],) + if runner._dependency_failed("dcim.device", key): + raise ForwardDependencySkipError( + f"Skipping {object_label} because dependency `dcim.device` " + f"failed for {key}.", + model_string=model_string, + context={"device": row["name"]}, + data=row, + ) from exc + raise ForwardSearchError( + f"Unable to find device `{row['name']}` for {object_label}.", + model_string=model_string, + context={"device": row["name"]}, + data=row, + ) from exc + + +def ensure_dlm_software_version(runner, row, *, with_dates=True): + SoftwareVersion = _dlm_model( + runner, "SoftwareVersion", "netbox_dlm.softwareversion" + ) + platform = _lookup_platform( + runner, row, "netbox_dlm.softwareversion", "DLM software version" + ) + values = { + "platform": platform, + "version": str(row.get("version") or "").strip(), + } + if with_dates: + end_of_support = _parse_date(row.get("end_of_support")) + if end_of_support: + values["end_of_support"] = end_of_support + if row.get("documentation_url"): + values["documentation_url"] = row["documentation_url"] + values = runner._model_field_values(SoftwareVersion, values) + if with_dates: + software_version, _ = runner._upsert_values_from_defaults( + "netbox_dlm.softwareversion", + SoftwareVersion, + values=values, + coalesce_sets=[("platform", "version")], + ) + else: + # Create-if-missing only: never overwrite end-of-life dates the + # versions map already applied. + software_version, _ = runner._coalesce_update_or_create( + SoftwareVersion, + coalesce_lookups=[{"platform": platform, "version": values.get("version")}], + create_values=values, + update_values={}, + ) + return software_version + + +def apply_netbox_dlm_softwareversion(runner, row): + return ensure_dlm_software_version(runner, row, with_dates=True) + + +def apply_netbox_dlm_hardwarenotice(runner, row): + HardwareNotice = _dlm_model(runner, "HardwareNotice", "netbox_dlm.hardwarenotice") + device_type = _lookup_device_type( + runner, row, "netbox_dlm.hardwarenotice", "DLM hardware notice" + ) + values = {"device_type": device_type} + for field in ("end_of_support", "end_of_security_patches", "end_of_sw_releases"): + parsed = _parse_date(row.get(field)) + if parsed: + values[field] = parsed + if row.get("documentation_url"): + values["documentation_url"] = row["documentation_url"] + values = runner._model_field_values(HardwareNotice, values) + notice, _ = runner._upsert_values_from_defaults( + "netbox_dlm.hardwarenotice", + HardwareNotice, + values=values, + coalesce_sets=[("device_type",)], + ) + return notice + + +def apply_netbox_dlm_devicesoftware(runner, row): + DeviceSoftware = _dlm_model(runner, "DeviceSoftware", "netbox_dlm.devicesoftware") + device = _lookup_device( + runner, row, "netbox_dlm.devicesoftware", "DLM device software" + ) + # Ensure the (platform, version) row exists even when the vendor publishes + # no end-of-life announcement (the versions map only emits announced ones); + # never overwrite dates the versions map already applied. + software_version = ensure_dlm_software_version(runner, row, with_dates=False) + values = runner._model_field_values( + DeviceSoftware, + {"device": device, "software_version": software_version}, + ) + device_software, _ = runner._upsert_values_from_defaults( + "netbox_dlm.devicesoftware", + DeviceSoftware, + values=values, + coalesce_sets=[("device",)], + ) + return device_software + + +def delete_netbox_dlm_softwareversion(runner, row): + SoftwareVersion = _dlm_model( + runner, "SoftwareVersion", "netbox_dlm.softwareversion" + ) + from dcim.models import Platform + + platform = runner._get_unique_or_raise( + Platform, {"slug": str(row.get("platform_slug") or "").strip()} + ) + if platform is None: + return False + return runner._delete_by_coalesce( + SoftwareVersion, + [{"platform": platform, "version": str(row.get("version") or "").strip()}], + ) + + +def delete_netbox_dlm_hardwarenotice(runner, row): + HardwareNotice = _dlm_model(runner, "HardwareNotice", "netbox_dlm.hardwarenotice") + from dcim.models import DeviceType + + device_type = runner._get_unique_or_raise( + DeviceType, {"slug": str(row.get("device_type_slug") or "").strip()} + ) + if device_type is None: + return False + return runner._delete_by_coalesce(HardwareNotice, [{"device_type": device_type}]) + + +def delete_netbox_dlm_devicesoftware(runner, row): + DeviceSoftware = _dlm_model(runner, "DeviceSoftware", "netbox_dlm.devicesoftware") + device = runner._lookup_device_by_name(row.get("name")) + if device is None: + return False + return runner._delete_by_coalesce(DeviceSoftware, [{"device": device}]) diff --git a/forward_netbox/utilities/sync_primitives.py b/forward_netbox/utilities/sync_primitives.py index 8da4065e..7b63898d 100644 --- a/forward_netbox/utilities/sync_primitives.py +++ b/forward_netbox/utilities/sync_primitives.py @@ -31,6 +31,9 @@ "ipam.vlan": (("site", "vid"),), "ipam.prefix": (("prefix", "vrf"), ("prefix", "vrf__isnull")), "netbox_peering_manager.peeringsession": (("bgp_peer",),), + "netbox_dlm.softwareversion": (("platform", "version"),), + "netbox_dlm.hardwarenotice": (("device_type",),), + "netbox_dlm.devicesoftware": (("device",),), "netbox_routing.bgprouter": ( ("assigned_object_type", "assigned_object_id", "asn"), ), @@ -742,6 +745,7 @@ def prime_dependency_lookup_caches(runner, model_string, rows): "netbox_routing.bgppeeraddressfamily": ("device",), "netbox_routing.ospfinstance": ("device",), "netbox_routing.ospfinterface": ("device",), + "netbox_dlm.devicesoftware": ("name",), } DEPENDENCY_PARENT_DEVICE_MODELS = tuple(DEPENDENCY_PARENT_DEVICE_FIELDS) @@ -856,6 +860,7 @@ def _dependency_device_names(model_string, rows): "netbox_routing.bgppeeraddressfamily": ("device",), "netbox_routing.ospfinstance": ("device",), "netbox_routing.ospfinterface": ("device",), + "netbox_dlm.devicesoftware": ("name",), }.get(model_string, ()) return { str(row.get(field)).strip() diff --git a/forward_netbox/utilities/sync_runner_adapters.py b/forward_netbox/utilities/sync_runner_adapters.py index 0e6c3ba5..cbe9e271 100644 --- a/forward_netbox/utilities/sync_runner_adapters.py +++ b/forward_netbox/utilities/sync_runner_adapters.py @@ -42,6 +42,12 @@ from .sync_device import apply_dcim_virtualchassis from .sync_device import delete_dcim_device from .sync_device import delete_dcim_virtualchassis +from .sync_dlm import apply_netbox_dlm_devicesoftware +from .sync_dlm import apply_netbox_dlm_hardwarenotice +from .sync_dlm import apply_netbox_dlm_softwareversion +from .sync_dlm import delete_netbox_dlm_devicesoftware +from .sync_dlm import delete_netbox_dlm_hardwarenotice +from .sync_dlm import delete_netbox_dlm_softwareversion from .sync_interface import apply_dcim_interface from .sync_interface import apply_dcim_macaddress from .sync_interface import apply_extras_taggeditem @@ -916,6 +922,24 @@ def _apply_netbox_routing_ospfinterface(self, row): def _apply_netbox_peering_manager_peeringsession(self, row): return apply_netbox_peering_manager_peeringsession(self, row) + def _apply_netbox_dlm_softwareversion(self, row): + return apply_netbox_dlm_softwareversion(self, row) + + def _apply_netbox_dlm_hardwarenotice(self, row): + return apply_netbox_dlm_hardwarenotice(self, row) + + def _apply_netbox_dlm_devicesoftware(self, row): + return apply_netbox_dlm_devicesoftware(self, row) + + def _delete_netbox_dlm_softwareversion(self, row): + return delete_netbox_dlm_softwareversion(self, row) + + def _delete_netbox_dlm_hardwarenotice(self, row): + return delete_netbox_dlm_hardwarenotice(self, row) + + def _delete_netbox_dlm_devicesoftware(self, row): + return delete_netbox_dlm_devicesoftware(self, row) + def _apply_netbox_cisco_aci_acifabric(self, row): return apply_netbox_cisco_aci_acifabric(self, row) diff --git a/pyproject.toml b/pyproject.toml index 1f23d5a4..c19199ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "forward-netbox" -version = "2.4.5" +version = "2.5.0" description = "NetBox plugin to sync Forward data into NetBox via built-in NQE queries" authors = ["Craig Johnson "] license = "MIT"