diff --git a/.changeset/0009-performance.md b/.changeset/0009-performance.md new file mode 100644 index 0000000..54a9549 --- /dev/null +++ b/.changeset/0009-performance.md @@ -0,0 +1,5 @@ +--- +"pan-scm-cli": minor +--- + +Major performance improvements. CLI startup (`scm --help`, `scm --version`) dropped from ~0.42s to ~0.11s via lazy command loading — modules import only when their command is dispatched. OAuth tokens are now cached per context in `~/.scm-cli/cache/` (0600 permissions), eliminating the token + signing-key roundtrips on every invocation; the cache auto-invalidates on rejection or credential change (disable with `SCM_NO_TOKEN_CACHE=1`). Bulk `scm load` commands now apply objects concurrently (5 workers, tune with `SCM_BULK_WORKERS`); position-sensitive rule types still load sequentially to preserve order. diff --git a/docs-site/docs/guide/configuration.md b/docs-site/docs/guide/configuration.md index 7f06c28..c86c68c 100644 --- a/docs-site/docs/guide/configuration.md +++ b/docs-site/docs/guide/configuration.md @@ -225,6 +225,33 @@ scm context test staging --mock scm context test ``` +### Token Cache + +OAuth tokens are cached per context in `~/.scm-cli/cache/` (file mode 0600) and +reused until shortly before expiry, so consecutive commands skip the token and +signing-key roundtrips. The cache is invalidated automatically when the API +rejects a cached token or when the cached credentials no longer match the +active context. + +```bash +# Disable the token cache entirely +export SCM_NO_TOKEN_CACHE=1 + +# Clear it manually +rm -rf ~/.scm-cli/cache +``` + +### Bulk Load Concurrency + +`scm load ...` commands apply objects concurrently (default 5 workers). +Position-sensitive types (security/NAT/PBF/QoS rules) always load sequentially +to preserve rule order. + +```bash +# Tune or serialize bulk loads +export SCM_BULK_WORKERS=1 +``` + ### Context Storage Contexts are stored in `~/.scm-cli/contexts/` as individual YAML files: diff --git a/src/scm_cli/__init__.py b/src/scm_cli/__init__.py index 0e38e5a..f129c35 100644 --- a/src/scm_cli/__init__.py +++ b/src/scm_cli/__init__.py @@ -1,12 +1,17 @@ """pan-scm-cli: CLI for Palo Alto Networks Strata Cloud Manager.""" -from importlib.metadata import version -__version__ = version("pan-scm-cli") +def __getattr__(name: str): + """Resolve __version__ lazily — importlib.metadata costs ~30ms at import.""" + if name == "__version__": + from importlib.metadata import version -from .main import app + return version("pan-scm-cli") + raise AttributeError(name) def main(): """Entry point for the scm command.""" + from .main import app + app() diff --git a/src/scm_cli/commands/deployment.py b/src/scm_cli/commands/deployment.py index 32e95c9..9dc0015 100644 --- a/src/scm_cli/commands/deployment.py +++ b/src/scm_cli/commands/deployment.py @@ -10,6 +10,7 @@ import typer import yaml +from ..utils.bulk import run_bulk from ..utils.config import load_from_yaml from ..utils.decorators import handle_command_errors from ..utils.output import OUTPUT_OPTION, OutputFormat, emit, error, info, success @@ -163,9 +164,7 @@ def load_bandwidth_allocation( typer.echo(yaml.dump(config["bandwidth_allocations"])) return None - # Apply each allocation - results = [] - for allocation_data in config["bandwidth_allocations"]: + def _apply(allocation_data: dict): # Extract description before validation since it's not in the model description = allocation_data.pop("description", "") @@ -173,7 +172,7 @@ def load_bandwidth_allocation( allocation = BandwidthAllocation(**allocation_data) # Call the SDK client to create the bandwidth allocation - result = scm_client.create_bandwidth_allocation( + return scm_client.create_bandwidth_allocation( name=allocation.name, bandwidth=allocation.bandwidth, spn_name_list=allocation.spn_name_list, @@ -181,6 +180,12 @@ def load_bandwidth_allocation( tags=allocation.tags, ) + # Apply each allocation concurrently, reporting outcomes in input order + results = [] + for _allocation_data, result, exc in run_bulk(config["bandwidth_allocations"], _apply): + if exc is not None: + raise exc # abort on first error, matching the previous sequential loop + results.append(result) # Output details about each allocation bandwidth_value = result.get("allocated_bandwidth", result.get("bandwidth", "N/A")) @@ -403,17 +408,18 @@ def load_service_connection( typer.echo(yaml.dump(config["service_connections"])) return None - # Apply each connection - results = [] - for connection_data in config["service_connections"]: + def _apply(connection_data: dict): # Validate using the Pydantic model connection = ServiceConnection(**connection_data) - # Convert to SDK model format - sdk_data = connection.to_sdk_model() - # Call the SDK client to create the service connection - result = scm_client.create_service_connection(**sdk_data) + return scm_client.create_service_connection(**connection.to_sdk_model()) + + # Apply each connection concurrently, reporting outcomes in input order + results = [] + for _connection_data, result, exc in run_bulk(config["service_connections"], _apply): + if exc is not None: + raise exc # abort on first error, matching the previous sequential loop results.append(result) # Show appropriate message based on action taken @@ -660,17 +666,18 @@ def load_remote_network( typer.echo(yaml.dump(config["remote_networks"])) return None - # Apply each network - results = [] - for network_data in config["remote_networks"]: + def _apply(network_data: dict): # Validate using the Pydantic model network = RemoteNetwork(**network_data) - # Convert to SDK model format - sdk_data = network.to_sdk_model() - # Call the SDK client to create the remote network - result = scm_client.create_remote_network(**sdk_data) + return scm_client.create_remote_network(**network.to_sdk_model()) + + # Apply each network concurrently, reporting outcomes in input order + results = [] + for _network_data, result, exc in run_bulk(config["remote_networks"], _apply): + if exc is not None: + raise exc # abort on first error, matching the previous sequential loop results.append(result) # Show appropriate message based on action taken @@ -1016,11 +1023,15 @@ def load_internal_dns_server( typer.echo(yaml.dump(config["internal_dns_servers"])) return None - results = [] - for server_data in config["internal_dns_servers"]: + def _apply(server_data: dict): server = InternalDNSServer(**server_data) - sdk_data = server.to_sdk_model() - result = scm_client.create_internal_dns_server(**sdk_data) + return scm_client.create_internal_dns_server(**server.to_sdk_model()) + + # Apply each server concurrently, reporting outcomes in input order + results = [] + for _server_data, result, exc in run_bulk(config["internal_dns_servers"], _apply): + if exc is not None: + raise exc # abort on first error, matching the previous sequential loop results.append(result) action = result.get("__action__", "created") diff --git a/src/scm_cli/commands/identity.py b/src/scm_cli/commands/identity.py index 405c01a..325889f 100644 --- a/src/scm_cli/commands/identity.py +++ b/src/scm_cli/commands/identity.py @@ -14,6 +14,7 @@ from pydantic import ValidationError from ..utils import validate_location_params +from ..utils.bulk import run_bulk from ..utils.decorators import handle_command_errors from ..utils.output import OUTPUT_OPTION, OutputFormat, emit, error, info, success from ..utils.sdk_client import scm_client @@ -71,6 +72,30 @@ def get_default_backup_filename(object_type: str, location_type: str, location_v return f"{object_type}-{safe_location}.yaml" +def _bulk_load_profiles(items: list[dict], apply_item, label: str, plural_label: str, file: Path) -> None: + """Apply profile items concurrently and report outcomes in input order. + + ``apply_item`` returns ``(profile, result)``; per-item failures are reported + with the same wording as the previous sequential loop and skipped. + """ + loaded_count = 0 + for _item_data, outcome, exc in run_bulk(items, apply_item): + if exc is not None: + error(f"Error loading {label}: {str(exc)}") + continue + profile, result = outcome + action = result.get("__action__", "created") + if action == "no_change": + info(f"No changes for {label}: {profile.name}") + elif action == "updated": + success(f"Updated {label}: {profile.name}") + else: + success(f"Created {label}: {profile.name}") + loaded_count += 1 + + success(f"Processed {loaded_count} {plural_label} from {file}") + + # ============================================================================================================================================================================================= # AUTHENTICATION PROFILE COMMANDS # ============================================================================================================================================================================================= @@ -229,33 +254,19 @@ def load_authentication_profile( typer.echo(yaml.dump(profiles, default_flow_style=False)) return - loaded_count = 0 - for profile_data in profiles: - try: - if folder: - profile_data["folder"] = folder - elif snippet: - profile_data["snippet"] = snippet - elif device: - profile_data["device"] = device - - profile = AuthenticationProfile(**profile_data) - sdk_data = profile.to_sdk_model() - result = scm_client.create_authentication_profile(**sdk_data) - - action = result.get("__action__", "created") - if action == "no_change": - info(f"No changes for authentication profile: {profile.name}") - elif action == "updated": - success(f"Updated authentication profile: {profile.name}") - else: - success(f"Created authentication profile: {profile.name}") - loaded_count += 1 - - except Exception as e: - error(f"Error loading authentication profile: {str(e)}") - - success(f"Processed {loaded_count} authentication profiles from {file}") + def _apply(profile_data: dict): + if folder: + profile_data["folder"] = folder + elif snippet: + profile_data["snippet"] = snippet + elif device: + profile_data["device"] = device + + profile = AuthenticationProfile(**profile_data) + sdk_data = profile.to_sdk_model() + return profile, scm_client.create_authentication_profile(**sdk_data) + + _bulk_load_profiles(profiles, _apply, "authentication profile", "authentication profiles", file) @backup_app.command("authentication-profile") @@ -441,33 +452,19 @@ def load_kerberos_server_profile( typer.echo(yaml.dump(profiles, default_flow_style=False)) return - loaded_count = 0 - for profile_data in profiles: - try: - if folder: - profile_data["folder"] = folder - elif snippet: - profile_data["snippet"] = snippet - elif device: - profile_data["device"] = device - - profile = KerberosServerProfile(**profile_data) - sdk_data = profile.to_sdk_model() - result = scm_client.create_kerberos_server_profile(**sdk_data) - - action = result.get("__action__", "created") - if action == "no_change": - info(f"No changes for Kerberos server profile: {profile.name}") - elif action == "updated": - success(f"Updated Kerberos server profile: {profile.name}") - else: - success(f"Created Kerberos server profile: {profile.name}") - loaded_count += 1 - - except Exception as e: - error(f"Error loading Kerberos server profile: {str(e)}") - - success(f"Processed {loaded_count} Kerberos server profiles from {file}") + def _apply(profile_data: dict): + if folder: + profile_data["folder"] = folder + elif snippet: + profile_data["snippet"] = snippet + elif device: + profile_data["device"] = device + + profile = KerberosServerProfile(**profile_data) + sdk_data = profile.to_sdk_model() + return profile, scm_client.create_kerberos_server_profile(**sdk_data) + + _bulk_load_profiles(profiles, _apply, "Kerberos server profile", "Kerberos server profiles", file) @backup_app.command("kerberos-server-profile") @@ -664,33 +661,19 @@ def load_ldap_server_profile( typer.echo(yaml.dump(profiles, default_flow_style=False)) return - loaded_count = 0 - for profile_data in profiles: - try: - if folder: - profile_data["folder"] = folder - elif snippet: - profile_data["snippet"] = snippet - elif device: - profile_data["device"] = device - - profile = LdapServerProfile(**profile_data) - sdk_data = profile.to_sdk_model() - result = scm_client.create_ldap_server_profile(**sdk_data) - - action = result.get("__action__", "created") - if action == "no_change": - info(f"No changes for LDAP server profile: {profile.name}") - elif action == "updated": - success(f"Updated LDAP server profile: {profile.name}") - else: - success(f"Created LDAP server profile: {profile.name}") - loaded_count += 1 - - except Exception as e: - error(f"Error loading LDAP server profile: {str(e)}") - - success(f"Processed {loaded_count} LDAP server profiles from {file}") + def _apply(profile_data: dict): + if folder: + profile_data["folder"] = folder + elif snippet: + profile_data["snippet"] = snippet + elif device: + profile_data["device"] = device + + profile = LdapServerProfile(**profile_data) + sdk_data = profile.to_sdk_model() + return profile, scm_client.create_ldap_server_profile(**sdk_data) + + _bulk_load_profiles(profiles, _apply, "LDAP server profile", "LDAP server profiles", file) @backup_app.command("ldap-server-profile") @@ -884,33 +867,19 @@ def load_radius_server_profile( typer.echo(yaml.dump(profiles, default_flow_style=False)) return - loaded_count = 0 - for profile_data in profiles: - try: - if folder: - profile_data["folder"] = folder - elif snippet: - profile_data["snippet"] = snippet - elif device: - profile_data["device"] = device - - profile = RadiusServerProfile(**profile_data) - sdk_data = profile.to_sdk_model() - result = scm_client.create_radius_server_profile(**sdk_data) - - action = result.get("__action__", "created") - if action == "no_change": - info(f"No changes for RADIUS server profile: {profile.name}") - elif action == "updated": - success(f"Updated RADIUS server profile: {profile.name}") - else: - success(f"Created RADIUS server profile: {profile.name}") - loaded_count += 1 - - except Exception as e: - error(f"Error loading RADIUS server profile: {str(e)}") - - success(f"Processed {loaded_count} RADIUS server profiles from {file}") + def _apply(profile_data: dict): + if folder: + profile_data["folder"] = folder + elif snippet: + profile_data["snippet"] = snippet + elif device: + profile_data["device"] = device + + profile = RadiusServerProfile(**profile_data) + sdk_data = profile.to_sdk_model() + return profile, scm_client.create_radius_server_profile(**sdk_data) + + _bulk_load_profiles(profiles, _apply, "RADIUS server profile", "RADIUS server profiles", file) @backup_app.command("radius-server-profile") @@ -1109,33 +1078,19 @@ def load_saml_server_profile( typer.echo(yaml.dump(profiles, default_flow_style=False)) return - loaded_count = 0 - for profile_data in profiles: - try: - if folder: - profile_data["folder"] = folder - elif snippet: - profile_data["snippet"] = snippet - elif device: - profile_data["device"] = device - - profile = SamlServerProfile(**profile_data) - sdk_data = profile.to_sdk_model() - result = scm_client.create_saml_server_profile(**sdk_data) - - action = result.get("__action__", "created") - if action == "no_change": - info(f"No changes for SAML server profile: {profile.name}") - elif action == "updated": - success(f"Updated SAML server profile: {profile.name}") - else: - success(f"Created SAML server profile: {profile.name}") - loaded_count += 1 - - except Exception as e: - error(f"Error loading SAML server profile: {str(e)}") - - success(f"Processed {loaded_count} SAML server profiles from {file}") + def _apply(profile_data: dict): + if folder: + profile_data["folder"] = folder + elif snippet: + profile_data["snippet"] = snippet + elif device: + profile_data["device"] = device + + profile = SamlServerProfile(**profile_data) + sdk_data = profile.to_sdk_model() + return profile, scm_client.create_saml_server_profile(**sdk_data) + + _bulk_load_profiles(profiles, _apply, "SAML server profile", "SAML server profiles", file) @backup_app.command("saml-server-profile") @@ -1328,33 +1283,19 @@ def load_tacacs_server_profile( typer.echo(yaml.dump(profiles, default_flow_style=False)) return - loaded_count = 0 - for profile_data in profiles: - try: - if folder: - profile_data["folder"] = folder - elif snippet: - profile_data["snippet"] = snippet - elif device: - profile_data["device"] = device - - profile = TacacsServerProfile(**profile_data) - sdk_data = profile.to_sdk_model() - result = scm_client.create_tacacs_server_profile(**sdk_data) - - action = result.get("__action__", "created") - if action == "no_change": - info(f"No changes for TACACS+ server profile: {profile.name}") - elif action == "updated": - success(f"Updated TACACS+ server profile: {profile.name}") - else: - success(f"Created TACACS+ server profile: {profile.name}") - loaded_count += 1 - - except Exception as e: - error(f"Error loading TACACS+ server profile: {str(e)}") - - success(f"Processed {loaded_count} TACACS+ server profiles from {file}") + def _apply(profile_data: dict): + if folder: + profile_data["folder"] = folder + elif snippet: + profile_data["snippet"] = snippet + elif device: + profile_data["device"] = device + + profile = TacacsServerProfile(**profile_data) + sdk_data = profile.to_sdk_model() + return profile, scm_client.create_tacacs_server_profile(**sdk_data) + + _bulk_load_profiles(profiles, _apply, "TACACS+ server profile", "TACACS+ server profiles", file) @backup_app.command("tacacs-server-profile") diff --git a/src/scm_cli/commands/mobile_agent.py b/src/scm_cli/commands/mobile_agent.py index 776eb73..2011eba 100644 --- a/src/scm_cli/commands/mobile_agent.py +++ b/src/scm_cli/commands/mobile_agent.py @@ -12,6 +12,7 @@ from pydantic import ValidationError from ..utils import validate_location_params +from ..utils.bulk import run_bulk from ..utils.config import load_from_yaml from ..utils.decorators import handle_command_errors from ..utils.output import OUTPUT_OPTION, OutputFormat, emit, error, info, success @@ -40,6 +41,42 @@ def get_default_backup_filename(object_type: str, location_type: str, location_v return f"{object_type}-{safe_location}.yaml" +def _bulk_load_items(items: list[dict], apply_item, label: str) -> list[dict]: + """Apply load items concurrently and report outcomes in input order. + + ``apply_item`` validates and creates a single item, returning the SDK + result dict. Per-item failures are reported with the same wording as the + previous sequential loop and skipped. + """ + results = [] + created_count = 0 + updated_count = 0 + no_change_count = 0 + + for item_data, result, exc in run_bulk(items, apply_item): + if exc is not None: + error(f"Error loading {label} '{item_data.get('name', 'unknown')}': {str(exc)}") + continue + + action = result.pop("__action__", "created") + if action == "created": + created_count += 1 + success(f"Created {label}: {result.get('name', 'N/A')}") + elif action == "updated": + updated_count += 1 + success(f"Updated {label}: {result.get('name', 'N/A')}") + elif action == "no_change": + no_change_count += 1 + info(f"No changes needed for {label}: {result.get('name', 'N/A')}") + + results.append(result) + + # Summary + info(f"Summary: {created_count} created, {updated_count} updated, {no_change_count} unchanged") + + return results + + # ============================================================================================================================================================================================= # TYPER APP CONFIGURATION # ============================================================================================================================================================================================= @@ -288,56 +325,28 @@ def load_auth_setting( typer.echo(yaml.dump(config["auth_settings"])) return None - # Apply each auth setting - results = [] - created_count = 0 - updated_count = 0 - no_change_count = 0 - - for setting_data in config["auth_settings"]: - try: - # Apply container override if specified - if folder: - setting_data["folder"] = folder - setting_data.pop("snippet", None) - setting_data.pop("device", None) - elif snippet: - setting_data["snippet"] = snippet - setting_data.pop("folder", None) - setting_data.pop("device", None) - elif device: - setting_data["device"] = device - setting_data.pop("folder", None) - setting_data.pop("snippet", None) - - # Validate using the Pydantic model - auth_setting = AuthSetting(**setting_data) - sdk_data = auth_setting.to_sdk_model() - - # Create the auth setting via SDK client - result = scm_client.create_auth_setting(**sdk_data) - - # Track action - action = result.pop("__action__", "created") - if action == "created": - created_count += 1 - success(f"Created auth setting: {result.get('name', 'N/A')}") - elif action == "updated": - updated_count += 1 - success(f"Updated auth setting: {result.get('name', 'N/A')}") - elif action == "no_change": - no_change_count += 1 - info(f"No changes needed for auth setting: {result.get('name', 'N/A')}") - - results.append(result) - - except Exception as e: - error(f"Error loading auth setting '{setting_data.get('name', 'unknown')}': {str(e)}") - - # Summary - info(f"Summary: {created_count} created, {updated_count} updated, {no_change_count} unchanged") - - return results + def _apply(setting_data: dict): + # Apply container override if specified + if folder: + setting_data["folder"] = folder + setting_data.pop("snippet", None) + setting_data.pop("device", None) + elif snippet: + setting_data["snippet"] = snippet + setting_data.pop("folder", None) + setting_data.pop("device", None) + elif device: + setting_data["device"] = device + setting_data.pop("folder", None) + setting_data.pop("snippet", None) + # Validate using the Pydantic model + item = AuthSetting(**setting_data) + sdk_data = item.to_sdk_model() + + # Create via SDK client + return scm_client.create_auth_setting(**sdk_data) + + return _bulk_load_items(config["auth_settings"], _apply, "auth setting") except ValidationError as e: error(f"Validation error: {e}") @@ -564,40 +573,18 @@ def load_forwarding_profile( typer.echo(yaml.dump(config["forwarding_profiles"])) return None - results = [] - created_count = 0 - updated_count = 0 - no_change_count = 0 - - for profile_data in config["forwarding_profiles"]: - try: - if folder: - profile_data["folder"] = folder - - profile = ForwardingProfile(**profile_data) - sdk_data = profile.to_sdk_model() - - result = scm_client.create_forwarding_profile(**sdk_data) - - action = result.pop("__action__", "created") - if action == "created": - created_count += 1 - success(f"Created forwarding profile: {result.get('name', 'N/A')}") - elif action == "updated": - updated_count += 1 - success(f"Updated forwarding profile: {result.get('name', 'N/A')}") - elif action == "no_change": - no_change_count += 1 - info(f"No changes needed for forwarding profile: {result.get('name', 'N/A')}") - - results.append(result) + def _apply(profile_data: dict): + # Apply folder override if specified + if folder: + profile_data["folder"] = folder + # Validate using the Pydantic model + item = ForwardingProfile(**profile_data) + sdk_data = item.to_sdk_model() - except Exception as e: - error(f"Error loading forwarding profile '{profile_data.get('name', 'unknown')}': {str(e)}") + # Create via SDK client + return scm_client.create_forwarding_profile(**sdk_data) - info(f"Summary: {created_count} created, {updated_count} updated, {no_change_count} unchanged") - - return results + return _bulk_load_items(config["forwarding_profiles"], _apply, "forwarding profile") except ValidationError as e: error(f"Validation error: {e}") @@ -809,40 +796,18 @@ def load_forwarding_profile_destination( typer.echo(yaml.dump(config["forwarding_profile_destinations"])) return None - results = [] - created_count = 0 - updated_count = 0 - no_change_count = 0 - - for destination_data in config["forwarding_profile_destinations"]: - try: - if folder: - destination_data["folder"] = folder - - destination = ForwardingProfileDestination(**destination_data) - sdk_data = destination.to_sdk_model() - - result = scm_client.create_forwarding_profile_destination(**sdk_data) - - action = result.pop("__action__", "created") - if action == "created": - created_count += 1 - success(f"Created forwarding profile destination: {result.get('name', 'N/A')}") - elif action == "updated": - updated_count += 1 - success(f"Updated forwarding profile destination: {result.get('name', 'N/A')}") - elif action == "no_change": - no_change_count += 1 - info(f"No changes needed for forwarding profile destination: {result.get('name', 'N/A')}") - - results.append(result) - - except Exception as e: - error(f"Error loading forwarding profile destination '{destination_data.get('name', 'unknown')}': {str(e)}") + def _apply(destination_data: dict): + # Apply folder override if specified + if folder: + destination_data["folder"] = folder + # Validate using the Pydantic model + item = ForwardingProfileDestination(**destination_data) + sdk_data = item.to_sdk_model() - info(f"Summary: {created_count} created, {updated_count} updated, {no_change_count} unchanged") + # Create via SDK client + return scm_client.create_forwarding_profile_destination(**sdk_data) - return results + return _bulk_load_items(config["forwarding_profile_destinations"], _apply, "forwarding profile destination") except ValidationError as e: error(f"Validation error: {e}") @@ -1081,40 +1046,18 @@ def load_agent_profile( typer.echo(yaml.dump(config["agent_profiles"])) return None - results = [] - created_count = 0 - updated_count = 0 - no_change_count = 0 - - for profile_data in config["agent_profiles"]: - try: - if folder: - profile_data["folder"] = folder - - agent_profile = AgentProfile(**profile_data) - sdk_data = agent_profile.to_sdk_model() - - result = scm_client.create_agent_profile(**sdk_data) - - action = result.pop("__action__", "created") - if action == "created": - created_count += 1 - success(f"Created agent profile: {result.get('name', 'N/A')}") - elif action == "updated": - updated_count += 1 - success(f"Updated agent profile: {result.get('name', 'N/A')}") - elif action == "no_change": - no_change_count += 1 - info(f"No changes needed for agent profile: {result.get('name', 'N/A')}") - - results.append(result) - - except Exception as e: - error(f"Error loading agent profile '{profile_data.get('name', 'unknown')}': {str(e)}") + def _apply(profile_data: dict): + # Apply folder override if specified + if folder: + profile_data["folder"] = folder + # Validate using the Pydantic model + item = AgentProfile(**profile_data) + sdk_data = item.to_sdk_model() - info(f"Summary: {created_count} created, {updated_count} updated, {no_change_count} unchanged") + # Create via SDK client + return scm_client.create_agent_profile(**sdk_data) - return results + return _bulk_load_items(config["agent_profiles"], _apply, "agent profile") except ValidationError as e: error(f"Validation error: {e}") @@ -1314,40 +1257,18 @@ def load_tunnel_profile( typer.echo(yaml.dump(config["tunnel_profiles"])) return None - results = [] - created_count = 0 - updated_count = 0 - no_change_count = 0 + def _apply(profile_data: dict): + # Apply folder override if specified + if folder: + profile_data["folder"] = folder + # Validate using the Pydantic model + item = TunnelProfile(**profile_data) + sdk_data = item.to_sdk_model() - for profile_data in config["tunnel_profiles"]: - try: - if folder: - profile_data["folder"] = folder + # Create via SDK client + return scm_client.create_tunnel_profile(**sdk_data) - tunnel_profile = TunnelProfile(**profile_data) - sdk_data = tunnel_profile.to_sdk_model() - - result = scm_client.create_tunnel_profile(**sdk_data) - - action = result.pop("__action__", "created") - if action == "created": - created_count += 1 - success(f"Created tunnel profile: {result.get('name', 'N/A')}") - elif action == "updated": - updated_count += 1 - success(f"Updated tunnel profile: {result.get('name', 'N/A')}") - elif action == "no_change": - no_change_count += 1 - info(f"No changes needed for tunnel profile: {result.get('name', 'N/A')}") - - results.append(result) - - except Exception as e: - error(f"Error loading tunnel profile '{profile_data.get('name', 'unknown')}': {str(e)}") - - info(f"Summary: {created_count} created, {updated_count} updated, {no_change_count} unchanged") - - return results + return _bulk_load_items(config["tunnel_profiles"], _apply, "tunnel profile") except ValidationError as e: error(f"Validation error: {e}") @@ -1657,42 +1578,18 @@ def load_infrastructure_setting( typer.echo(yaml.dump(config["infrastructure_settings"])) return None - results = [] - created_count = 0 - updated_count = 0 - no_change_count = 0 - - for setting_data in config["infrastructure_settings"]: - try: - # Apply folder override if specified (only valid value is 'Mobile Users') - if folder: - setting_data["folder"] = folder + def _apply(setting_data: dict): + # Apply folder override if specified + if folder: + setting_data["folder"] = folder + # Validate using the Pydantic model + item = InfrastructureSetting(**setting_data) + sdk_data = item.to_sdk_model() - # Validate using the Pydantic model - infrastructure_setting = InfrastructureSetting(**setting_data) - sdk_data = infrastructure_setting.to_sdk_model() + # Create via SDK client + return scm_client.create_infrastructure_setting(**sdk_data) - result = scm_client.create_infrastructure_setting(**sdk_data) - - action = result.pop("__action__", "created") - if action == "created": - created_count += 1 - success(f"Created infrastructure setting: {result.get('name', 'N/A')}") - elif action == "updated": - updated_count += 1 - success(f"Updated infrastructure setting: {result.get('name', 'N/A')}") - elif action == "no_change": - no_change_count += 1 - info(f"No changes needed for infrastructure setting: {result.get('name', 'N/A')}") - - results.append(result) - - except Exception as e: - error(f"Error loading infrastructure setting '{setting_data.get('name', 'unknown')}': {str(e)}") - - info(f"Summary: {created_count} created, {updated_count} updated, {no_change_count} unchanged") - - return results + return _bulk_load_items(config["infrastructure_settings"], _apply, "infrastructure setting") except ValidationError as e: error(f"Validation error: {e}") @@ -1872,46 +1769,18 @@ def load_forwarding_profile_source_application( typer.echo(yaml.dump(config["forwarding_profile_source_applications"])) return None - # Apply each source application - results = [] - created_count = 0 - updated_count = 0 - no_change_count = 0 - - for app_data in config["forwarding_profile_source_applications"]: - try: - # Apply container override if specified - if folder: - app_data["folder"] = folder + def _apply(app_data: dict): + # Apply folder override if specified + if folder: + app_data["folder"] = folder + # Validate using the Pydantic model + item = ForwardingProfileSourceApplication(**app_data) + sdk_data = item.to_sdk_model() - # Validate using the Pydantic model - source_application = ForwardingProfileSourceApplication(**app_data) - sdk_data = source_application.to_sdk_model() + # Create via SDK client + return scm_client.create_forwarding_profile_source_application(**sdk_data) - # Create the source application via SDK client - result = scm_client.create_forwarding_profile_source_application(**sdk_data) - - # Track action - action = result.pop("__action__", "created") - if action == "created": - created_count += 1 - success(f"Created forwarding profile source application: {result.get('name', 'N/A')}") - elif action == "updated": - updated_count += 1 - success(f"Updated forwarding profile source application: {result.get('name', 'N/A')}") - elif action == "no_change": - no_change_count += 1 - info(f"No changes needed for forwarding profile source application: {result.get('name', 'N/A')}") - - results.append(result) - - except Exception as e: - error(f"Error loading forwarding profile source application '{app_data.get('name', 'unknown')}': {str(e)}") - - # Summary - info(f"Summary: {created_count} created, {updated_count} updated, {no_change_count} unchanged") - - return results + return _bulk_load_items(config["forwarding_profile_source_applications"], _apply, "forwarding profile source application") except ValidationError as e: error(f"Validation error: {e}") @@ -2132,46 +2001,18 @@ def load_forwarding_profile_user_location( typer.echo(yaml.dump(config["forwarding_profile_user_locations"])) return None - # Apply each user location - results = [] - created_count = 0 - updated_count = 0 - no_change_count = 0 - - for location_data in config["forwarding_profile_user_locations"]: - try: - # Apply container override if specified - if folder: - location_data["folder"] = folder - - # Validate using the Pydantic model - user_location = ForwardingProfileUserLocation(**location_data) - sdk_data = user_location.to_sdk_model() + def _apply(location_data: dict): + # Apply folder override if specified + if folder: + location_data["folder"] = folder + # Validate using the Pydantic model + item = ForwardingProfileUserLocation(**location_data) + sdk_data = item.to_sdk_model() - # Create the user location via SDK client - result = scm_client.create_forwarding_profile_user_location(**sdk_data) + # Create via SDK client + return scm_client.create_forwarding_profile_user_location(**sdk_data) - # Track action - action = result.pop("__action__", "created") - if action == "created": - created_count += 1 - success(f"Created forwarding profile user location: {result.get('name', 'N/A')}") - elif action == "updated": - updated_count += 1 - success(f"Updated forwarding profile user location: {result.get('name', 'N/A')}") - elif action == "no_change": - no_change_count += 1 - info(f"No changes needed for forwarding profile user location: {result.get('name', 'N/A')}") - - results.append(result) - - except Exception as e: - error(f"Error loading forwarding profile user location '{location_data.get('name', 'unknown')}': {str(e)}") - - # Summary - info(f"Summary: {created_count} created, {updated_count} updated, {no_change_count} unchanged") - - return results + return _bulk_load_items(config["forwarding_profile_user_locations"], _apply, "forwarding profile user location") except ValidationError as e: error(f"Validation error: {e}") @@ -2431,46 +2272,18 @@ def load_forwarding_profile_regional_and_custom_proxy( typer.echo(yaml.dump(config["forwarding_profile_regional_and_custom_proxies"])) return None - # Apply each regional and custom proxy - results = [] - created_count = 0 - updated_count = 0 - no_change_count = 0 - - for proxy_data in config["forwarding_profile_regional_and_custom_proxies"]: - try: - # Apply container override if specified - if folder: - proxy_data["folder"] = folder - - # Validate using the Pydantic model - regional_proxy = ForwardingProfileRegionalAndCustomProxy(**proxy_data) - sdk_data = regional_proxy.to_sdk_model() - - # Create the regional and custom proxy via SDK client - result = scm_client.create_forwarding_profile_regional_and_custom_proxy(**sdk_data) - - # Track action - action = result.pop("__action__", "created") - if action == "created": - created_count += 1 - success(f"Created forwarding profile regional and custom proxy: {result.get('name', 'N/A')}") - elif action == "updated": - updated_count += 1 - success(f"Updated forwarding profile regional and custom proxy: {result.get('name', 'N/A')}") - elif action == "no_change": - no_change_count += 1 - info(f"No changes needed for forwarding profile regional and custom proxy: {result.get('name', 'N/A')}") - - results.append(result) - - except Exception as e: - error(f"Error loading forwarding profile regional and custom proxy '{proxy_data.get('name', 'unknown')}': {str(e)}") - - # Summary - info(f"Summary: {created_count} created, {updated_count} updated, {no_change_count} unchanged") - - return results + def _apply(proxy_data: dict): + # Apply folder override if specified + if folder: + proxy_data["folder"] = folder + # Validate using the Pydantic model + item = ForwardingProfileRegionalAndCustomProxy(**proxy_data) + sdk_data = item.to_sdk_model() + + # Create via SDK client + return scm_client.create_forwarding_profile_regional_and_custom_proxy(**sdk_data) + + return _bulk_load_items(config["forwarding_profile_regional_and_custom_proxies"], _apply, "forwarding profile regional and custom proxy") except ValidationError as e: error(f"Validation error: {e}") diff --git a/src/scm_cli/commands/network.py b/src/scm_cli/commands/network.py index 53976f2..2574d60 100644 --- a/src/scm_cli/commands/network.py +++ b/src/scm_cli/commands/network.py @@ -13,6 +13,7 @@ import yaml from ..utils import validate_location_params +from ..utils.bulk import run_bulk from ..utils.config import load_from_yaml from ..utils.decorators import handle_command_errors from ..utils.output import OUTPUT_OPTION, OutputFormat, emit, error, info, success @@ -336,29 +337,32 @@ def load_ike_crypto_profile( if not isinstance(profiles, list): profiles = [profiles] created_count = 0 - for profile_data in profiles: - try: - validated_profile = IKECryptoProfile(**profile_data) - if folder: - validated_profile.folder = folder - validated_profile.snippet = None - validated_profile.device = None - elif snippet: - validated_profile.snippet = snippet - validated_profile.folder = None - validated_profile.device = None - elif device: - validated_profile.device = device - validated_profile.folder = None - validated_profile.snippet = None - sdk_data = validated_profile.to_sdk_model() - scm_client.create_ike_crypto_profile(sdk_data) - created_count += 1 - container = validated_profile.folder or validated_profile.snippet or validated_profile.device - success(f"Created IKE crypto profile: {validated_profile.name} in {container}") - except Exception as e: - error(f"Error processing IKE crypto profile: {str(e)}") + + def _apply(profile_data: dict): + validated_profile = IKECryptoProfile(**profile_data) + if folder: + validated_profile.folder = folder + validated_profile.snippet = None + validated_profile.device = None + elif snippet: + validated_profile.snippet = snippet + validated_profile.folder = None + validated_profile.device = None + elif device: + validated_profile.device = device + validated_profile.folder = None + validated_profile.snippet = None + sdk_data = validated_profile.to_sdk_model() + scm_client.create_ike_crypto_profile(sdk_data) + return validated_profile + + for _item, validated_profile, exc in run_bulk(profiles, _apply): + if exc is not None: + error(f"Error processing IKE crypto profile: {str(exc)}") continue + created_count += 1 + container = validated_profile.folder or validated_profile.snippet or validated_profile.device + success(f"Created IKE crypto profile: {validated_profile.name} in {container}") info(f"\nSummary: Processed {created_count} IKE crypto profiles") @@ -512,37 +516,41 @@ def load_aggregate_interface( created_count = 0 updated_count = 0 no_change_count = 0 - for iface_data in interfaces: - try: - if folder: - iface_data["folder"] = folder - iface_data.pop("snippet", None) - iface_data.pop("device", None) - elif snippet: - iface_data["snippet"] = snippet - iface_data.pop("folder", None) - iface_data.pop("device", None) - elif device: - iface_data["device"] = device - iface_data.pop("folder", None) - iface_data.pop("snippet", None) - validated_iface = AggregateInterface(**iface_data) - sdk_data = validated_iface.to_sdk_model() - result = scm_client.create_aggregate_interface(sdk_data) - action = result.pop("__action__", "created") - container = validated_iface.folder or validated_iface.snippet or validated_iface.device - if action == "created": - created_count += 1 - success(f"Created aggregate interface: {validated_iface.name} in {container}") - elif action == "updated": - updated_count += 1 - success(f"Updated aggregate interface: {validated_iface.name} in {container}") - else: - no_change_count += 1 - info(f"No changes needed for aggregate interface: {validated_iface.name} in {container}") - except Exception as e: - error(f"Error processing aggregate interface: {str(e)}") + + def _apply(iface_data: dict): + if folder: + iface_data["folder"] = folder + iface_data.pop("snippet", None) + iface_data.pop("device", None) + elif snippet: + iface_data["snippet"] = snippet + iface_data.pop("folder", None) + iface_data.pop("device", None) + elif device: + iface_data["device"] = device + iface_data.pop("folder", None) + iface_data.pop("snippet", None) + validated_iface = AggregateInterface(**iface_data) + sdk_data = validated_iface.to_sdk_model() + result = scm_client.create_aggregate_interface(sdk_data) + return validated_iface, result + + for _item, outcome, exc in run_bulk(interfaces, _apply): + if exc is not None: + error(f"Error processing aggregate interface: {str(exc)}") continue + validated_iface, result = outcome + action = result.pop("__action__", "created") + container = validated_iface.folder or validated_iface.snippet or validated_iface.device + if action == "created": + created_count += 1 + success(f"Created aggregate interface: {validated_iface.name} in {container}") + elif action == "updated": + updated_count += 1 + success(f"Updated aggregate interface: {validated_iface.name} in {container}") + else: + no_change_count += 1 + info(f"No changes needed for aggregate interface: {validated_iface.name} in {container}") info(f"\nSummary: Processed {created_count + updated_count + no_change_count} aggregate interfaces") if created_count > 0: info(f" - Created: {created_count}") @@ -699,37 +707,41 @@ def load_ike_gateway( created_count = 0 updated_count = 0 no_change_count = 0 - for gateway_data in gateways: - try: - if folder: - gateway_data["folder"] = folder - gateway_data.pop("snippet", None) - gateway_data.pop("device", None) - elif snippet: - gateway_data["snippet"] = snippet - gateway_data.pop("folder", None) - gateway_data.pop("device", None) - elif device: - gateway_data["device"] = device - gateway_data.pop("folder", None) - gateway_data.pop("snippet", None) - validated_gw = IKEGateway(**gateway_data) - sdk_data = validated_gw.to_sdk_model() - result = scm_client.create_ike_gateway(sdk_data) - action = result.pop("__action__", "created") - container = validated_gw.folder or validated_gw.snippet or validated_gw.device - if action == "created": - created_count += 1 - success(f"Created IKE gateway: {validated_gw.name} in {container}") - elif action == "updated": - updated_count += 1 - success(f"Updated IKE gateway: {validated_gw.name} in {container}") - else: - no_change_count += 1 - info(f"No changes needed for IKE gateway: {validated_gw.name} in {container}") - except Exception as e: - error(f"Error processing IKE gateway: {str(e)}") + + def _apply(gateway_data: dict): + if folder: + gateway_data["folder"] = folder + gateway_data.pop("snippet", None) + gateway_data.pop("device", None) + elif snippet: + gateway_data["snippet"] = snippet + gateway_data.pop("folder", None) + gateway_data.pop("device", None) + elif device: + gateway_data["device"] = device + gateway_data.pop("folder", None) + gateway_data.pop("snippet", None) + validated_gw = IKEGateway(**gateway_data) + sdk_data = validated_gw.to_sdk_model() + result = scm_client.create_ike_gateway(sdk_data) + return validated_gw, result + + for _item, outcome, exc in run_bulk(gateways, _apply): + if exc is not None: + error(f"Error processing IKE gateway: {str(exc)}") continue + validated_gw, result = outcome + action = result.pop("__action__", "created") + container = validated_gw.folder or validated_gw.snippet or validated_gw.device + if action == "created": + created_count += 1 + success(f"Created IKE gateway: {validated_gw.name} in {container}") + elif action == "updated": + updated_count += 1 + success(f"Updated IKE gateway: {validated_gw.name} in {container}") + else: + no_change_count += 1 + info(f"No changes needed for IKE gateway: {validated_gw.name} in {container}") info(f"\nSummary: Processed {created_count + updated_count + no_change_count} IKE gateways") if created_count > 0: info(f" - Created: {created_count}") @@ -985,21 +997,25 @@ def load_security_zone( typer.echo(yaml.dump(config["security_zones"])) return None - # Apply each zone - results = [] - for zone_data in config["security_zones"]: + def _apply(zone_data: dict): # Validate using the Pydantic model zone = Zone(**zone_data) # Convert to the SDK model and create the zone sdk_data = zone.to_sdk_model() - result = scm_client.create_zone( + return scm_client.create_zone( folder=zone.folder, name=sdk_data["name"], mode=sdk_data["mode"], interfaces=sdk_data["interfaces"], ) + # Apply each zone + results = [] + for _item, result, exc in run_bulk(config["security_zones"], _apply): + if exc is not None: + raise exc # abort on first error, matching the previous sequential behavior + results.append(result) success(f"Applied zone: {result['name']} in folder {result['folder']}") @@ -1197,12 +1213,11 @@ def load_ipsec_crypto_profile( typer.echo(yaml.dump(config["ipsec_crypto_profiles"])) return None - results = [] - for profile_data in config["ipsec_crypto_profiles"]: + def _apply(profile_data: dict): profile = IPSecCryptoProfile(**profile_data) sdk_data = profile.to_sdk_model() - result = scm_client.create_ipsec_crypto_profile( + return scm_client.create_ipsec_crypto_profile( folder=profile.folder or "Texas", name=sdk_data["name"], esp_encryption=sdk_data["esp"]["encryption"], @@ -1212,6 +1227,11 @@ def load_ipsec_crypto_profile( lifesize=sdk_data.get("lifesize"), ) + results = [] + for _item, result, exc in run_bulk(config["ipsec_crypto_profiles"], _apply): + if exc is not None: + raise exc # abort on first error, matching the previous sequential behavior + results.append(result) action = result.get("__action__", "applied") success(f"IPsec crypto profile '{result['name']}' {action} in folder {result.get('folder', 'N/A')}") @@ -1425,6 +1445,7 @@ def load_nat_rule( updated_count = 0 no_change_count = 0 + # sequential: rule order matters for rule_data in nat_rules: try: # Apply container override @@ -1665,37 +1686,41 @@ def load_dhcp_interface( created_count = 0 updated_count = 0 no_change_count = 0 - for iface_data in interfaces: - try: - if folder: - iface_data["folder"] = folder - iface_data.pop("snippet", None) - iface_data.pop("device", None) - elif snippet: - iface_data["snippet"] = snippet - iface_data.pop("folder", None) - iface_data.pop("device", None) - elif device: - iface_data["device"] = device - iface_data.pop("folder", None) - iface_data.pop("snippet", None) - validated_iface = DhcpInterface(**iface_data) - sdk_data = validated_iface.to_sdk_model() - result = scm_client.create_dhcp_interface(sdk_data) - action = result.pop("__action__", "created") - container = validated_iface.folder or validated_iface.snippet or validated_iface.device - if action == "created": - created_count += 1 - success(f"Created DHCP interface: {validated_iface.name} in {container}") - elif action == "updated": - updated_count += 1 - success(f"Updated DHCP interface: {validated_iface.name} in {container}") - else: - no_change_count += 1 - info(f"No changes needed for DHCP interface: {validated_iface.name} in {container}") - except Exception as e: - error(f"Error processing DHCP interface: {str(e)}") + + def _apply(iface_data: dict): + if folder: + iface_data["folder"] = folder + iface_data.pop("snippet", None) + iface_data.pop("device", None) + elif snippet: + iface_data["snippet"] = snippet + iface_data.pop("folder", None) + iface_data.pop("device", None) + elif device: + iface_data["device"] = device + iface_data.pop("folder", None) + iface_data.pop("snippet", None) + validated_iface = DhcpInterface(**iface_data) + sdk_data = validated_iface.to_sdk_model() + result = scm_client.create_dhcp_interface(sdk_data) + return validated_iface, result + + for _item, outcome, exc in run_bulk(interfaces, _apply): + if exc is not None: + error(f"Error processing DHCP interface: {str(exc)}") continue + validated_iface, result = outcome + action = result.pop("__action__", "created") + container = validated_iface.folder or validated_iface.snippet or validated_iface.device + if action == "created": + created_count += 1 + success(f"Created DHCP interface: {validated_iface.name} in {container}") + elif action == "updated": + updated_count += 1 + success(f"Updated DHCP interface: {validated_iface.name} in {container}") + else: + no_change_count += 1 + info(f"No changes needed for DHCP interface: {validated_iface.name} in {container}") info(f"\nSummary: Processed {created_count + updated_count + no_change_count} DHCP interfaces") if created_count > 0: info(f" - Created: {created_count}") @@ -1840,37 +1865,41 @@ def load_ethernet_interface( created_count = 0 updated_count = 0 no_change_count = 0 - for iface_data in interfaces: - try: - if folder: - iface_data["folder"] = folder - iface_data.pop("snippet", None) - iface_data.pop("device", None) - elif snippet: - iface_data["snippet"] = snippet - iface_data.pop("folder", None) - iface_data.pop("device", None) - elif device: - iface_data["device"] = device - iface_data.pop("folder", None) - iface_data.pop("snippet", None) - validated_iface = EthernetInterface(**iface_data) - sdk_data = validated_iface.to_sdk_model() - result = scm_client.create_ethernet_interface(sdk_data) - action = result.pop("__action__", "created") - container = validated_iface.folder or validated_iface.snippet or validated_iface.device - if action == "created": - created_count += 1 - success(f"Created ethernet interface: {validated_iface.name} in {container}") - elif action == "updated": - updated_count += 1 - success(f"Updated ethernet interface: {validated_iface.name} in {container}") - else: - no_change_count += 1 - info(f"No changes needed for ethernet interface: {validated_iface.name} in {container}") - except Exception as e: - error(f"Error processing ethernet interface: {str(e)}") + + def _apply(iface_data: dict): + if folder: + iface_data["folder"] = folder + iface_data.pop("snippet", None) + iface_data.pop("device", None) + elif snippet: + iface_data["snippet"] = snippet + iface_data.pop("folder", None) + iface_data.pop("device", None) + elif device: + iface_data["device"] = device + iface_data.pop("folder", None) + iface_data.pop("snippet", None) + validated_iface = EthernetInterface(**iface_data) + sdk_data = validated_iface.to_sdk_model() + result = scm_client.create_ethernet_interface(sdk_data) + return validated_iface, result + + for _item, outcome, exc in run_bulk(interfaces, _apply): + if exc is not None: + error(f"Error processing ethernet interface: {str(exc)}") continue + validated_iface, result = outcome + action = result.pop("__action__", "created") + container = validated_iface.folder or validated_iface.snippet or validated_iface.device + if action == "created": + created_count += 1 + success(f"Created ethernet interface: {validated_iface.name} in {container}") + elif action == "updated": + updated_count += 1 + success(f"Updated ethernet interface: {validated_iface.name} in {container}") + else: + no_change_count += 1 + info(f"No changes needed for ethernet interface: {validated_iface.name} in {container}") info(f"\nSummary: Processed {created_count + updated_count + no_change_count} ethernet interfaces") if created_count > 0: info(f" - Created: {created_count}") @@ -2039,37 +2068,41 @@ def load_layer2_subinterface( created_count = 0 updated_count = 0 no_change_count = 0 - for iface_data in interfaces: - try: - if folder: - iface_data["folder"] = folder - iface_data.pop("snippet", None) - iface_data.pop("device", None) - elif snippet: - iface_data["snippet"] = snippet - iface_data.pop("folder", None) - iface_data.pop("device", None) - elif device: - iface_data["device"] = device - iface_data.pop("folder", None) - iface_data.pop("snippet", None) - validated_iface = Layer2Subinterface(**iface_data) - sdk_data = validated_iface.to_sdk_model() - result = scm_client.create_layer2_subinterface(sdk_data) - action = result.pop("__action__", "created") - container = validated_iface.folder or validated_iface.snippet or validated_iface.device - if action == "created": - created_count += 1 - success(f"Created layer2 subinterface: {validated_iface.name} in {container}") - elif action == "updated": - updated_count += 1 - success(f"Updated layer2 subinterface: {validated_iface.name} in {container}") - else: - no_change_count += 1 - info(f"No changes needed for layer2 subinterface: {validated_iface.name} in {container}") - except Exception as e: - error(f"Error processing layer2 subinterface: {str(e)}") + + def _apply(iface_data: dict): + if folder: + iface_data["folder"] = folder + iface_data.pop("snippet", None) + iface_data.pop("device", None) + elif snippet: + iface_data["snippet"] = snippet + iface_data.pop("folder", None) + iface_data.pop("device", None) + elif device: + iface_data["device"] = device + iface_data.pop("folder", None) + iface_data.pop("snippet", None) + validated_iface = Layer2Subinterface(**iface_data) + sdk_data = validated_iface.to_sdk_model() + result = scm_client.create_layer2_subinterface(sdk_data) + return validated_iface, result + + for _item, outcome, exc in run_bulk(interfaces, _apply): + if exc is not None: + error(f"Error processing layer2 subinterface: {str(exc)}") continue + validated_iface, result = outcome + action = result.pop("__action__", "created") + container = validated_iface.folder or validated_iface.snippet or validated_iface.device + if action == "created": + created_count += 1 + success(f"Created layer2 subinterface: {validated_iface.name} in {container}") + elif action == "updated": + updated_count += 1 + success(f"Updated layer2 subinterface: {validated_iface.name} in {container}") + else: + no_change_count += 1 + info(f"No changes needed for layer2 subinterface: {validated_iface.name} in {container}") info(f"\nSummary: Processed {created_count + updated_count + no_change_count} layer2 subinterfaces") if created_count > 0: info(f" - Created: {created_count}") @@ -2221,37 +2254,41 @@ def load_layer3_subinterface( created_count = 0 updated_count = 0 no_change_count = 0 - for iface_data in interfaces: - try: - if folder: - iface_data["folder"] = folder - iface_data.pop("snippet", None) - iface_data.pop("device", None) - elif snippet: - iface_data["snippet"] = snippet - iface_data.pop("folder", None) - iface_data.pop("device", None) - elif device: - iface_data["device"] = device - iface_data.pop("folder", None) - iface_data.pop("snippet", None) - validated_iface = Layer3Subinterface(**iface_data) - sdk_data = validated_iface.to_sdk_model() - result = scm_client.create_layer3_subinterface(sdk_data) - action = result.pop("__action__", "created") - container = validated_iface.folder or validated_iface.snippet or validated_iface.device - if action == "created": - created_count += 1 - success(f"Created layer3 subinterface: {validated_iface.name} in {container}") - elif action == "updated": - updated_count += 1 - success(f"Updated layer3 subinterface: {validated_iface.name} in {container}") - else: - no_change_count += 1 - info(f"No changes needed for layer3 subinterface: {validated_iface.name} in {container}") - except Exception as e: - error(f"Error processing layer3 subinterface: {str(e)}") + + def _apply(iface_data: dict): + if folder: + iface_data["folder"] = folder + iface_data.pop("snippet", None) + iface_data.pop("device", None) + elif snippet: + iface_data["snippet"] = snippet + iface_data.pop("folder", None) + iface_data.pop("device", None) + elif device: + iface_data["device"] = device + iface_data.pop("folder", None) + iface_data.pop("snippet", None) + validated_iface = Layer3Subinterface(**iface_data) + sdk_data = validated_iface.to_sdk_model() + result = scm_client.create_layer3_subinterface(sdk_data) + return validated_iface, result + + for _item, outcome, exc in run_bulk(interfaces, _apply): + if exc is not None: + error(f"Error processing layer3 subinterface: {str(exc)}") continue + validated_iface, result = outcome + action = result.pop("__action__", "created") + container = validated_iface.folder or validated_iface.snippet or validated_iface.device + if action == "created": + created_count += 1 + success(f"Created layer3 subinterface: {validated_iface.name} in {container}") + elif action == "updated": + updated_count += 1 + success(f"Updated layer3 subinterface: {validated_iface.name} in {container}") + else: + no_change_count += 1 + info(f"No changes needed for layer3 subinterface: {validated_iface.name} in {container}") info(f"\nSummary: Processed {created_count + updated_count + no_change_count} layer3 subinterfaces") if created_count > 0: info(f" - Created: {created_count}") @@ -2414,37 +2451,41 @@ def load_loopback_interface( created_count = 0 updated_count = 0 no_change_count = 0 - for iface_data in interfaces: - try: - if folder: - iface_data["folder"] = folder - iface_data.pop("snippet", None) - iface_data.pop("device", None) - elif snippet: - iface_data["snippet"] = snippet - iface_data.pop("folder", None) - iface_data.pop("device", None) - elif device: - iface_data["device"] = device - iface_data.pop("folder", None) - iface_data.pop("snippet", None) - validated_iface = LoopbackInterface(**iface_data) - sdk_data = validated_iface.to_sdk_model() - result = scm_client.create_loopback_interface(sdk_data) - action = result.pop("__action__", "created") - container = validated_iface.folder or validated_iface.snippet or validated_iface.device - if action == "created": - created_count += 1 - success(f"Created loopback interface: {validated_iface.name} in {container}") - elif action == "updated": - updated_count += 1 - success(f"Updated loopback interface: {validated_iface.name} in {container}") - else: - no_change_count += 1 - info(f"No changes needed for loopback interface: {validated_iface.name} in {container}") - except Exception as e: - error(f"Error processing loopback interface: {str(e)}") + + def _apply(iface_data: dict): + if folder: + iface_data["folder"] = folder + iface_data.pop("snippet", None) + iface_data.pop("device", None) + elif snippet: + iface_data["snippet"] = snippet + iface_data.pop("folder", None) + iface_data.pop("device", None) + elif device: + iface_data["device"] = device + iface_data.pop("folder", None) + iface_data.pop("snippet", None) + validated_iface = LoopbackInterface(**iface_data) + sdk_data = validated_iface.to_sdk_model() + result = scm_client.create_loopback_interface(sdk_data) + return validated_iface, result + + for _item, outcome, exc in run_bulk(interfaces, _apply): + if exc is not None: + error(f"Error processing loopback interface: {str(exc)}") continue + validated_iface, result = outcome + action = result.pop("__action__", "created") + container = validated_iface.folder or validated_iface.snippet or validated_iface.device + if action == "created": + created_count += 1 + success(f"Created loopback interface: {validated_iface.name} in {container}") + elif action == "updated": + updated_count += 1 + success(f"Updated loopback interface: {validated_iface.name} in {container}") + else: + no_change_count += 1 + info(f"No changes needed for loopback interface: {validated_iface.name} in {container}") info(f"\nSummary: Processed {created_count + updated_count + no_change_count} loopback interfaces") if created_count > 0: info(f" - Created: {created_count}") @@ -2604,37 +2645,41 @@ def load_tunnel_interface( created_count = 0 updated_count = 0 no_change_count = 0 - for iface_data in interfaces: - try: - if folder: - iface_data["folder"] = folder - iface_data.pop("snippet", None) - iface_data.pop("device", None) - elif snippet: - iface_data["snippet"] = snippet - iface_data.pop("folder", None) - iface_data.pop("device", None) - elif device: - iface_data["device"] = device - iface_data.pop("folder", None) - iface_data.pop("snippet", None) - validated_iface = TunnelInterface(**iface_data) - sdk_data = validated_iface.to_sdk_model() - result = scm_client.create_tunnel_interface(sdk_data) - action = result.pop("__action__", "created") - container = validated_iface.folder or validated_iface.snippet or validated_iface.device - if action == "created": - created_count += 1 - success(f"Created tunnel interface: {validated_iface.name} in {container}") - elif action == "updated": - updated_count += 1 - success(f"Updated tunnel interface: {validated_iface.name} in {container}") - else: - no_change_count += 1 - info(f"No changes needed for tunnel interface: {validated_iface.name} in {container}") - except Exception as e: - error(f"Error processing tunnel interface: {str(e)}") + + def _apply(iface_data: dict): + if folder: + iface_data["folder"] = folder + iface_data.pop("snippet", None) + iface_data.pop("device", None) + elif snippet: + iface_data["snippet"] = snippet + iface_data.pop("folder", None) + iface_data.pop("device", None) + elif device: + iface_data["device"] = device + iface_data.pop("folder", None) + iface_data.pop("snippet", None) + validated_iface = TunnelInterface(**iface_data) + sdk_data = validated_iface.to_sdk_model() + result = scm_client.create_tunnel_interface(sdk_data) + return validated_iface, result + + for _item, outcome, exc in run_bulk(interfaces, _apply): + if exc is not None: + error(f"Error processing tunnel interface: {str(exc)}") continue + validated_iface, result = outcome + action = result.pop("__action__", "created") + container = validated_iface.folder or validated_iface.snippet or validated_iface.device + if action == "created": + created_count += 1 + success(f"Created tunnel interface: {validated_iface.name} in {container}") + elif action == "updated": + updated_count += 1 + success(f"Updated tunnel interface: {validated_iface.name} in {container}") + else: + no_change_count += 1 + info(f"No changes needed for tunnel interface: {validated_iface.name} in {container}") info(f"\nSummary: Processed {created_count + updated_count + no_change_count} tunnel interfaces") if created_count > 0: info(f" - Created: {created_count}") @@ -2791,37 +2836,41 @@ def load_vlan_interface( created_count = 0 updated_count = 0 no_change_count = 0 - for iface_data in interfaces: - try: - if folder: - iface_data["folder"] = folder - iface_data.pop("snippet", None) - iface_data.pop("device", None) - elif snippet: - iface_data["snippet"] = snippet - iface_data.pop("folder", None) - iface_data.pop("device", None) - elif device: - iface_data["device"] = device - iface_data.pop("folder", None) - iface_data.pop("snippet", None) - validated_iface = VlanInterface(**iface_data) - sdk_data = validated_iface.to_sdk_model() - result = scm_client.create_vlan_interface(sdk_data) - action = result.pop("__action__", "created") - container = validated_iface.folder or validated_iface.snippet or validated_iface.device - if action == "created": - created_count += 1 - success(f"Created VLAN interface: {validated_iface.name} in {container}") - elif action == "updated": - updated_count += 1 - success(f"Updated VLAN interface: {validated_iface.name} in {container}") - else: - no_change_count += 1 - info(f"No changes needed for VLAN interface: {validated_iface.name} in {container}") - except Exception as e: - error(f"Error processing VLAN interface: {str(e)}") + + def _apply(iface_data: dict): + if folder: + iface_data["folder"] = folder + iface_data.pop("snippet", None) + iface_data.pop("device", None) + elif snippet: + iface_data["snippet"] = snippet + iface_data.pop("folder", None) + iface_data.pop("device", None) + elif device: + iface_data["device"] = device + iface_data.pop("folder", None) + iface_data.pop("snippet", None) + validated_iface = VlanInterface(**iface_data) + sdk_data = validated_iface.to_sdk_model() + result = scm_client.create_vlan_interface(sdk_data) + return validated_iface, result + + for _item, outcome, exc in run_bulk(interfaces, _apply): + if exc is not None: + error(f"Error processing VLAN interface: {str(exc)}") continue + validated_iface, result = outcome + action = result.pop("__action__", "created") + container = validated_iface.folder or validated_iface.snippet or validated_iface.device + if action == "created": + created_count += 1 + success(f"Created VLAN interface: {validated_iface.name} in {container}") + elif action == "updated": + updated_count += 1 + success(f"Updated VLAN interface: {validated_iface.name} in {container}") + else: + no_change_count += 1 + info(f"No changes needed for VLAN interface: {validated_iface.name} in {container}") info(f"\nSummary: Processed {created_count + updated_count + no_change_count} VLAN interfaces") if created_count > 0: info(f" - Created: {created_count}") @@ -2981,37 +3030,41 @@ def load_bgp_address_family_profile( created_count = 0 updated_count = 0 no_change_count = 0 - for profile_data in profiles: - try: - if folder: - profile_data["folder"] = folder - profile_data.pop("snippet", None) - profile_data.pop("device", None) - elif snippet: - profile_data["snippet"] = snippet - profile_data.pop("folder", None) - profile_data.pop("device", None) - elif device: - profile_data["device"] = device - profile_data.pop("folder", None) - profile_data.pop("snippet", None) - validated = BgpAddressFamilyProfile(**profile_data) - sdk_data = validated.to_sdk_model() - result = scm_client.create_bgp_address_family_profile(sdk_data) - action = result.pop("__action__", "created") - container = validated.folder or validated.snippet or validated.device - if action == "created": - created_count += 1 - success(f"Created BGP address family profile: {validated.name} in {container}") - elif action == "updated": - updated_count += 1 - success(f"Updated BGP address family profile: {validated.name} in {container}") - else: - no_change_count += 1 - info(f"No changes needed for BGP address family profile: {validated.name} in {container}") - except Exception as e: - error(f"Error processing BGP address family profile: {str(e)}") + + def _apply(profile_data: dict): + if folder: + profile_data["folder"] = folder + profile_data.pop("snippet", None) + profile_data.pop("device", None) + elif snippet: + profile_data["snippet"] = snippet + profile_data.pop("folder", None) + profile_data.pop("device", None) + elif device: + profile_data["device"] = device + profile_data.pop("folder", None) + profile_data.pop("snippet", None) + validated = BgpAddressFamilyProfile(**profile_data) + sdk_data = validated.to_sdk_model() + result = scm_client.create_bgp_address_family_profile(sdk_data) + return validated, result + + for _item, outcome, exc in run_bulk(profiles, _apply): + if exc is not None: + error(f"Error processing BGP address family profile: {str(exc)}") continue + validated, result = outcome + action = result.pop("__action__", "created") + container = validated.folder or validated.snippet or validated.device + if action == "created": + created_count += 1 + success(f"Created BGP address family profile: {validated.name} in {container}") + elif action == "updated": + updated_count += 1 + success(f"Updated BGP address family profile: {validated.name} in {container}") + else: + no_change_count += 1 + info(f"No changes needed for BGP address family profile: {validated.name} in {container}") info(f"\nSummary: Processed {created_count + updated_count + no_change_count} BGP address family profiles") if created_count > 0: info(f" - Created: {created_count}") @@ -3156,37 +3209,41 @@ def load_bgp_auth_profile( created_count = 0 updated_count = 0 no_change_count = 0 - for profile_data in profiles: - try: - if folder: - profile_data["folder"] = folder - profile_data.pop("snippet", None) - profile_data.pop("device", None) - elif snippet: - profile_data["snippet"] = snippet - profile_data.pop("folder", None) - profile_data.pop("device", None) - elif device: - profile_data["device"] = device - profile_data.pop("folder", None) - profile_data.pop("snippet", None) - validated = BgpAuthProfile(**profile_data) - sdk_data = validated.to_sdk_model() - result = scm_client.create_bgp_auth_profile(sdk_data) - action = result.pop("__action__", "created") - container = validated.folder or validated.snippet or validated.device - if action == "created": - created_count += 1 - success(f"Created BGP auth profile: {validated.name} in {container}") - elif action == "updated": - updated_count += 1 - success(f"Updated BGP auth profile: {validated.name} in {container}") - else: - no_change_count += 1 - info(f"No changes needed for BGP auth profile: {validated.name} in {container}") - except Exception as e: - error(f"Error processing BGP auth profile: {str(e)}") + + def _apply(profile_data: dict): + if folder: + profile_data["folder"] = folder + profile_data.pop("snippet", None) + profile_data.pop("device", None) + elif snippet: + profile_data["snippet"] = snippet + profile_data.pop("folder", None) + profile_data.pop("device", None) + elif device: + profile_data["device"] = device + profile_data.pop("folder", None) + profile_data.pop("snippet", None) + validated = BgpAuthProfile(**profile_data) + sdk_data = validated.to_sdk_model() + result = scm_client.create_bgp_auth_profile(sdk_data) + return validated, result + + for _item, outcome, exc in run_bulk(profiles, _apply): + if exc is not None: + error(f"Error processing BGP auth profile: {str(exc)}") continue + validated, result = outcome + action = result.pop("__action__", "created") + container = validated.folder or validated.snippet or validated.device + if action == "created": + created_count += 1 + success(f"Created BGP auth profile: {validated.name} in {container}") + elif action == "updated": + updated_count += 1 + success(f"Updated BGP auth profile: {validated.name} in {container}") + else: + no_change_count += 1 + info(f"No changes needed for BGP auth profile: {validated.name} in {container}") info(f"\nSummary: Processed {created_count + updated_count + no_change_count} BGP auth profiles") @@ -3319,37 +3376,41 @@ def load_ospf_auth_profile( created_count = 0 updated_count = 0 no_change_count = 0 - for profile_data in profiles: - try: - if folder: - profile_data["folder"] = folder - profile_data.pop("snippet", None) - profile_data.pop("device", None) - elif snippet: - profile_data["snippet"] = snippet - profile_data.pop("folder", None) - profile_data.pop("device", None) - elif device: - profile_data["device"] = device - profile_data.pop("folder", None) - profile_data.pop("snippet", None) - validated = OspfAuthProfile(**profile_data) - sdk_data = validated.to_sdk_model() - result = scm_client.create_ospf_auth_profile(sdk_data) - action = result.pop("__action__", "created") - container = validated.folder or validated.snippet or validated.device - if action == "created": - created_count += 1 - success(f"Created OSPF auth profile: {validated.name} in {container}") - elif action == "updated": - updated_count += 1 - success(f"Updated OSPF auth profile: {validated.name} in {container}") - else: - no_change_count += 1 - info(f"No changes needed for OSPF auth profile: {validated.name} in {container}") - except Exception as e: - error(f"Error processing OSPF auth profile: {str(e)}") + + def _apply(profile_data: dict): + if folder: + profile_data["folder"] = folder + profile_data.pop("snippet", None) + profile_data.pop("device", None) + elif snippet: + profile_data["snippet"] = snippet + profile_data.pop("folder", None) + profile_data.pop("device", None) + elif device: + profile_data["device"] = device + profile_data.pop("folder", None) + profile_data.pop("snippet", None) + validated = OspfAuthProfile(**profile_data) + sdk_data = validated.to_sdk_model() + result = scm_client.create_ospf_auth_profile(sdk_data) + return validated, result + + for _item, outcome, exc in run_bulk(profiles, _apply): + if exc is not None: + error(f"Error processing OSPF auth profile: {str(exc)}") continue + validated, result = outcome + action = result.pop("__action__", "created") + container = validated.folder or validated.snippet or validated.device + if action == "created": + created_count += 1 + success(f"Created OSPF auth profile: {validated.name} in {container}") + elif action == "updated": + updated_count += 1 + success(f"Updated OSPF auth profile: {validated.name} in {container}") + else: + no_change_count += 1 + info(f"No changes needed for OSPF auth profile: {validated.name} in {container}") info(f"\nSummary: Processed {created_count + updated_count + no_change_count} OSPF auth profiles") @@ -3491,37 +3552,41 @@ def load_route_access_list( created_count = 0 updated_count = 0 no_change_count = 0 - for item_data in items: - try: - if folder: - item_data["folder"] = folder - item_data.pop("snippet", None) - item_data.pop("device", None) - elif snippet: - item_data["snippet"] = snippet - item_data.pop("folder", None) - item_data.pop("device", None) - elif device: - item_data["device"] = device - item_data.pop("folder", None) - item_data.pop("snippet", None) - validated = RouteAccessList(**item_data) - sdk_data = validated.to_sdk_model() - result = scm_client.create_route_access_list(sdk_data) - action = result.pop("__action__", "created") - container = validated.folder or validated.snippet or validated.device - if action == "created": - created_count += 1 - success(f"Created route access list: {validated.name} in {container}") - elif action == "updated": - updated_count += 1 - success(f"Updated route access list: {validated.name} in {container}") - else: - no_change_count += 1 - info(f"No changes needed for route access list: {validated.name} in {container}") - except Exception as e: - error(f"Error processing route access list: {str(e)}") + + def _apply(item_data: dict): + if folder: + item_data["folder"] = folder + item_data.pop("snippet", None) + item_data.pop("device", None) + elif snippet: + item_data["snippet"] = snippet + item_data.pop("folder", None) + item_data.pop("device", None) + elif device: + item_data["device"] = device + item_data.pop("folder", None) + item_data.pop("snippet", None) + validated = RouteAccessList(**item_data) + sdk_data = validated.to_sdk_model() + result = scm_client.create_route_access_list(sdk_data) + return validated, result + + for _item, outcome, exc in run_bulk(items, _apply): + if exc is not None: + error(f"Error processing route access list: {str(exc)}") continue + validated, result = outcome + action = result.pop("__action__", "created") + container = validated.folder or validated.snippet or validated.device + if action == "created": + created_count += 1 + success(f"Created route access list: {validated.name} in {container}") + elif action == "updated": + updated_count += 1 + success(f"Updated route access list: {validated.name} in {container}") + else: + no_change_count += 1 + info(f"No changes needed for route access list: {validated.name} in {container}") info(f"\nSummary: Processed {created_count + updated_count + no_change_count} route access lists") @@ -3663,37 +3728,41 @@ def load_route_prefix_list( created_count = 0 updated_count = 0 no_change_count = 0 - for item_data in items: - try: - if folder: - item_data["folder"] = folder - item_data.pop("snippet", None) - item_data.pop("device", None) - elif snippet: - item_data["snippet"] = snippet - item_data.pop("folder", None) - item_data.pop("device", None) - elif device: - item_data["device"] = device - item_data.pop("folder", None) - item_data.pop("snippet", None) - validated = RoutePrefixList(**item_data) - sdk_data = validated.to_sdk_model() - result = scm_client.create_route_prefix_list(sdk_data) - action = result.pop("__action__", "created") - container = validated.folder or validated.snippet or validated.device - if action == "created": - created_count += 1 - success(f"Created route prefix list: {validated.name} in {container}") - elif action == "updated": - updated_count += 1 - success(f"Updated route prefix list: {validated.name} in {container}") - else: - no_change_count += 1 - info(f"No changes needed for route prefix list: {validated.name} in {container}") - except Exception as e: - error(f"Error processing route prefix list: {str(e)}") + + def _apply(item_data: dict): + if folder: + item_data["folder"] = folder + item_data.pop("snippet", None) + item_data.pop("device", None) + elif snippet: + item_data["snippet"] = snippet + item_data.pop("folder", None) + item_data.pop("device", None) + elif device: + item_data["device"] = device + item_data.pop("folder", None) + item_data.pop("snippet", None) + validated = RoutePrefixList(**item_data) + sdk_data = validated.to_sdk_model() + result = scm_client.create_route_prefix_list(sdk_data) + return validated, result + + for _item, outcome, exc in run_bulk(items, _apply): + if exc is not None: + error(f"Error processing route prefix list: {str(exc)}") continue + validated, result = outcome + action = result.pop("__action__", "created") + container = validated.folder or validated.snippet or validated.device + if action == "created": + created_count += 1 + success(f"Created route prefix list: {validated.name} in {container}") + elif action == "updated": + updated_count += 1 + success(f"Updated route prefix list: {validated.name} in {container}") + else: + no_change_count += 1 + info(f"No changes needed for route prefix list: {validated.name} in {container}") info(f"\nSummary: Processed {created_count + updated_count + no_change_count} route prefix lists") @@ -3833,30 +3902,34 @@ def load_bgp_filtering_profile( info(f" Would process: {p.get('name', 'N/A')}") return created_count = 0 - for profile_data in profiles: - try: - if folder: - profile_data["folder"] = folder - profile_data.pop("snippet", None) - profile_data.pop("device", None) - elif snippet: - profile_data["snippet"] = snippet - profile_data.pop("folder", None) - profile_data.pop("device", None) - elif device: - profile_data["device"] = device - profile_data.pop("folder", None) - profile_data.pop("snippet", None) - validated = BgpFilteringProfile(**profile_data) - sdk_data = validated.to_sdk_model() - result = scm_client.create_bgp_filtering_profile(sdk_data) - action = result.pop("__action__", "created") - container = validated.folder or validated.snippet or validated.device - created_count += 1 - success(f"{action.capitalize()} BGP filtering profile: {validated.name} in {container}") - except Exception as e: - error(f"Error processing BGP filtering profile: {str(e)}") + + def _apply(profile_data: dict): + if folder: + profile_data["folder"] = folder + profile_data.pop("snippet", None) + profile_data.pop("device", None) + elif snippet: + profile_data["snippet"] = snippet + profile_data.pop("folder", None) + profile_data.pop("device", None) + elif device: + profile_data["device"] = device + profile_data.pop("folder", None) + profile_data.pop("snippet", None) + validated = BgpFilteringProfile(**profile_data) + sdk_data = validated.to_sdk_model() + result = scm_client.create_bgp_filtering_profile(sdk_data) + return validated, result + + for _item, outcome, exc in run_bulk(profiles, _apply): + if exc is not None: + error(f"Error processing BGP filtering profile: {str(exc)}") continue + validated, result = outcome + action = result.pop("__action__", "created") + container = validated.folder or validated.snippet or validated.device + created_count += 1 + success(f"{action.capitalize()} BGP filtering profile: {validated.name} in {container}") info(f"\nSummary: Processed {created_count} BGP filtering profiles") @@ -3993,30 +4066,34 @@ def load_bgp_redistribution_profile( info(f" Would process: {p.get('name', 'N/A')}") return created_count = 0 - for profile_data in profiles: - try: - if folder: - profile_data["folder"] = folder - profile_data.pop("snippet", None) - profile_data.pop("device", None) - elif snippet: - profile_data["snippet"] = snippet - profile_data.pop("folder", None) - profile_data.pop("device", None) - elif device: - profile_data["device"] = device - profile_data.pop("folder", None) - profile_data.pop("snippet", None) - validated = BgpRedistributionProfile(**profile_data) - sdk_data = validated.to_sdk_model() - result = scm_client.create_bgp_redistribution_profile(sdk_data) - action = result.pop("__action__", "created") - container = validated.folder or validated.snippet or validated.device - created_count += 1 - success(f"{action.capitalize()} BGP redistribution profile: {validated.name} in {container}") - except Exception as e: - error(f"Error processing BGP redistribution profile: {str(e)}") + + def _apply(profile_data: dict): + if folder: + profile_data["folder"] = folder + profile_data.pop("snippet", None) + profile_data.pop("device", None) + elif snippet: + profile_data["snippet"] = snippet + profile_data.pop("folder", None) + profile_data.pop("device", None) + elif device: + profile_data["device"] = device + profile_data.pop("folder", None) + profile_data.pop("snippet", None) + validated = BgpRedistributionProfile(**profile_data) + sdk_data = validated.to_sdk_model() + result = scm_client.create_bgp_redistribution_profile(sdk_data) + return validated, result + + for _item, outcome, exc in run_bulk(profiles, _apply): + if exc is not None: + error(f"Error processing BGP redistribution profile: {str(exc)}") continue + validated, result = outcome + action = result.pop("__action__", "created") + container = validated.folder or validated.snippet or validated.device + created_count += 1 + success(f"{action.capitalize()} BGP redistribution profile: {validated.name} in {container}") info(f"\nSummary: Processed {created_count} BGP redistribution profiles") @@ -4153,30 +4230,34 @@ def load_bgp_route_map( info(f" Would process: {item.get('name', 'N/A')}") return created_count = 0 - for item_data in items: - try: - if folder: - item_data["folder"] = folder - item_data.pop("snippet", None) - item_data.pop("device", None) - elif snippet: - item_data["snippet"] = snippet - item_data.pop("folder", None) - item_data.pop("device", None) - elif device: - item_data["device"] = device - item_data.pop("folder", None) - item_data.pop("snippet", None) - validated = BgpRouteMap(**item_data) - sdk_data = validated.to_sdk_model() - result = scm_client.create_bgp_route_map(sdk_data) - action = result.pop("__action__", "created") - container = validated.folder or validated.snippet or validated.device - created_count += 1 - success(f"{action.capitalize()} BGP route map: {validated.name} in {container}") - except Exception as e: - error(f"Error processing BGP route map: {str(e)}") + + def _apply(item_data: dict): + if folder: + item_data["folder"] = folder + item_data.pop("snippet", None) + item_data.pop("device", None) + elif snippet: + item_data["snippet"] = snippet + item_data.pop("folder", None) + item_data.pop("device", None) + elif device: + item_data["device"] = device + item_data.pop("folder", None) + item_data.pop("snippet", None) + validated = BgpRouteMap(**item_data) + sdk_data = validated.to_sdk_model() + result = scm_client.create_bgp_route_map(sdk_data) + return validated, result + + for _item, outcome, exc in run_bulk(items, _apply): + if exc is not None: + error(f"Error processing BGP route map: {str(exc)}") continue + validated, result = outcome + action = result.pop("__action__", "created") + container = validated.folder or validated.snippet or validated.device + created_count += 1 + success(f"{action.capitalize()} BGP route map: {validated.name} in {container}") info(f"\nSummary: Processed {created_count} BGP route maps") @@ -4313,30 +4394,34 @@ def load_bgp_route_map_redistribution( info(f" Would process: {item.get('name', 'N/A')}") return created_count = 0 - for item_data in items: - try: - if folder: - item_data["folder"] = folder - item_data.pop("snippet", None) - item_data.pop("device", None) - elif snippet: - item_data["snippet"] = snippet - item_data.pop("folder", None) - item_data.pop("device", None) - elif device: - item_data["device"] = device - item_data.pop("folder", None) - item_data.pop("snippet", None) - validated = BgpRouteMapRedistribution(**item_data) - sdk_data = validated.to_sdk_model() - result = scm_client.create_bgp_route_map_redistribution(sdk_data) - action = result.pop("__action__", "created") - container = validated.folder or validated.snippet or validated.device - created_count += 1 - success(f"{action.capitalize()} BGP route map redistribution: {validated.name} in {container}") - except Exception as e: - error(f"Error processing BGP route map redistribution: {str(e)}") + + def _apply(item_data: dict): + if folder: + item_data["folder"] = folder + item_data.pop("snippet", None) + item_data.pop("device", None) + elif snippet: + item_data["snippet"] = snippet + item_data.pop("folder", None) + item_data.pop("device", None) + elif device: + item_data["device"] = device + item_data.pop("folder", None) + item_data.pop("snippet", None) + validated = BgpRouteMapRedistribution(**item_data) + sdk_data = validated.to_sdk_model() + result = scm_client.create_bgp_route_map_redistribution(sdk_data) + return validated, result + + for _item, outcome, exc in run_bulk(items, _apply): + if exc is not None: + error(f"Error processing BGP route map redistribution: {str(exc)}") continue + validated, result = outcome + action = result.pop("__action__", "created") + container = validated.folder or validated.snippet or validated.device + created_count += 1 + success(f"{action.capitalize()} BGP route map redistribution: {validated.name} in {container}") info(f"\nSummary: Processed {created_count} BGP route map redistributions") @@ -4481,37 +4566,41 @@ def load_dns_proxy( created_count = 0 updated_count = 0 no_change_count = 0 - for proxy_data in proxies: - try: - if folder: - proxy_data["folder"] = folder - proxy_data.pop("snippet", None) - proxy_data.pop("device", None) - elif snippet: - proxy_data["snippet"] = snippet - proxy_data.pop("folder", None) - proxy_data.pop("device", None) - elif device: - proxy_data["device"] = device - proxy_data.pop("folder", None) - proxy_data.pop("snippet", None) - validated = DnsProxy(**proxy_data) - sdk_data = validated.to_sdk_model() - result = scm_client.create_dns_proxy(sdk_data) - action = result.pop("__action__", "created") - container = validated.folder or validated.snippet or validated.device - if action == "created": - created_count += 1 - success(f"Created DNS proxy: {validated.name} in {container}") - elif action == "updated": - updated_count += 1 - success(f"Updated DNS proxy: {validated.name} in {container}") - else: - no_change_count += 1 - info(f"No changes needed for DNS proxy: {validated.name} in {container}") - except Exception as e: - error(f"Error processing DNS proxy: {str(e)}") + + def _apply(proxy_data: dict): + if folder: + proxy_data["folder"] = folder + proxy_data.pop("snippet", None) + proxy_data.pop("device", None) + elif snippet: + proxy_data["snippet"] = snippet + proxy_data.pop("folder", None) + proxy_data.pop("device", None) + elif device: + proxy_data["device"] = device + proxy_data.pop("folder", None) + proxy_data.pop("snippet", None) + validated = DnsProxy(**proxy_data) + sdk_data = validated.to_sdk_model() + result = scm_client.create_dns_proxy(sdk_data) + return validated, result + + for _item, outcome, exc in run_bulk(proxies, _apply): + if exc is not None: + error(f"Error processing DNS proxy: {str(exc)}") continue + validated, result = outcome + action = result.pop("__action__", "created") + container = validated.folder or validated.snippet or validated.device + if action == "created": + created_count += 1 + success(f"Created DNS proxy: {validated.name} in {container}") + elif action == "updated": + updated_count += 1 + success(f"Updated DNS proxy: {validated.name} in {container}") + else: + no_change_count += 1 + info(f"No changes needed for DNS proxy: {validated.name} in {container}") info(f"\nSummary: Processed {created_count + updated_count + no_change_count} DNS proxies") if created_count > 0: info(f" - Created: {created_count}") @@ -4662,6 +4751,7 @@ def load_pbf_rule( created_count = 0 updated_count = 0 no_change_count = 0 + # sequential: rule order matters for rule_data in rules: try: if folder: @@ -4845,37 +4935,41 @@ def load_qos_profile( created_count = 0 updated_count = 0 no_change_count = 0 - for profile_data in profiles: - try: - if folder: - profile_data["folder"] = folder - profile_data.pop("snippet", None) - profile_data.pop("device", None) - elif snippet: - profile_data["snippet"] = snippet - profile_data.pop("folder", None) - profile_data.pop("device", None) - elif device: - profile_data["device"] = device - profile_data.pop("folder", None) - profile_data.pop("snippet", None) - validated = QosProfile(**profile_data) - sdk_data = validated.to_sdk_model() - result = scm_client.create_qos_profile(sdk_data) - action = result.pop("__action__", "created") - container = validated.folder or validated.snippet or validated.device - if action == "created": - created_count += 1 - success(f"Created QoS profile: {validated.name} in {container}") - elif action == "updated": - updated_count += 1 - success(f"Updated QoS profile: {validated.name} in {container}") - else: - no_change_count += 1 - info(f"No changes needed for QoS profile: {validated.name} in {container}") - except Exception as e: - error(f"Error processing QoS profile: {str(e)}") + + def _apply(profile_data: dict): + if folder: + profile_data["folder"] = folder + profile_data.pop("snippet", None) + profile_data.pop("device", None) + elif snippet: + profile_data["snippet"] = snippet + profile_data.pop("folder", None) + profile_data.pop("device", None) + elif device: + profile_data["device"] = device + profile_data.pop("folder", None) + profile_data.pop("snippet", None) + validated = QosProfile(**profile_data) + sdk_data = validated.to_sdk_model() + result = scm_client.create_qos_profile(sdk_data) + return validated, result + + for _item, outcome, exc in run_bulk(profiles, _apply): + if exc is not None: + error(f"Error processing QoS profile: {str(exc)}") continue + validated, result = outcome + action = result.pop("__action__", "created") + container = validated.folder or validated.snippet or validated.device + if action == "created": + created_count += 1 + success(f"Created QoS profile: {validated.name} in {container}") + elif action == "updated": + updated_count += 1 + success(f"Updated QoS profile: {validated.name} in {container}") + else: + no_change_count += 1 + info(f"No changes needed for QoS profile: {validated.name} in {container}") info(f"\nSummary: Processed {created_count + updated_count + no_change_count} QoS profiles") if created_count > 0: info(f" - Created: {created_count}") @@ -5025,6 +5119,7 @@ def load_qos_rule( created_count = 0 updated_count = 0 no_change_count = 0 + # sequential: rule order matters for rule_data in rules: try: if folder: diff --git a/src/scm_cli/commands/objects.py b/src/scm_cli/commands/objects.py index 3aa60bd..1f013ac 100644 --- a/src/scm_cli/commands/objects.py +++ b/src/scm_cli/commands/objects.py @@ -12,6 +12,7 @@ # Removed unused import: from the `..utils.config` import load_from_yaml from ..utils import parse_comma_separated_list, validate_location_params +from ..utils.bulk import run_bulk from ..utils.decorators import handle_command_errors from ..utils.output import OUTPUT_OPTION, OutputFormat, emit, error, info, redact, success, warning from ..utils.sdk_client import scm_client @@ -41,6 +42,9 @@ # HELPER FUNCTIONS # ============================================================================================================================================================================================= +# Sentinel returned by bulk-load workers for items skipped due to unsupported container overrides +_SKIPPED = object() + # ============================================================================================================================================================================================= # TYPER APP CONFIGURATION @@ -559,49 +563,54 @@ def load_address_group( return # Apply each address group + def _apply(ag_data: dict): + # Apply container override if specified + if folder: + ag_data["folder"] = folder + ag_data.pop("snippet", None) + ag_data.pop("device", None) + elif snippet or device: + return _SKIPPED + + # Validate using the Pydantic model + address_group = AddressGroup(**ag_data) + + # Call the SDK client to create the address group + return scm_client.create_address_group( + folder=address_group.folder, + name=address_group.name, + type=address_group.type, + members=address_group.members, + description=address_group.description, + tags=address_group.tags, + ) + + outcomes = run_bulk(address_groups, _apply) + results: list[dict[str, Any]] = [] created_count = 0 updated_count = 0 - for ag_data in address_groups: - try: - # Apply container override if specified - if folder: - ag_data["folder"] = folder - ag_data.pop("snippet", None) - ag_data.pop("device", None) - elif snippet: + for ag_data, result, exc in outcomes: + if exc is not None: + error(f"Error processing address group '{ag_data.get('name', 'unknown')}': {str(exc)}") + # Continue processing other objects + continue + + if result is _SKIPPED: + if snippet: warning(f"Warning: Address groups do not support snippets. Skipping group '{ag_data.get('name', 'unknown')}'") - continue - elif device: + else: warning(f"Warning: Address groups do not support devices. Skipping group '{ag_data.get('name', 'unknown')}'") - continue - - # Validate using the Pydantic model - address_group = AddressGroup(**ag_data) - - # Call the SDK client to create the address group - result = scm_client.create_address_group( - folder=address_group.folder, - name=address_group.name, - type=address_group.type, - members=address_group.members, - description=address_group.description, - tags=address_group.tags, - ) - - results.append(result) + continue - # Track if created or updated based on response - if "created" in str(result).lower(): - created_count += 1 - else: - updated_count += 1 + results.append(result) - except Exception as e: - error(f"Error processing address group '{ag_data.get('name', 'unknown')}': {str(e)}") - # Continue processing other objects - continue + # Track if created or updated based on response + if "created" in str(result).lower(): + created_count += 1 + else: + updated_count += 1 # Display summary with counts success(f"Successfully processed {len(results)} address group(s):") @@ -856,54 +865,56 @@ def load_address( return # Apply each address + def _apply(addr_data: dict): + # Apply container override if specified + if folder: + addr_data["folder"] = folder + addr_data.pop("snippet", None) + addr_data.pop("device", None) + elif snippet: + addr_data["snippet"] = snippet + addr_data.pop("folder", None) + addr_data.pop("device", None) + elif device: + addr_data["device"] = device + addr_data.pop("folder", None) + addr_data.pop("snippet", None) + + # Validate using the Pydantic model + address = Address(**addr_data) + + # Call the SDK client to create the address + return scm_client.create_address( + folder=address.folder, + name=address.name, + description=address.description, + tags=address.tags, + ip_netmask=address.ip_netmask, + ip_range=address.ip_range, + ip_wildcard=address.ip_wildcard, + fqdn=address.fqdn, + ) + + outcomes = run_bulk(addresses, _apply) + results: list[dict[str, Any]] = [] created_count = 0 updated_count = 0 - for addr_data in addresses: - try: - # Apply container override if specified - if folder: - addr_data["folder"] = folder - addr_data.pop("snippet", None) - addr_data.pop("device", None) - elif snippet: - addr_data["snippet"] = snippet - addr_data.pop("folder", None) - addr_data.pop("device", None) - elif device: - addr_data["device"] = device - addr_data.pop("folder", None) - addr_data.pop("snippet", None) - - # Validate using the Pydantic model - address = Address(**addr_data) - - # Call the SDK client to create the address - result = scm_client.create_address( - folder=address.folder, - name=address.name, - description=address.description, - tags=address.tags, - ip_netmask=address.ip_netmask, - ip_range=address.ip_range, - ip_wildcard=address.ip_wildcard, - fqdn=address.fqdn, - ) - - results.append(result) - - # Track if created or updated based on response - if "created" in str(result).lower(): - created_count += 1 - else: - updated_count += 1 - - except Exception as e: - error(f"Error processing address '{addr_data.get('name', 'unknown')}': {str(e)}") + for addr_data, result, exc in outcomes: + if exc is not None: + error(f"Error processing address '{addr_data.get('name', 'unknown')}': {str(exc)}") # Continue processing other addresses continue + results.append(result) + + # Track if created or updated based on response + if "created" in str(result).lower(): + created_count += 1 + else: + updated_count += 1 + # Display summary with counts success(f"Successfully processed {len(results)} address(es):") if created_count > 0: @@ -1162,60 +1173,65 @@ def load_application( return # Apply each application + def _apply(app_data: dict): + # Apply container override if specified + if folder: + app_data["folder"] = folder + app_data.pop("snippet", None) + app_data.pop("device", None) + elif snippet or device: + return _SKIPPED + + # Validate using the Pydantic model + application = Application(**app_data) + + # Call the SDK client to create the application + return scm_client.create_application( + folder=application.folder, + name=application.name, + category=application.category, + subcategory=application.subcategory, + technology=application.technology, + risk=application.risk, + description=application.description, + ports=application.ports, + evasive=application.evasive, + pervasive=application.pervasive, + excessive_bandwidth_use=application.excessive_bandwidth_use, + used_by_malware=application.used_by_malware, + transfers_files=application.transfers_files, + has_known_vulnerabilities=application.has_known_vulnerabilities, + tunnels_other_apps=application.tunnels_other_apps, + prone_to_misuse=application.prone_to_misuse, + no_certifications=application.no_certifications, + ) + + outcomes = run_bulk(applications, _apply) + results: list[dict[str, Any]] = [] created_count = 0 updated_count = 0 - for app_data in applications: - try: - # Apply container override if specified - if folder: - app_data["folder"] = folder - app_data.pop("snippet", None) - app_data.pop("device", None) - elif snippet: + for app_data, result, exc in outcomes: + if exc is not None: + error(f"Error processing application '{app_data.get('name', 'unknown')}': {str(exc)}") + # Continue processing other objects + continue + + if result is _SKIPPED: + if snippet: warning(f"Warning: Applications do not support snippets. Skipping application '{app_data.get('name', 'unknown')}'") - continue - elif device: + else: warning(f"Warning: Applications do not support devices. Skipping application '{app_data.get('name', 'unknown')}'") - continue - - # Validate using the Pydantic model - application = Application(**app_data) - - # Call the SDK client to create the application - result = scm_client.create_application( - folder=application.folder, - name=application.name, - category=application.category, - subcategory=application.subcategory, - technology=application.technology, - risk=application.risk, - description=application.description, - ports=application.ports, - evasive=application.evasive, - pervasive=application.pervasive, - excessive_bandwidth_use=application.excessive_bandwidth_use, - used_by_malware=application.used_by_malware, - transfers_files=application.transfers_files, - has_known_vulnerabilities=application.has_known_vulnerabilities, - tunnels_other_apps=application.tunnels_other_apps, - prone_to_misuse=application.prone_to_misuse, - no_certifications=application.no_certifications, - ) - - results.append(result) + continue - # Track if created or updated based on response - if "created" in str(result).lower(): - created_count += 1 - else: - updated_count += 1 + results.append(result) - except Exception as e: - error(f"Error processing application '{app_data.get('name', 'unknown')}': {str(e)}") - # Continue processing other objects - continue + # Track if created or updated based on response + if "created" in str(result).lower(): + created_count += 1 + else: + updated_count += 1 # Display summary with counts success(f"Successfully processed {len(results)} application(s):") @@ -1475,43 +1491,45 @@ def load_application_group( return # Apply each application group + def _apply(group_data: dict): + # Apply container overrides if specified + if folder: + group_data["folder"] = folder + group_data.pop("snippet", None) + group_data.pop("device", None) + elif snippet: + group_data["snippet"] = snippet + group_data.pop("folder", None) + group_data.pop("device", None) + elif device: + group_data["device"] = device + group_data.pop("folder", None) + group_data.pop("snippet", None) + + # Validate using the Pydantic model + app_group = ApplicationGroup(**group_data) + + # Call the SDK client to create the application group + return scm_client.create_application_group( + folder=app_group.folder, + name=app_group.name, + members=app_group.members, + ) + + outcomes = run_bulk(application_groups, _apply) + results: list[dict[str, Any]] = [] created_count = 0 updated_count = 0 - for group_data in application_groups: - try: - # Apply container overrides if specified - if folder: - group_data["folder"] = folder - group_data.pop("snippet", None) - group_data.pop("device", None) - elif snippet: - group_data["snippet"] = snippet - group_data.pop("folder", None) - group_data.pop("device", None) - elif device: - group_data["device"] = device - group_data.pop("folder", None) - group_data.pop("snippet", None) - - # Validate using the Pydantic model - app_group = ApplicationGroup(**group_data) - - # Call the SDK client to create the application group - result = scm_client.create_application_group( - folder=app_group.folder, - name=app_group.name, - members=app_group.members, - ) - - results.append(result) - created_count += 1 - - except Exception as e: - error(f"Error processing application group '{group_data.get('name', 'unknown')}': {str(e)}") + for group_data, result, exc in outcomes: + if exc is not None: + error(f"Error processing application group '{group_data.get('name', 'unknown')}': {str(exc)}") continue + results.append(result) + created_count += 1 + # Display summary with counts success(f"Successfully processed {len(results)} application group(s)") if created_count > 0: @@ -1724,55 +1742,57 @@ def load_application_filter( return # Apply each application filter + def _apply(filter_data: dict): + # Apply container overrides if specified + if folder: + filter_data["folder"] = folder + filter_data.pop("snippet", None) + filter_data.pop("device", None) + elif snippet: + filter_data["snippet"] = snippet + filter_data.pop("folder", None) + filter_data.pop("device", None) + elif device: + filter_data["device"] = device + filter_data.pop("folder", None) + filter_data.pop("snippet", None) + + # Validate using the Pydantic model + app_filter = ApplicationFilter(**filter_data) + + # Call the SDK client to create the application filter + return scm_client.create_application_filter( + folder=app_filter.folder, + name=app_filter.name, + category=app_filter.category, + subcategory=app_filter.subcategory, + technology=app_filter.technology, + risk=app_filter.risk, + evasive=app_filter.evasive, + pervasive=app_filter.pervasive, + excessive_bandwidth_use=app_filter.excessive_bandwidth_use, + used_by_malware=app_filter.used_by_malware, + transfers_files=app_filter.transfers_files, + has_known_vulnerabilities=app_filter.has_known_vulnerabilities, + tunnels_other_apps=app_filter.tunnels_other_apps, + prone_to_misuse=app_filter.prone_to_misuse, + no_certifications=app_filter.no_certifications, + ) + + outcomes = run_bulk(application_filters, _apply) + results: list[dict[str, Any]] = [] created_count = 0 updated_count = 0 - for filter_data in application_filters: - try: - # Apply container overrides if specified - if folder: - filter_data["folder"] = folder - filter_data.pop("snippet", None) - filter_data.pop("device", None) - elif snippet: - filter_data["snippet"] = snippet - filter_data.pop("folder", None) - filter_data.pop("device", None) - elif device: - filter_data["device"] = device - filter_data.pop("folder", None) - filter_data.pop("snippet", None) - - # Validate using the Pydantic model - app_filter = ApplicationFilter(**filter_data) - - # Call the SDK client to create the application filter - result = scm_client.create_application_filter( - folder=app_filter.folder, - name=app_filter.name, - category=app_filter.category, - subcategory=app_filter.subcategory, - technology=app_filter.technology, - risk=app_filter.risk, - evasive=app_filter.evasive, - pervasive=app_filter.pervasive, - excessive_bandwidth_use=app_filter.excessive_bandwidth_use, - used_by_malware=app_filter.used_by_malware, - transfers_files=app_filter.transfers_files, - has_known_vulnerabilities=app_filter.has_known_vulnerabilities, - tunnels_other_apps=app_filter.tunnels_other_apps, - prone_to_misuse=app_filter.prone_to_misuse, - no_certifications=app_filter.no_certifications, - ) - - results.append(result) - created_count += 1 - - except Exception as e: - error(f"Error processing application filter '{filter_data.get('name', 'unknown')}': {str(e)}") + for filter_data, result, exc in outcomes: + if exc is not None: + error(f"Error processing application filter '{filter_data.get('name', 'unknown')}': {str(exc)}") continue + results.append(result) + created_count += 1 + # Display summary with counts success(f"Successfully processed {len(results)} application filter(s)") if created_count > 0: @@ -2033,45 +2053,47 @@ def load_dynamic_user_group( return # Apply each dynamic user group + def _apply(group_data: dict): + # Apply container overrides if specified + if folder: + group_data["folder"] = folder + group_data.pop("snippet", None) + group_data.pop("device", None) + elif snippet: + group_data["snippet"] = snippet + group_data.pop("folder", None) + group_data.pop("device", None) + elif device: + group_data["device"] = device + group_data.pop("folder", None) + group_data.pop("snippet", None) + + # Validate using the Pydantic model + dug = DynamicUserGroup(**group_data) + + # Call the SDK client to create the dynamic user group + return scm_client.create_dynamic_user_group( + folder=dug.folder, + name=dug.name, + filter=dug.filter, + description=dug.description, + tags=dug.tags, + ) + + outcomes = run_bulk(dynamic_user_groups, _apply) + results: list[dict[str, Any]] = [] created_count = 0 updated_count = 0 - for group_data in dynamic_user_groups: - try: - # Apply container overrides if specified - if folder: - group_data["folder"] = folder - group_data.pop("snippet", None) - group_data.pop("device", None) - elif snippet: - group_data["snippet"] = snippet - group_data.pop("folder", None) - group_data.pop("device", None) - elif device: - group_data["device"] = device - group_data.pop("folder", None) - group_data.pop("snippet", None) - - # Validate using the Pydantic model - dug = DynamicUserGroup(**group_data) - - # Call the SDK client to create the dynamic user group - result = scm_client.create_dynamic_user_group( - folder=dug.folder, - name=dug.name, - filter=dug.filter, - description=dug.description, - tags=dug.tags, - ) - - results.append(result) - created_count += 1 - - except Exception as e: - error(f"Error processing dynamic user group '{group_data.get('name', 'unknown')}': {str(e)}") + for group_data, result, exc in outcomes: + if exc is not None: + error(f"Error processing dynamic user group '{group_data.get('name', 'unknown')}': {str(exc)}") continue + results.append(result) + created_count += 1 + # Display summary with counts success(f"Successfully processed {len(results)} dynamic user group(s)") if created_count > 0: @@ -2319,20 +2341,20 @@ def load_external_dynamic_list( results: list[dict[str, Any]] = [] loaded_count = 0 - for idx, edl_config in enumerate(external_dynamic_lists, 1): - try: - # Override container if specified in command line - if location_value: - edl_config[location_type] = location_value - # Remove other container fields - for container in ["folder", "snippet", "device"]: - if container != location_type and container in edl_config: - del edl_config[container] - - # Validate the configuration - edl = ExternalDynamicList(**edl_config) + if dry_run: + for idx, edl_config in enumerate(external_dynamic_lists, 1): + try: + # Override container if specified in command line + if location_value: + edl_config[location_type] = location_value + # Remove other container fields + for container in ["folder", "snippet", "device"]: + if container != location_type and container in edl_config: + del edl_config[container] + + # Validate the configuration + edl = ExternalDynamicList(**edl_config) - if dry_run: typer.echo(f"\n[{idx}] External Dynamic List: {edl.name}") typer.echo(f" Container: {getattr(edl, location_type or 'folder')}") typer.echo(f" Type: {edl.type}") @@ -2342,42 +2364,70 @@ def load_external_dynamic_list( if edl.recurring: typer.echo(f" Update Frequency: {edl.recurring}") results.append({"action": "would create/update", "name": edl.name}) - else: - # Convert to SDK model format - sdk_data = edl.to_sdk_model() - - # Extract container params - container_params = {} - if "folder" in edl_config: - container_params["folder"] = edl_config["folder"] - elif "snippet" in edl_config: - container_params["snippet"] = edl_config["snippet"] - elif "device" in edl_config: - container_params["device"] = edl_config["device"] - # Create the EDL using the SDK data - result = scm_client.create_external_dynamic_list( - **container_params, - **sdk_data, + except Exception as e: + error(f"Error with external dynamic list '{edl_config.get('name', 'unknown')}': {str(e)}") + results.append( + { + "action": "error", + "name": edl_config.get("name", "unknown"), + "error": str(e), + } ) - success(f"Loaded external dynamic list: {edl.name}") - loaded_count += 1 + continue + else: + + def _apply(edl_config: dict): + # Override container if specified in command line + if location_value: + edl_config[location_type] = location_value + # Remove other container fields + for container in ["folder", "snippet", "device"]: + if container != location_type and container in edl_config: + del edl_config[container] + + # Validate the configuration + edl = ExternalDynamicList(**edl_config) + + # Convert to SDK model format + sdk_data = edl.to_sdk_model() + + # Extract container params + container_params = {} + if "folder" in edl_config: + container_params["folder"] = edl_config["folder"] + elif "snippet" in edl_config: + container_params["snippet"] = edl_config["snippet"] + elif "device" in edl_config: + container_params["device"] = edl_config["device"] + # Create the EDL using the SDK data + result = scm_client.create_external_dynamic_list( + **container_params, + **sdk_data, + ) + return edl, result + + for edl_config, outcome, exc in run_bulk(external_dynamic_lists, _apply): + if exc is not None: + error(f"Error with external dynamic list '{edl_config.get('name', 'unknown')}': {str(exc)}") results.append( { - "action": "created/updated", - "name": edl.name, - "result": result, + "action": "error", + "name": edl_config.get("name", "unknown"), + "error": str(exc), } ) - except Exception as e: - error(f"Error with external dynamic list '{edl_config.get('name', 'unknown')}': {str(e)}") + continue + + edl, result = outcome + success(f"Loaded external dynamic list: {edl.name}") + loaded_count += 1 results.append( { - "action": "error", - "name": edl_config.get("name", "unknown"), - "error": str(e), + "action": "created/updated", + "name": edl.name, + "result": result, } ) - continue # Summary if dry_run: @@ -2763,8 +2813,38 @@ def load_hip_object( results: list[dict[str, Any]] = [] loaded_count = 0 - for idx, hip_data in enumerate(hip_objects, 1): - try: + if dry_run: + for idx, hip_data in enumerate(hip_objects, 1): + try: + # Override container if specified in command line + if location_value: + hip_data[location_type] = location_value + # Remove other container fields + for container in ["folder", "snippet", "device"]: + if container != location_type and container in hip_data: + del hip_data[container] + + # Validate using the Pydantic model + hip_obj = HIPObject(**hip_data) + + typer.echo(f"\n[{idx}] HIP Object: {hip_obj.name}") + typer.echo(f" Container: {getattr(hip_obj, location_type or 'folder')}") + if hip_obj.description: + typer.echo(f" Description: {hip_obj.description}") + results.append({"action": "would create/update", "name": hip_obj.name}) + except Exception as e: + error(f"Error with HIP object '{hip_data.get('name', 'unknown')}': {str(e)}") + results.append( + { + "action": "error", + "name": hip_data.get("name", "unknown"), + "error": str(e), + } + ) + continue + else: + + def _apply(hip_data: dict): # Override container if specified in command line if location_value: hip_data[location_type] = location_value @@ -2776,42 +2856,39 @@ def load_hip_object( # Validate using the Pydantic model hip_obj = HIPObject(**hip_data) - if dry_run: - typer.echo(f"\n[{idx}] HIP Object: {hip_obj.name}") - typer.echo(f" Container: {getattr(hip_obj, location_type or 'folder')}") - if hip_obj.description: - typer.echo(f" Description: {hip_obj.description}") - results.append({"action": "would create/update", "name": hip_obj.name}) - else: - # Convert to SDK model format - sdk_data = hip_obj.to_sdk_model() - - # Call the SDK client to create the HIP object - container_params = {location_type or "folder": getattr(hip_obj, location_type or "folder")} - result = scm_client.create_hip_object( - **container_params, - **sdk_data, - ) + # Convert to SDK model format + sdk_data = hip_obj.to_sdk_model() + + # Call the SDK client to create the HIP object + container_params = {location_type or "folder": getattr(hip_obj, location_type or "folder")} + result = scm_client.create_hip_object( + **container_params, + **sdk_data, + ) + return hip_obj, result - success(f"Loaded HIP object: {hip_obj.name}") - loaded_count += 1 + for hip_data, outcome, exc in run_bulk(hip_objects, _apply): + if exc is not None: + error(f"Error with HIP object '{hip_data.get('name', 'unknown')}': {str(exc)}") results.append( { - "action": "created/updated", - "name": hip_obj.name, - "result": result, + "action": "error", + "name": hip_data.get("name", "unknown"), + "error": str(exc), } ) - except Exception as e: - error(f"Error with HIP object '{hip_data.get('name', 'unknown')}': {str(e)}") + continue + + hip_obj, result = outcome + success(f"Loaded HIP object: {hip_obj.name}") + loaded_count += 1 results.append( { - "action": "error", - "name": hip_data.get("name", "unknown"), - "error": str(e), + "action": "created/updated", + "name": hip_obj.name, + "result": result, } ) - continue # Summary if dry_run: @@ -3124,8 +3201,39 @@ def load_hip_profile( results: list[dict[str, Any]] = [] loaded_count = 0 - for idx, profile_data in enumerate(hip_profiles, 1): - try: + if dry_run: + for idx, profile_data in enumerate(hip_profiles, 1): + try: + # Override container if specified in command line + if location_value: + profile_data[location_type] = location_value + # Remove other container fields + for container in ["folder", "snippet", "device"]: + if container != location_type and container in profile_data: + del profile_data[container] + + # Validate the configuration + profile = HIPProfile(**profile_data) + + typer.echo(f"\n[{idx}] HIP Profile: {profile.name}") + typer.echo(f" Container: {getattr(profile, location_type or 'folder')}") + typer.echo(f" Match: {profile.match}") + if profile.description: + typer.echo(f" Description: {profile.description}") + results.append({"action": "would create/update", "name": profile.name}) + except Exception as e: + error(f"Error with HIP profile '{profile_data.get('name', 'unknown')}': {str(e)}") + results.append( + { + "action": "error", + "name": profile_data.get("name", "unknown"), + "error": str(e), + } + ) + continue + else: + + def _apply(profile_data: dict): # Override container if specified in command line if location_value: profile_data[location_type] = location_value @@ -3137,44 +3245,41 @@ def load_hip_profile( # Validate the configuration profile = HIPProfile(**profile_data) - if dry_run: - typer.echo(f"\n[{idx}] HIP Profile: {profile.name}") - typer.echo(f" Container: {getattr(profile, location_type or 'folder')}") - typer.echo(f" Match: {profile.match}") - if profile.description: - typer.echo(f" Description: {profile.description}") - results.append({"action": "would create/update", "name": profile.name}) - else: - # Convert to SDK model format - profile_sdk = profile.to_sdk_model() - - # Call the SDK client to create the HIP profile - container_params = {location_type or "folder": getattr(profile, location_type or "folder")} - scm_client.create_hip_profile( - **container_params, - name=profile_sdk["name"], - match=profile_sdk["match"], - description=profile_sdk.get("description"), - ) - success(f"Loaded HIP profile: {profile.name}") - loaded_count += 1 + # Convert to SDK model format + profile_sdk = profile.to_sdk_model() + + # Call the SDK client to create the HIP profile + container_params = {location_type or "folder": getattr(profile, location_type or "folder")} + scm_client.create_hip_profile( + **container_params, + name=profile_sdk["name"], + match=profile_sdk["match"], + description=profile_sdk.get("description"), + ) + return profile, profile_sdk + + for profile_data, outcome, exc in run_bulk(hip_profiles, _apply): + if exc is not None: + error(f"Error with HIP profile '{profile_data.get('name', 'unknown')}': {str(exc)}") results.append( { - "action": "created/updated", - "name": profile.name, - "result": profile_sdk, + "action": "error", + "name": profile_data.get("name", "unknown"), + "error": str(exc), } ) - except Exception as e: - error(f"Error with HIP profile '{profile_data.get('name', 'unknown')}': {str(e)}") + continue + + profile, profile_sdk = outcome + success(f"Loaded HIP profile: {profile.name}") + loaded_count += 1 results.append( { - "action": "error", - "name": profile_data.get("name", "unknown"), - "error": str(e), + "action": "created/updated", + "name": profile.name, + "result": profile_sdk, } ) - continue # Summary if dry_run: @@ -3387,20 +3492,20 @@ def load_http_server_profile( results: list[dict[str, Any]] = [] loaded_count = 0 - for idx, profile_data in enumerate(http_server_profiles, 1): - try: - # Override container if specified in command line - if location_value: - profile_data[location_type] = location_value - # Remove other container fields - for container in ["folder", "snippet", "device"]: - if container != location_type and container in profile_data: - del profile_data[container] - - # Validate using the Pydantic model - profile = HTTPServerProfile(**profile_data) + if dry_run: + for idx, profile_data in enumerate(http_server_profiles, 1): + try: + # Override container if specified in command line + if location_value: + profile_data[location_type] = location_value + # Remove other container fields + for container in ["folder", "snippet", "device"]: + if container != location_type and container in profile_data: + del profile_data[container] + + # Validate using the Pydantic model + profile = HTTPServerProfile(**profile_data) - if dry_run: typer.echo(f"\n[{idx}] HTTP Server Profile: {profile.name}") typer.echo(f" Container: {getattr(profile, location_type or 'folder')}") typer.echo(f" Servers: {len(profile.servers)}") @@ -3411,35 +3516,63 @@ def load_http_server_profile( if profile.tag_registration: typer.echo(f" Tag Registration: {profile.tag_registration}") results.append({"action": "would create/update", "name": profile.name}) - else: - # Convert to SDK model format - profile_sdk = profile.to_sdk_model() - - # Call the SDK client to create the HTTP server profile - container_params = {location_type or "folder": getattr(profile, location_type or "folder")} - scm_client.create_http_server_profile( - **container_params, - **profile_sdk, + except Exception as e: + error(f"Error with HTTP server profile '{profile_data.get('name', 'unknown')}': {str(e)}") + results.append( + { + "action": "error", + "name": profile_data.get("name", "unknown"), + "error": str(e), + } ) - success(f"Loaded HTTP server profile: {profile.name}") - loaded_count += 1 + continue + else: + + def _apply(profile_data: dict): + # Override container if specified in command line + if location_value: + profile_data[location_type] = location_value + # Remove other container fields + for container in ["folder", "snippet", "device"]: + if container != location_type and container in profile_data: + del profile_data[container] + + # Validate using the Pydantic model + profile = HTTPServerProfile(**profile_data) + + # Convert to SDK model format + profile_sdk = profile.to_sdk_model() + + # Call the SDK client to create the HTTP server profile + container_params = {location_type or "folder": getattr(profile, location_type or "folder")} + scm_client.create_http_server_profile( + **container_params, + **profile_sdk, + ) + return profile, profile_sdk + + for profile_data, outcome, exc in run_bulk(http_server_profiles, _apply): + if exc is not None: + error(f"Error with HTTP server profile '{profile_data.get('name', 'unknown')}': {str(exc)}") results.append( { - "action": "created/updated", - "name": profile.name, - "result": profile_sdk, + "action": "error", + "name": profile_data.get("name", "unknown"), + "error": str(exc), } ) - except Exception as e: - error(f"Error with HTTP server profile '{profile_data.get('name', 'unknown')}': {str(e)}") + continue + + profile, profile_sdk = outcome + success(f"Loaded HTTP server profile: {profile.name}") + loaded_count += 1 results.append( { - "action": "error", - "name": profile_data.get("name", "unknown"), - "error": str(e), + "action": "created/updated", + "name": profile.name, + "result": profile_sdk, } ) - continue # Summary if dry_run: @@ -3677,20 +3810,20 @@ def load_log_forwarding_profile( results: list[dict[str, Any]] = [] loaded_count = 0 - for idx, profile_data in enumerate(log_forwarding_profiles, 1): - try: - # Override container if specified in command line - if location_value: - profile_data[location_type] = location_value - # Remove other container fields - for container in ["folder", "snippet", "device"]: - if container != location_type and container in profile_data: - del profile_data[container] - - # Validate using Pydantic model - profile = LogForwardingProfile(**profile_data) + if dry_run: + for idx, profile_data in enumerate(log_forwarding_profiles, 1): + try: + # Override container if specified in command line + if location_value: + profile_data[location_type] = location_value + # Remove other container fields + for container in ["folder", "snippet", "device"]: + if container != location_type and container in profile_data: + del profile_data[container] + + # Validate using Pydantic model + profile = LogForwardingProfile(**profile_data) - if dry_run: typer.echo(f"\n[{idx}] Log Forwarding Profile: {profile.name}") typer.echo(f" Container: {getattr(profile, location_type or 'folder')}") if profile.description: @@ -3702,36 +3835,63 @@ def load_log_forwarding_profile( for match_idx, match in enumerate(profile.match_list): typer.echo(f" Match {match_idx + 1}: {match.get('name', 'unnamed')} - {match.get('log_type', 'N/A')}") results.append({"action": "would create/update", "name": profile.name}) - else: - # Create the log forwarding profile - container_params = {location_type or "folder": getattr(profile, location_type or "folder")} - result = scm_client.create_log_forwarding_profile( - **container_params, - name=profile.name, - description=profile.description, - enhanced_application_logging=profile.enhanced_application_logging or False, - match_list=profile.match_list, + except Exception as e: + error(f"Error with log forwarding profile '{profile_data.get('name', 'unknown')}': {str(e)}") + results.append( + { + "action": "error", + "name": profile_data.get("name", "unknown"), + "error": str(e), + } ) + continue + else: - success(f"Loaded log forwarding profile: {profile.name}") - loaded_count += 1 + def _apply(profile_data: dict): + # Override container if specified in command line + if location_value: + profile_data[location_type] = location_value + # Remove other container fields + for container in ["folder", "snippet", "device"]: + if container != location_type and container in profile_data: + del profile_data[container] + + # Validate using Pydantic model + profile = LogForwardingProfile(**profile_data) + + # Create the log forwarding profile + container_params = {location_type or "folder": getattr(profile, location_type or "folder")} + result = scm_client.create_log_forwarding_profile( + **container_params, + name=profile.name, + description=profile.description, + enhanced_application_logging=profile.enhanced_application_logging or False, + match_list=profile.match_list, + ) + return profile, result + + for profile_data, outcome, exc in run_bulk(log_forwarding_profiles, _apply): + if exc is not None: + error(f"Error with log forwarding profile '{profile_data.get('name', 'unknown')}': {str(exc)}") results.append( { - "action": "created/updated", - "name": profile.name, - "result": result, + "action": "error", + "name": profile_data.get("name", "unknown"), + "error": str(exc), } ) - except Exception as e: - error(f"Error with log forwarding profile '{profile_data.get('name', 'unknown')}': {str(e)}") + continue + + profile, result = outcome + success(f"Loaded log forwarding profile: {profile.name}") + loaded_count += 1 results.append( { - "action": "error", - "name": profile_data.get("name", "unknown"), - "error": str(e), + "action": "created/updated", + "name": profile.name, + "result": result, } ) - continue # Summary if dry_run: @@ -3968,40 +4128,41 @@ def load_region( regions = [regions] # Process each region - created_count = 0 - for region_data in regions: - try: - # Validate with Pydantic model - validated_region = Region(**region_data) - - # Override container if specified - if folder: - validated_region.folder = folder - validated_region.snippet = None - validated_region.device = None - elif snippet: - validated_region.snippet = snippet - validated_region.folder = None - validated_region.device = None - elif device: - validated_region.device = device - validated_region.folder = None - validated_region.snippet = None + def _apply(region_data: dict): + # Validate with Pydantic model + validated_region = Region(**region_data) + + # Override container if specified + if folder: + validated_region.folder = folder + validated_region.snippet = None + validated_region.device = None + elif snippet: + validated_region.snippet = snippet + validated_region.folder = None + validated_region.device = None + elif device: + validated_region.device = device + validated_region.folder = None + validated_region.snippet = None + + # Convert to SDK format + sdk_data = validated_region.to_sdk_model() + + # Create/update the region + scm_client.create_region(sdk_data) + return validated_region - # Convert to SDK format - sdk_data = validated_region.to_sdk_model() - - # Create/update the region - scm_client.create_region(sdk_data) + created_count = 0 + for _region_data, validated_region, exc in run_bulk(regions, _apply): + if exc is not None: + error(f"Error processing region: {str(exc)}") + continue - created_count += 1 + created_count += 1 - container = validated_region.folder or validated_region.snippet or validated_region.device - success(f"Created region: {validated_region.name} in {container}") - - except Exception as e: - error(f"Error processing region: {str(e)}") - continue + container = validated_region.folder or validated_region.snippet or validated_region.device + success(f"Created region: {validated_region.name} in {container}") success(f"Summary: Processed {created_count} regions") @@ -4052,25 +4213,26 @@ def load_quarantined_device( devices = [devices] # Process each device - created_count = 0 - for device_data in devices: - try: - # Validate with Pydantic model - validated_device = QuarantinedDevice(**device_data) - - # Convert to SDK format - sdk_data = validated_device.to_sdk_model() + def _apply(device_data: dict): + # Validate with Pydantic model + validated_device = QuarantinedDevice(**device_data) - # Create the quarantined device - scm_client.create_quarantined_device(sdk_data) + # Convert to SDK format + sdk_data = validated_device.to_sdk_model() - created_count += 1 - success(f"Created quarantined device: {validated_device.host_id}") + # Create the quarantined device + scm_client.create_quarantined_device(sdk_data) + return validated_device - except Exception as e: - error(f"Error processing quarantined device: {str(e)}") + created_count = 0 + for _device_data, validated_device, exc in run_bulk(devices, _apply): + if exc is not None: + error(f"Error processing quarantined device: {str(exc)}") continue + created_count += 1 + success(f"Created quarantined device: {validated_device.host_id}") + success(f"Summary: Processed {created_count} quarantined devices") @@ -4373,20 +4535,20 @@ def load_service( results: list[dict[str, Any]] = [] loaded_count = 0 - for idx, service_data in enumerate(services, 1): - try: - # Override container if specified in command line - if location_value: - service_data[location_type] = location_value - # Remove other container fields - for container in ["folder", "snippet", "device"]: - if container != location_type and container in service_data: - del service_data[container] - - # Validate using Pydantic model - service = Service(**service_data) + if dry_run: + for idx, service_data in enumerate(services, 1): + try: + # Override container if specified in command line + if location_value: + service_data[location_type] = location_value + # Remove other container fields + for container in ["folder", "snippet", "device"]: + if container != location_type and container in service_data: + del service_data[container] + + # Validate using Pydantic model + service = Service(**service_data) - if dry_run: typer.echo(f"\n[{idx}] Service: {service.name}") typer.echo(f" Container: {getattr(service, location_type or 'folder')}") if service.description: @@ -4408,37 +4570,63 @@ def load_service( if service.tag: typer.echo(f" Tags: {', '.join(service.tag)}") results.append({"action": "would create/update", "name": service.name}) - else: - # Create the service - container_params = {location_type or "folder": getattr(service, location_type or "folder")} - result = scm_client.create_service( - **container_params, - name=service.name, - protocol=service.protocol, - description=service.description, - tag=service.tag, + except Exception as e: + error(f"Error with service '{service_data.get('name', 'unknown')}': {str(e)}") + results.append( + { + "action": "error", + "name": service_data.get("name", "unknown"), + "error": str(e), + } ) + continue + else: + + def _apply(service_data: dict): + # Override container if specified in command line + if location_value: + service_data[location_type] = location_value + # Remove other container fields + for container in ["folder", "snippet", "device"]: + if container != location_type and container in service_data: + del service_data[container] + + # Validate using Pydantic model + service = Service(**service_data) + + # Create the service + container_params = {location_type or "folder": getattr(service, location_type or "folder")} + result = scm_client.create_service( + **container_params, + name=service.name, + protocol=service.protocol, + description=service.description, + tag=service.tag, + ) + return service, result - success(f"Loaded service: {service.name}") - loaded_count += 1 + for service_data, outcome, exc in run_bulk(services, _apply): + if exc is not None: + error(f"Error with service '{service_data.get('name', 'unknown')}': {str(exc)}") results.append( { - "action": "created/updated", - "name": service.name, - "result": result, + "action": "error", + "name": service_data.get("name", "unknown"), + "error": str(exc), } ) + continue - except Exception as e: - error(f"Error with service '{service_data.get('name', 'unknown')}': {str(e)}") + service, result = outcome + success(f"Loaded service: {service.name}") + loaded_count += 1 results.append( { - "action": "error", - "name": service_data.get("name", "unknown"), - "error": str(e), + "action": "created/updated", + "name": service.name, + "result": result, } ) - continue # Summary if dry_run: @@ -4691,8 +4879,39 @@ def load_service_group( results: list[dict[str, Any]] = [] loaded_count = 0 - for idx, group_data in enumerate(service_groups, 1): - try: + if dry_run: + for idx, group_data in enumerate(service_groups, 1): + try: + # Override container if specified in command line + if location_value: + group_data[location_type] = location_value + # Remove other container fields + for container in ["folder", "snippet", "device"]: + if container != location_type and container in group_data: + del group_data[container] + + # Validate using Pydantic model + service_group = ServiceGroup(**group_data) + + typer.echo(f"\n[{idx}] Service Group: {service_group.name}") + typer.echo(f" Container: {getattr(service_group, location_type or 'folder')}") + typer.echo(f" Members ({len(service_group.members)}): {', '.join(service_group.members)}") + if service_group.tag: + typer.echo(f" Tags: {', '.join(service_group.tag)}") + results.append({"action": "would create/update", "name": service_group.name}) + except Exception as e: + error(f"Error with service group '{group_data.get('name', 'unknown')}': {str(e)}") + results.append( + { + "action": "error", + "name": group_data.get("name", "unknown"), + "error": str(e), + } + ) + continue + else: + + def _apply(group_data: dict): # Override container if specified in command line if location_value: group_data[location_type] = location_value @@ -4704,43 +4923,38 @@ def load_service_group( # Validate using Pydantic model service_group = ServiceGroup(**group_data) - if dry_run: - typer.echo(f"\n[{idx}] Service Group: {service_group.name}") - typer.echo(f" Container: {getattr(service_group, location_type or 'folder')}") - typer.echo(f" Members ({len(service_group.members)}): {', '.join(service_group.members)}") - if service_group.tag: - typer.echo(f" Tags: {', '.join(service_group.tag)}") - results.append({"action": "would create/update", "name": service_group.name}) - else: - # Create the service group - container_params = {location_type or "folder": getattr(service_group, location_type or "folder")} - result = scm_client.create_service_group( - **container_params, - name=service_group.name, - members=service_group.members, - tag=service_group.tag, - ) + # Create the service group + container_params = {location_type or "folder": getattr(service_group, location_type or "folder")} + result = scm_client.create_service_group( + **container_params, + name=service_group.name, + members=service_group.members, + tag=service_group.tag, + ) + return service_group, result - success(f"Loaded service group: {service_group.name}") - loaded_count += 1 + for group_data, outcome, exc in run_bulk(service_groups, _apply): + if exc is not None: + error(f"Error with service group '{group_data.get('name', 'unknown')}': {str(exc)}") results.append( { - "action": "created/updated", - "name": service_group.name, - "result": result, + "action": "error", + "name": group_data.get("name", "unknown"), + "error": str(exc), } ) + continue - except Exception as e: - error(f"Error with service group '{group_data.get('name', 'unknown')}': {str(e)}") + service_group, result = outcome + success(f"Loaded service group: {service_group.name}") + loaded_count += 1 results.append( { - "action": "error", - "name": group_data.get("name", "unknown"), - "error": str(e), + "action": "created/updated", + "name": service_group.name, + "result": result, } ) - continue # Summary if dry_run: @@ -4983,20 +5197,20 @@ def load_syslog_server_profile( results: list[dict[str, Any]] = [] loaded_count = 0 - for idx, profile_data in enumerate(syslog_server_profiles, 1): - try: - # Override container if specified in command line - if location_value: - profile_data[location_type] = location_value - # Remove other container fields - for container in ["folder", "snippet", "device"]: - if container != location_type and container in profile_data: - del profile_data[container] - - # Validate with Pydantic model - profile = SyslogServerProfile(**profile_data) + if dry_run: + for idx, profile_data in enumerate(syslog_server_profiles, 1): + try: + # Override container if specified in command line + if location_value: + profile_data[location_type] = location_value + # Remove other container fields + for container in ["folder", "snippet", "device"]: + if container != location_type and container in profile_data: + del profile_data[container] + + # Validate with Pydantic model + profile = SyslogServerProfile(**profile_data) - if dry_run: typer.echo(f"\n[{idx}] Syslog Server Profile: {profile.name}") typer.echo(f" Container: {getattr(profile, location_type or 'folder')}") if profile.description: @@ -5010,33 +5224,59 @@ def load_syslog_server_profile( if profile.tag: typer.echo(f" Tags: {', '.join(profile.tag)}") results.append({"action": "would create/update", "name": profile.name}) - else: - # Convert to SDK format - sdk_data = profile.to_sdk_model() + except Exception as e: + error(f"Error with syslog server profile '{profile_data.get('name', 'unknown')}': {str(e)}") + results.append( + { + "action": "error", + "name": profile_data.get("name", "unknown"), + "error": str(e), + } + ) + continue + else: - # Create/update the profile - scm_client.create_syslog_server_profile(sdk_data) + def _apply(profile_data: dict): + # Override container if specified in command line + if location_value: + profile_data[location_type] = location_value + # Remove other container fields + for container in ["folder", "snippet", "device"]: + if container != location_type and container in profile_data: + del profile_data[container] + + # Validate with Pydantic model + profile = SyslogServerProfile(**profile_data) - success(f"Loaded syslog server profile: {profile.name}") - loaded_count += 1 + # Convert to SDK format + sdk_data = profile.to_sdk_model() + + # Create/update the profile + scm_client.create_syslog_server_profile(sdk_data) + return profile, sdk_data + + for profile_data, outcome, exc in run_bulk(syslog_server_profiles, _apply): + if exc is not None: + error(f"Error with syslog server profile '{profile_data.get('name', 'unknown')}': {str(exc)}") results.append( { - "action": "created/updated", - "name": profile.name, - "result": sdk_data, + "action": "error", + "name": profile_data.get("name", "unknown"), + "error": str(exc), } ) + continue - except Exception as e: - error(f"Error with syslog server profile '{profile_data.get('name', 'unknown')}': {str(e)}") + profile, sdk_data = outcome + success(f"Loaded syslog server profile: {profile.name}") + loaded_count += 1 results.append( { - "action": "error", - "name": profile_data.get("name", "unknown"), - "error": str(e), + "action": "created/updated", + "name": profile.name, + "result": sdk_data, } ) - continue # Summary if dry_run: @@ -5306,21 +5546,24 @@ def load_schedule( schedules = [schedules] # Process each schedule - created_count = 0 - for schedule_data in schedules: - try: - # Create/update the schedule directly (YAML already has SDK format) - scm_client.create_schedule(schedule_data) + def _apply(schedule_data: dict): + # Create/update the schedule directly (YAML already has SDK format) + scm_client.create_schedule(schedule_data) - created_count += 1 - - container = schedule_data.get("folder") or schedule_data.get("snippet") or schedule_data.get("device") - success(f"Created schedule: {schedule_data['name']} in {container}") + container = schedule_data.get("folder") or schedule_data.get("snippet") or schedule_data.get("device") + return schedule_data["name"], container - except Exception as e: - error(f"Error processing schedule: {str(e)}") + created_count = 0 + for _schedule_data, outcome, exc in run_bulk(schedules, _apply): + if exc is not None: + error(f"Error processing schedule: {str(exc)}") continue + created_count += 1 + + name, container = outcome + success(f"Created schedule: {name} in {container}") + success(f"Summary: Processed {created_count} schedules") @@ -5591,40 +5834,41 @@ def load_tag( tags = [tags] # Process each tag - created_count = 0 - for tag_data in tags: - try: - # Validate with Pydantic model - validated_tag = Tag(**tag_data) - - # Override container if specified - if folder: - validated_tag.folder = folder - validated_tag.snippet = None - validated_tag.device = None - elif snippet: - validated_tag.snippet = snippet - validated_tag.folder = None - validated_tag.device = None - elif device: - validated_tag.device = device - validated_tag.folder = None - validated_tag.snippet = None - - # Convert to SDK format - sdk_data = validated_tag.to_sdk_model() - - # Create/update the tag - scm_client.create_tag(sdk_data) + def _apply(tag_data: dict): + # Validate with Pydantic model + validated_tag = Tag(**tag_data) + + # Override container if specified + if folder: + validated_tag.folder = folder + validated_tag.snippet = None + validated_tag.device = None + elif snippet: + validated_tag.snippet = snippet + validated_tag.folder = None + validated_tag.device = None + elif device: + validated_tag.device = device + validated_tag.folder = None + validated_tag.snippet = None + + # Convert to SDK format + sdk_data = validated_tag.to_sdk_model() + + # Create/update the tag + scm_client.create_tag(sdk_data) + return validated_tag - created_count += 1 + created_count = 0 + for _tag_data, validated_tag, exc in run_bulk(tags, _apply): + if exc is not None: + error(f"Error processing tag: {str(exc)}") + continue - container = validated_tag.folder or validated_tag.snippet or validated_tag.device - success(f"Created tag: {validated_tag.name} in {container}") + created_count += 1 - except Exception as e: - error(f"Error processing tag: {str(e)}") - continue + container = validated_tag.folder or validated_tag.snippet or validated_tag.device + success(f"Created tag: {validated_tag.name} in {container}") success(f"Summary: Processed {created_count} tags") @@ -5852,25 +6096,36 @@ def load_auto_tag_action( raise typer.Exit(code=1) created_count = 0 - for entry in config["auto_tag_actions"]: - try: - validated = AutoTagAction(**entry) - sdk_data = validated.to_sdk_model() + if dry_run: + for entry in config["auto_tag_actions"]: + try: + validated = AutoTagAction(**entry) + validated.to_sdk_model() - if dry_run: info(f"[DRY RUN] Would create auto tag action: {validated.name}") created_count += 1 + + except Exception as e: + error(f"Error processing auto tag action: {str(e)}") continue + else: + + def _apply(entry: dict): + validated = AutoTagAction(**entry) + sdk_data = validated.to_sdk_model() scm_client.create_auto_tag_action(sdk_data) + return validated + + for entry, validated, exc in run_bulk(config["auto_tag_actions"], _apply): + if exc is not None: + error(f"Error processing auto tag action: {str(exc)}") + continue + created_count += 1 container = entry.get("folder") or entry.get("snippet") or entry.get("device") success(f"Created auto tag action: {validated.name} in {container}") - except Exception as e: - error(f"Error processing auto tag action: {str(e)}") - continue - success(f"Summary: Processed {created_count} auto tag actions") diff --git a/src/scm_cli/commands/security.py b/src/scm_cli/commands/security.py index 778310a..120aac0 100644 --- a/src/scm_cli/commands/security.py +++ b/src/scm_cli/commands/security.py @@ -13,6 +13,7 @@ import yaml from ..utils import parse_comma_separated_list, validate_location_params +from ..utils.bulk import run_bulk from ..utils.decorators import handle_command_errors from ..utils.output import OUTPUT_OPTION, OutputFormat, emit, error, info, success, warning from ..utils.sdk_client import scm_client @@ -439,6 +440,7 @@ def load_security_rule( created_count = 0 updated_count = 0 + # sequential: rule order matters for rule_data in rules: try: # Apply container override if specified @@ -866,56 +868,58 @@ def load_anti_spyware_profile( return [] # Apply each anti-spyware profile + def _apply(profile_data: dict): + # Apply container override if specified + if folder: + profile_data["folder"] = folder + profile_data.pop("snippet", None) + profile_data.pop("device", None) + elif snippet: + profile_data["snippet"] = snippet + profile_data.pop("folder", None) + profile_data.pop("device", None) + elif device: + profile_data["device"] = device + profile_data.pop("folder", None) + profile_data.pop("snippet", None) + + # Validate using the Pydantic model + profile = AntiSpywareProfile(**profile_data) + + # Call the SDK client to create the anti-spyware profile + sdk_data = profile.to_sdk_model() + + # Extract container params + container_kwargs = {} + if sdk_data.get("folder"): + container_kwargs["folder"] = sdk_data.pop("folder") + elif sdk_data.get("snippet"): + container_kwargs["snippet"] = sdk_data.pop("snippet") + elif sdk_data.get("device"): + container_kwargs["device"] = sdk_data.pop("device") + + return scm_client.create_anti_spyware_profile(**container_kwargs, **sdk_data) + + outcomes = run_bulk(profiles, _apply) + results = [] created_count = 0 updated_count = 0 - for profile_data in profiles: - try: - # Apply container override if specified - if folder: - profile_data["folder"] = folder - profile_data.pop("snippet", None) - profile_data.pop("device", None) - elif snippet: - profile_data["snippet"] = snippet - profile_data.pop("folder", None) - profile_data.pop("device", None) - elif device: - profile_data["device"] = device - profile_data.pop("folder", None) - profile_data.pop("snippet", None) - - # Validate using the Pydantic model - profile = AntiSpywareProfile(**profile_data) - - # Call the SDK client to create the anti-spyware profile - sdk_data = profile.to_sdk_model() - - # Extract container params - container_kwargs = {} - if sdk_data.get("folder"): - container_kwargs["folder"] = sdk_data.pop("folder") - elif sdk_data.get("snippet"): - container_kwargs["snippet"] = sdk_data.pop("snippet") - elif sdk_data.get("device"): - container_kwargs["device"] = sdk_data.pop("device") - - result = scm_client.create_anti_spyware_profile(**container_kwargs, **sdk_data) - - results.append(result) - - # Track if created or updated based on response - if "created" in str(result).lower(): - created_count += 1 - else: - updated_count += 1 - - except Exception as e: - error(f"Error processing anti-spyware profile '{profile_data.get('name', 'unknown')}': {str(e)}") + for profile_data, result, exc in outcomes: + if exc is not None: + error(f"Error processing anti-spyware profile '{profile_data.get('name', 'unknown')}': {str(exc)}") # Continue processing other profiles continue + results.append(result) + + # Track if created or updated based on response + if "created" in str(result).lower(): + created_count += 1 + else: + updated_count += 1 + # Display summary with counts success(f"Successfully processed {len(results)} anti-spyware profile(s):") if created_count > 0: @@ -1239,56 +1243,58 @@ def load_decryption_profile( return [] # Apply each decryption profile + def _apply(profile_data: dict): + # Apply container override if specified + if folder: + profile_data["folder"] = folder + profile_data.pop("snippet", None) + profile_data.pop("device", None) + elif snippet: + profile_data["snippet"] = snippet + profile_data.pop("folder", None) + profile_data.pop("device", None) + elif device: + profile_data["device"] = device + profile_data.pop("folder", None) + profile_data.pop("snippet", None) + + # Validate using the Pydantic model + profile = DecryptionProfile(**profile_data) + + # Call the SDK client to create the decryption profile + sdk_data = profile.to_sdk_model() + + # Extract container params + container_kwargs = {} + if sdk_data.get("folder"): + container_kwargs["folder"] = sdk_data.pop("folder") + elif sdk_data.get("snippet"): + container_kwargs["snippet"] = sdk_data.pop("snippet") + elif sdk_data.get("device"): + container_kwargs["device"] = sdk_data.pop("device") + + return scm_client.create_decryption_profile(**container_kwargs, **sdk_data) + + outcomes = run_bulk(profiles, _apply) + results = [] created_count = 0 updated_count = 0 - for profile_data in profiles: - try: - # Apply container override if specified - if folder: - profile_data["folder"] = folder - profile_data.pop("snippet", None) - profile_data.pop("device", None) - elif snippet: - profile_data["snippet"] = snippet - profile_data.pop("folder", None) - profile_data.pop("device", None) - elif device: - profile_data["device"] = device - profile_data.pop("folder", None) - profile_data.pop("snippet", None) - - # Validate using the Pydantic model - profile = DecryptionProfile(**profile_data) - - # Call the SDK client to create the decryption profile - sdk_data = profile.to_sdk_model() - - # Extract container params - container_kwargs = {} - if sdk_data.get("folder"): - container_kwargs["folder"] = sdk_data.pop("folder") - elif sdk_data.get("snippet"): - container_kwargs["snippet"] = sdk_data.pop("snippet") - elif sdk_data.get("device"): - container_kwargs["device"] = sdk_data.pop("device") - - result = scm_client.create_decryption_profile(**container_kwargs, **sdk_data) - - results.append(result) - - # Track if created or updated based on response - if "created" in str(result).lower(): - created_count += 1 - else: - updated_count += 1 - - except Exception as e: - error(f"Error processing decryption profile '{profile_data.get('name', 'unknown')}': {str(e)}") + for profile_data, result, exc in outcomes: + if exc is not None: + error(f"Error processing decryption profile '{profile_data.get('name', 'unknown')}': {str(exc)}") # Continue processing other profiles continue + results.append(result) + + # Track if created or updated based on response + if "created" in str(result).lower(): + created_count += 1 + else: + updated_count += 1 + # Display summary with counts success(f"Successfully processed {len(results)} decryption profile(s):") if created_count > 0: @@ -1616,56 +1622,58 @@ def load_wildfire_antivirus_profile( return [] # Apply each WildFire antivirus profile + def _apply(profile_data: dict): + # Apply container override if specified + if folder: + profile_data["folder"] = folder + profile_data.pop("snippet", None) + profile_data.pop("device", None) + elif snippet: + profile_data["snippet"] = snippet + profile_data.pop("folder", None) + profile_data.pop("device", None) + elif device: + profile_data["device"] = device + profile_data.pop("folder", None) + profile_data.pop("snippet", None) + + # Validate using the Pydantic model + profile = WildfireAntivirusProfile(**profile_data) + + # Call the SDK client to create the WildFire antivirus profile + sdk_data = profile.to_sdk_model() + + # Extract container params + container_kwargs = {} + if sdk_data.get("folder"): + container_kwargs["folder"] = sdk_data.pop("folder") + elif sdk_data.get("snippet"): + container_kwargs["snippet"] = sdk_data.pop("snippet") + elif sdk_data.get("device"): + container_kwargs["device"] = sdk_data.pop("device") + + return scm_client.create_wildfire_antivirus_profile(**container_kwargs, **sdk_data) + + outcomes = run_bulk(profiles, _apply) + results = [] created_count = 0 updated_count = 0 - for profile_data in profiles: - try: - # Apply container override if specified - if folder: - profile_data["folder"] = folder - profile_data.pop("snippet", None) - profile_data.pop("device", None) - elif snippet: - profile_data["snippet"] = snippet - profile_data.pop("folder", None) - profile_data.pop("device", None) - elif device: - profile_data["device"] = device - profile_data.pop("folder", None) - profile_data.pop("snippet", None) - - # Validate using the Pydantic model - profile = WildfireAntivirusProfile(**profile_data) - - # Call the SDK client to create the WildFire antivirus profile - sdk_data = profile.to_sdk_model() - - # Extract container params - container_kwargs = {} - if sdk_data.get("folder"): - container_kwargs["folder"] = sdk_data.pop("folder") - elif sdk_data.get("snippet"): - container_kwargs["snippet"] = sdk_data.pop("snippet") - elif sdk_data.get("device"): - container_kwargs["device"] = sdk_data.pop("device") - - result = scm_client.create_wildfire_antivirus_profile(**container_kwargs, **sdk_data) - - results.append(result) - - # Track if created or updated based on response - if "created" in str(result).lower(): - created_count += 1 - else: - updated_count += 1 - - except Exception as e: - error(f"Error processing WildFire antivirus profile '{profile_data.get('name', 'unknown')}': {str(e)}") + for profile_data, result, exc in outcomes: + if exc is not None: + error(f"Error processing WildFire antivirus profile '{profile_data.get('name', 'unknown')}': {str(exc)}") # Continue processing other profiles continue + results.append(result) + + # Track if created or updated based on response + if "created" in str(result).lower(): + created_count += 1 + else: + updated_count += 1 + # Display summary with counts success(f"Successfully processed {len(results)} WildFire antivirus profile(s):") if created_count > 0: @@ -1980,57 +1988,59 @@ def load_dns_security_profile( return [] # Apply each DNS security profile + def _apply(profile_data: dict): + # Apply container override if specified + if folder: + profile_data["folder"] = folder + profile_data.pop("snippet", None) + profile_data.pop("device", None) + elif snippet: + profile_data["snippet"] = snippet + profile_data.pop("folder", None) + profile_data.pop("device", None) + elif device: + profile_data["device"] = device + profile_data.pop("folder", None) + profile_data.pop("snippet", None) + + # Validate using the Pydantic model + profile = DNSSecurityProfile(**profile_data) + + # Call the SDK client to create the DNS security profile + sdk_data = profile.to_sdk_model() + + # Extract container params + container_kwargs = {} + if sdk_data.get("folder"): + container_kwargs["folder"] = sdk_data.pop("folder") + elif sdk_data.get("snippet"): + container_kwargs["snippet"] = sdk_data.pop("snippet") + elif sdk_data.get("device"): + container_kwargs["device"] = sdk_data.pop("device") + + return scm_client.create_dns_security_profile(**container_kwargs, **sdk_data) + + outcomes = run_bulk(profiles, _apply) + results = [] created_count = 0 updated_count = 0 - for profile_data in profiles: - try: - # Apply container override if specified - if folder: - profile_data["folder"] = folder - profile_data.pop("snippet", None) - profile_data.pop("device", None) - elif snippet: - profile_data["snippet"] = snippet - profile_data.pop("folder", None) - profile_data.pop("device", None) - elif device: - profile_data["device"] = device - profile_data.pop("folder", None) - profile_data.pop("snippet", None) - - # Validate using the Pydantic model - profile = DNSSecurityProfile(**profile_data) - - # Call the SDK client to create the DNS security profile - sdk_data = profile.to_sdk_model() - - # Extract container params - container_kwargs = {} - if sdk_data.get("folder"): - container_kwargs["folder"] = sdk_data.pop("folder") - elif sdk_data.get("snippet"): - container_kwargs["snippet"] = sdk_data.pop("snippet") - elif sdk_data.get("device"): - container_kwargs["device"] = sdk_data.pop("device") - - result = scm_client.create_dns_security_profile(**container_kwargs, **sdk_data) - - results.append(result) - - # Track if created or updated based on __action__ field - action = result.get("__action__", "") - if action == "created": - created_count += 1 - elif action == "updated": - updated_count += 1 - - except Exception as e: - error(f"Error processing DNS security profile '{profile_data.get('name', 'unknown')}': {str(e)}") + for profile_data, result, exc in outcomes: + if exc is not None: + error(f"Error processing DNS security profile '{profile_data.get('name', 'unknown')}': {str(exc)}") # Continue processing other profiles continue + results.append(result) + + # Track if created or updated based on __action__ field + action = result.get("__action__", "") + if action == "created": + created_count += 1 + elif action == "updated": + updated_count += 1 + # Display summary with counts success(f"Successfully processed {len(results)} DNS security profile(s):") if created_count > 0: @@ -2328,56 +2338,58 @@ def load_vulnerability_protection_profile( return [] # Apply each vulnerability protection profile + def _apply(profile_data: dict): + # Apply container override if specified + if folder: + profile_data["folder"] = folder + profile_data.pop("snippet", None) + profile_data.pop("device", None) + elif snippet: + profile_data["snippet"] = snippet + profile_data.pop("folder", None) + profile_data.pop("device", None) + elif device: + profile_data["device"] = device + profile_data.pop("folder", None) + profile_data.pop("snippet", None) + + # Validate using the Pydantic model + profile = VulnerabilityProtectionProfile(**profile_data) + + # Call the SDK client to create the vulnerability protection profile + sdk_data = profile.to_sdk_model() + + # Extract container params + container_kwargs = {} + if sdk_data.get("folder"): + container_kwargs["folder"] = sdk_data.pop("folder") + elif sdk_data.get("snippet"): + container_kwargs["snippet"] = sdk_data.pop("snippet") + elif sdk_data.get("device"): + container_kwargs["device"] = sdk_data.pop("device") + + return scm_client.create_vulnerability_protection_profile(**container_kwargs, **sdk_data) + + outcomes = run_bulk(profiles, _apply) + results = [] created_count = 0 updated_count = 0 - for profile_data in profiles: - try: - # Apply container override if specified - if folder: - profile_data["folder"] = folder - profile_data.pop("snippet", None) - profile_data.pop("device", None) - elif snippet: - profile_data["snippet"] = snippet - profile_data.pop("folder", None) - profile_data.pop("device", None) - elif device: - profile_data["device"] = device - profile_data.pop("folder", None) - profile_data.pop("snippet", None) - - # Validate using the Pydantic model - profile = VulnerabilityProtectionProfile(**profile_data) - - # Call the SDK client to create the vulnerability protection profile - sdk_data = profile.to_sdk_model() - - # Extract container params - container_kwargs = {} - if sdk_data.get("folder"): - container_kwargs["folder"] = sdk_data.pop("folder") - elif sdk_data.get("snippet"): - container_kwargs["snippet"] = sdk_data.pop("snippet") - elif sdk_data.get("device"): - container_kwargs["device"] = sdk_data.pop("device") - - result = scm_client.create_vulnerability_protection_profile(**container_kwargs, **sdk_data) - - results.append(result) - - # Track if created or updated based on response - if "created" in str(result).lower(): - created_count += 1 - else: - updated_count += 1 - - except Exception as e: - error(f"Error processing vulnerability protection profile '{profile_data.get('name', 'unknown')}': {str(e)}") + for profile_data, result, exc in outcomes: + if exc is not None: + error(f"Error processing vulnerability protection profile '{profile_data.get('name', 'unknown')}': {str(exc)}") # Continue processing other profiles continue + results.append(result) + + # Track if created or updated based on response + if "created" in str(result).lower(): + created_count += 1 + else: + updated_count += 1 + # Display summary with counts success(f"Successfully processed {len(results)} vulnerability protection profile(s):") if created_count > 0: @@ -2698,56 +2710,58 @@ def load_url_category( return [] # Apply each URL category + def _apply(category_data: dict): + # Apply container override if specified + if folder: + category_data["folder"] = folder + category_data.pop("snippet", None) + category_data.pop("device", None) + elif snippet: + category_data["snippet"] = snippet + category_data.pop("folder", None) + category_data.pop("device", None) + elif device: + category_data["device"] = device + category_data.pop("folder", None) + category_data.pop("snippet", None) + + # Validate using the Pydantic model + category = URLCategory(**category_data) + + # Call the SDK client to create the URL category + sdk_data = category.to_sdk_model() + + # Extract container params + container_kwargs = {} + if sdk_data.get("folder"): + container_kwargs["folder"] = sdk_data.pop("folder") + elif sdk_data.get("snippet"): + container_kwargs["snippet"] = sdk_data.pop("snippet") + elif sdk_data.get("device"): + container_kwargs["device"] = sdk_data.pop("device") + + return scm_client.create_url_category(**container_kwargs, **sdk_data) + + outcomes = run_bulk(categories, _apply) + results = [] created_count = 0 updated_count = 0 - for category_data in categories: - try: - # Apply container override if specified - if folder: - category_data["folder"] = folder - category_data.pop("snippet", None) - category_data.pop("device", None) - elif snippet: - category_data["snippet"] = snippet - category_data.pop("folder", None) - category_data.pop("device", None) - elif device: - category_data["device"] = device - category_data.pop("folder", None) - category_data.pop("snippet", None) - - # Validate using the Pydantic model - category = URLCategory(**category_data) - - # Call the SDK client to create the URL category - sdk_data = category.to_sdk_model() - - # Extract container params - container_kwargs = {} - if sdk_data.get("folder"): - container_kwargs["folder"] = sdk_data.pop("folder") - elif sdk_data.get("snippet"): - container_kwargs["snippet"] = sdk_data.pop("snippet") - elif sdk_data.get("device"): - container_kwargs["device"] = sdk_data.pop("device") - - result = scm_client.create_url_category(**container_kwargs, **sdk_data) - - results.append(result) - - # Track if created or updated based on response - if result.get("__action__") == "created": - created_count += 1 - else: - updated_count += 1 - - except Exception as e: - error(f"Error processing URL category '{category_data.get('name', 'unknown')}': {str(e)}") + for category_data, result, exc in outcomes: + if exc is not None: + error(f"Error processing URL category '{category_data.get('name', 'unknown')}': {str(exc)}") # Continue processing other categories continue + results.append(result) + + # Track if created or updated based on response + if result.get("__action__") == "created": + created_count += 1 + else: + updated_count += 1 + # Display summary with counts success(f"Successfully processed {len(results)} URL category(ies):") if created_count > 0: @@ -3008,6 +3022,7 @@ def load_app_override_rule( return [] results = [] + # sequential: rule order matters for rule_data in rules: try: if folder: @@ -3279,6 +3294,7 @@ def load_authentication_rule( return [] results = [] + # sequential: rule order matters for rule_data in rules: try: if folder: @@ -3553,6 +3569,7 @@ def load_decryption_rule( return [] results = [] + # sequential: rule order matters for rule_data in rules: try: if folder: @@ -3825,39 +3842,41 @@ def load_url_access_profile( typer.echo(yaml.dump(profiles)) return [] - results = [] - for profile_data in profiles: - try: - if folder: - profile_data["folder"] = folder - profile_data.pop("snippet", None) - profile_data.pop("device", None) - elif snippet: - profile_data["snippet"] = snippet - profile_data.pop("folder", None) - profile_data.pop("device", None) - elif device: - profile_data["device"] = device - profile_data.pop("folder", None) - profile_data.pop("snippet", None) - - profile = URLAccessProfile(**profile_data) - sdk_data = profile.to_sdk_model() + def _apply(profile_data: dict): + if folder: + profile_data["folder"] = folder + profile_data.pop("snippet", None) + profile_data.pop("device", None) + elif snippet: + profile_data["snippet"] = snippet + profile_data.pop("folder", None) + profile_data.pop("device", None) + elif device: + profile_data["device"] = device + profile_data.pop("folder", None) + profile_data.pop("snippet", None) + + profile = URLAccessProfile(**profile_data) + sdk_data = profile.to_sdk_model() + + container_kwargs = {} + if sdk_data.get("folder"): + container_kwargs["folder"] = sdk_data.pop("folder") + elif sdk_data.get("snippet"): + container_kwargs["snippet"] = sdk_data.pop("snippet") + elif sdk_data.get("device"): + container_kwargs["device"] = sdk_data.pop("device") + + return scm_client.create_url_access_profile(**container_kwargs, **sdk_data) + + outcomes = run_bulk(profiles, _apply) - container_kwargs = {} - if sdk_data.get("folder"): - container_kwargs["folder"] = sdk_data.pop("folder") - elif sdk_data.get("snippet"): - container_kwargs["snippet"] = sdk_data.pop("snippet") - elif sdk_data.get("device"): - container_kwargs["device"] = sdk_data.pop("device") - - result = scm_client.create_url_access_profile(**container_kwargs, **sdk_data) - results.append(result) - - except Exception as e: - error(f"Error processing URL access profile '{profile_data.get('name', 'unknown')}': {str(e)}") + results = [] + for profile_data, result, exc in outcomes: + if exc is not None: + error(f"Error processing URL access profile '{profile_data.get('name', 'unknown')}': {str(exc)}") continue + results.append(result) success(f"Successfully processed {len(results)} URL access profile(s)") return results diff --git a/src/scm_cli/commands/setup.py b/src/scm_cli/commands/setup.py index c76f4a1..27d4ba5 100644 --- a/src/scm_cli/commands/setup.py +++ b/src/scm_cli/commands/setup.py @@ -11,6 +11,7 @@ import yaml from pydantic import ValidationError +from ..utils.bulk import run_bulk from ..utils.config import load_from_yaml from ..utils.decorators import handle_command_errors from ..utils.output import OUTPUT_OPTION, OutputFormat, emit, error, info, success @@ -275,6 +276,7 @@ def load_folder( typer.echo(yaml.dump(config["folders"])) return None + # sequential: folder hierarchy — a child folder may reference a parent defined earlier in the same file results = [] for folder_data in config["folders"]: try: @@ -436,15 +438,19 @@ def load_label( typer.echo(yaml.dump(config["labels"])) return None + def _apply(label_data: dict): + label_model = Label(**label_data) + return label_model, scm_client.create_label(**label_model.to_sdk_model()) + + # Apply each label concurrently, reporting outcomes in input order results = [] - for label_data in config["labels"]: - try: - label_model = Label(**label_data) - except ValidationError as e: - error(f"Validation error: {e}") - raise typer.Exit(code=1) from e - sdk_data = label_model.to_sdk_model() - result = scm_client.create_label(**sdk_data) + for _label_data, outcome, exc in run_bulk(config["labels"], _apply): + if isinstance(exc, ValidationError): + error(f"Validation error: {exc}") + raise typer.Exit(code=1) from exc + if exc is not None: + raise exc # abort on first error, matching the previous sequential loop + label_model, result = outcome results.append(result) action = result.get("__action__", "created") @@ -601,15 +607,19 @@ def load_snippet( typer.echo(yaml.dump(config["snippets"])) return None + def _apply(snippet_data: dict): + snippet_model = Snippet(**snippet_data) + return snippet_model, scm_client.create_snippet(**snippet_model.to_sdk_model()) + + # Apply each snippet concurrently, reporting outcomes in input order results = [] - for snippet_data in config["snippets"]: - try: - snippet_model = Snippet(**snippet_data) - except ValidationError as e: - error(f"Validation error: {e}") - raise typer.Exit(code=1) from e - sdk_data = snippet_model.to_sdk_model() - result = scm_client.create_snippet(**sdk_data) + for _snippet_data, outcome, exc in run_bulk(config["snippets"], _apply): + if isinstance(exc, ValidationError): + error(f"Validation error: {exc}") + raise typer.Exit(code=1) from exc + if exc is not None: + raise exc # abort on first error, matching the previous sequential loop + snippet_model, result = outcome results.append(result) action = result.get("__action__", "created") @@ -783,15 +793,19 @@ def load_variable( typer.echo(yaml.dump(config["variables"])) return None + def _apply(var_data: dict): + variable_model = Variable(**var_data) + return variable_model, scm_client.create_variable(**variable_model.to_sdk_model()) + + # Apply each variable concurrently, reporting outcomes in input order results = [] - for var_data in config["variables"]: - try: - variable_model = Variable(**var_data) - except ValidationError as e: - error(f"Validation error: {e}") - raise typer.Exit(code=1) from e - sdk_data = variable_model.to_sdk_model() - result = scm_client.create_variable(**sdk_data) + for _var_data, outcome, exc in run_bulk(config["variables"], _apply): + if isinstance(exc, ValidationError): + error(f"Validation error: {exc}") + raise typer.Exit(code=1) from exc + if exc is not None: + raise exc # abort on first error, matching the previous sequential loop + variable_model, result = outcome results.append(result) action = result.get("__action__", "created") @@ -955,14 +969,19 @@ def load_device( typer.echo(yaml.dump(config["devices"])) return None + def _apply(device_data: dict): + device_model = Device(**device_data) + return device_model, scm_client.update_device(**device_model.to_sdk_model()) + + # Apply each device concurrently, reporting outcomes in input order results = [] - for device_data in config["devices"]: - try: - device_model = Device(**device_data) - except ValidationError as e: - error(f"Validation error: {e}") - raise typer.Exit(code=1) from e - result = scm_client.update_device(**device_model.to_sdk_model()) + for _device_data, outcome, exc in run_bulk(config["devices"], _apply): + if isinstance(exc, ValidationError): + error(f"Validation error: {exc}") + raise typer.Exit(code=1) from exc + if exc is not None: + raise exc # abort on first error, matching the previous sequential loop + device_model, result = outcome results.append(result) action = result.get("__action__", "updated") diff --git a/src/scm_cli/main.py b/src/scm_cli/main.py index bcccf7c..16b2ac8 100644 --- a/src/scm_cli/main.py +++ b/src/scm_cli/main.py @@ -1,13 +1,94 @@ """Main entry point for the scm-cli tool. This module initializes the Typer CLI application and registers subcommands for the -various SCM configuration actions (set, delete, load) and object types. +various SCM configuration actions (set, delete, show, load, backup, move) and the +standalone top-level commands. + +Command modules are loaded lazily: help listings render from static metadata, and a +module is only imported when one of its commands is actually dispatched. This keeps +`scm --help` / `scm --version` free of the SDK/validator import cost. """ +import click import typer +from typer.core import TyperGroup + +# ============================================================================================================================================================================================= +# LAZY COMMAND LOADING +# ============================================================================================================================================================================================= + +# Lazy specs map a subcommand name to (module attribute path, help text). Module +# paths are relative to this package so the module resolves correctly whether the +# package is imported as `scm_cli` or `src.scm_cli` (tests use both). +_PACKAGE = __package__ + + +def _lazy_group(lazy_map: dict[str, tuple[str, str]]) -> type[TyperGroup]: + """Build a TyperGroup subclass whose subcommands load on dispatch. + + Help listings use the static help text from the spec (no imports); actual + dispatch (including ` --help`) imports the target module. + """ -# Import object type modules -from .commands import commit, context, deployment, identity, incidents, insights, jobs, local, mobile_agent, network, objects, operations, posture, security, setup + class LazyGroup(TyperGroup): + _lazy = lazy_map + + def list_commands(self, ctx: click.Context) -> list[str]: + return sorted(set(super().list_commands(ctx)) | set(self._lazy)) + + def get_command(self, ctx: click.Context, name: str) -> click.Command | None: + # Used by help rendering and completion: return a lightweight stub + # carrying the static help so nothing gets imported. + if name in self._lazy: + _, help_text = self._lazy[name] + return click.Command(name=name, help=help_text, short_help=help_text) + return super().get_command(ctx, name) + + def resolve_command(self, ctx: click.Context, args: list[str]) -> tuple[str | None, click.Command | None, list[str]]: + # Used only for actual dispatch: import the real subcommand. + name = args[0] if args else "" + if name in self._lazy: + return name, self._load(name), args[1:] + return super().resolve_command(ctx, args) + + def _load(self, name: str) -> click.Command: + import importlib + + attr_path, help_text = self._lazy[name] + module_name, attr = attr_path.split(":") + module = importlib.import_module(f"{_PACKAGE}.commands.{module_name}") + command = typer.main.get_command(getattr(module, attr)) + command.name = name + if help_text and not command.help: + command.help = help_text + return command + + return LazyGroup + + +def _action_spec(action: str, verb: str) -> dict[str, tuple[str, str]]: + """Lazy spec for one action group: category -> module app.""" + return { + "identity": (f"identity:{action}_app", f"{verb} identity configurations"), + "mobile-agent": (f"mobile_agent:{action}_app", f"{verb} mobile agent configurations"), + "network": (f"network:{action}_app", f"{verb} network configurations"), + "object": (f"objects:{action}_app", f"{verb} object configurations"), + "sase": (f"deployment:{action}_app", f"{verb} SASE configurations"), + "security": (f"security:{action}_app", f"{verb} security configurations"), + "setup": (f"setup:{action}_app", f"{verb} setup configurations"), + } + + +_TOP_LEVEL_SPEC: dict[str, tuple[str, str]] = { + "commit": ("commit:app", "Commit staged configuration changes"), + "context": ("context:app", "Manage authentication contexts"), + "incidents": ("incidents:app", "Search and view security incidents"), + "insights": ("insights:app", "Query monitoring insights"), + "jobs": ("jobs:app", "Manage SCM jobs"), + "local": ("local:app", "Retrieve local device configurations"), + "operations": ("operations:app", "Run device operations"), + "posture": ("posture:posture_app", "Firewall posture / BPA assessment"), +} # ============================================================================================================================================================================================= # MAIN CLI APPLICATION @@ -16,282 +97,31 @@ app = typer.Typer( name="scm", help="CLI for Palo Alto Networks Strata Cloud Manager", + cls=_lazy_group(_TOP_LEVEL_SPEC), ) # ============================================================================================================================================================================================= -# ACTION APP GROUPS -# ============================================================================================================================================================================================= - -# Create app groups for each action -backup_app = typer.Typer( - help="Backup configurations to YAML files", - name="backup", -) -delete_app = typer.Typer( - help="Remove configurations", - name="delete", -) -load_app = typer.Typer( - help="Load configurations from YAML files", - name="load", -) -set_app = typer.Typer( - help="Create or update configurations", - name="set", -) -move_app = typer.Typer( - help="Move rules to a new position", - name="move", -) -show_app = typer.Typer( - help="Display configurations", - name="show", -) - -# ============================================================================================================================================================================================= -# APP REGISTRATION +# ACTION APP GROUPS (categories load lazily per action) # ============================================================================================================================================================================================= -# ----------------------------------------------------------------------------------- Register Action Apps ----------------------------------------------------------------------------------- - -app.add_typer( - backup_app, - name="backup", -) -app.add_typer( - delete_app, - name="delete", -) -app.add_typer( - load_app, - name="load", -) -app.add_typer( - move_app, - name="move", -) -app.add_typer( - set_app, - name="set", -) -app.add_typer( - show_app, - name="show", -) - -# --------------------------------------------------------------------------------- Register Module Commands --------------------------------------------------------------------------------- +backup_app = typer.Typer(help="Backup configurations to YAML files", cls=_lazy_group(_action_spec("backup", "Backup"))) +delete_app = typer.Typer(help="Remove configurations", cls=_lazy_group(_action_spec("delete", "Delete"))) +load_app = typer.Typer(help="Load configurations from YAML files", cls=_lazy_group(_action_spec("load", "Load"))) +move_app = typer.Typer(help="Move rules to a new position", cls=_lazy_group({"security": ("security:move_app", "Move security rules")})) +set_app = typer.Typer(help="Create or update configurations", cls=_lazy_group(_action_spec("set", "Set"))) +show_app = typer.Typer(help="Display configurations", cls=_lazy_group(_action_spec("show", "Show"))) -# Backup commands -backup_app.add_typer( - identity.backup_app, - name="identity", - help="Backup identity configurations", -) -backup_app.add_typer( - mobile_agent.backup_app, - name="mobile-agent", - help="Backup mobile agent configurations", -) -backup_app.add_typer( - network.backup_app, - name="network", - help="Backup network configurations", -) -backup_app.add_typer( - objects.backup_app, - name="object", - help="Backup object configurations", -) -backup_app.add_typer( - deployment.backup_app, - name="sase", - help="Backup SASE configurations", -) -backup_app.add_typer( - security.backup_app, - name="security", - help="Backup security configurations", -) -backup_app.add_typer( - setup.backup_app, - name="setup", - help="Backup setup configurations", -) - -# Delete commands -delete_app.add_typer( - identity.delete_app, - name="identity", - help="Delete identity configurations", -) -delete_app.add_typer( - mobile_agent.delete_app, - name="mobile-agent", - help="Delete mobile agent configurations", -) -delete_app.add_typer( - network.delete_app, - name="network", - help="Delete network configurations", -) -delete_app.add_typer( - objects.delete_app, - name="object", - help="Delete object configurations", -) -delete_app.add_typer( - deployment.delete_app, - name="sase", - help="Delete SASE configurations", -) -delete_app.add_typer( - security.delete_app, - name="security", - help="Delete security configurations", -) -delete_app.add_typer( - setup.delete_app, - name="setup", - help="Delete setup configurations", -) - -# Load commands -load_app.add_typer( - identity.load_app, - name="identity", - help="Load identity configurations", -) -load_app.add_typer( - mobile_agent.load_app, - name="mobile-agent", - help="Load mobile agent configurations", -) -load_app.add_typer( - network.load_app, - name="network", - help="Load network configurations", -) -load_app.add_typer( - objects.load_app, - name="object", - help="Load object configurations", -) -load_app.add_typer( - deployment.load_app, - name="sase", - help="Load SASE configurations", -) -load_app.add_typer( - security.load_app, - name="security", - help="Load security configurations", -) -load_app.add_typer( - setup.load_app, - name="setup", - help="Load setup configurations", -) - -# Move commands -move_app.add_typer( - security.move_app, - name="security", - help="Move security rules", -) - -# Set commands -set_app.add_typer( - identity.set_app, - name="identity", - help="Set identity configurations", -) -set_app.add_typer( - mobile_agent.set_app, - name="mobile-agent", - help="Set mobile agent configurations", -) -set_app.add_typer( - network.set_app, - name="network", - help="Set network configurations", -) -set_app.add_typer( - objects.set_app, - name="object", - help="Set object configurations", -) -set_app.add_typer( - deployment.set_app, - name="sase", - help="Set SASE configurations", -) -set_app.add_typer( - security.set_app, - name="security", - help="Set security configurations", -) -set_app.add_typer( - setup.set_app, - name="setup", - help="Set setup configurations", -) - -# Show commands -show_app.add_typer( - identity.show_app, - name="identity", - help="Show identity configurations", -) -show_app.add_typer( - mobile_agent.show_app, - name="mobile-agent", - help="Show mobile agent configurations", -) -show_app.add_typer( - network.show_app, - name="network", - help="Show network configurations", -) -show_app.add_typer( - objects.show_app, - name="object", - help="Show object configurations", -) -show_app.add_typer( - deployment.show_app, - name="sase", - help="Show SASE configurations", -) -show_app.add_typer( - security.show_app, - name="security", - help="Show security configurations", -) -show_app.add_typer( - setup.show_app, - name="setup", - help="Show setup configurations", -) +app.add_typer(backup_app, name="backup") +app.add_typer(delete_app, name="delete") +app.add_typer(load_app, name="load") +app.add_typer(move_app, name="move") +app.add_typer(set_app, name="set") +app.add_typer(show_app, name="show") # ============================================================================================================================================================================================= -# CLI COMMANDS +# GLOBAL OPTIONS # ============================================================================================================================================================================================= -# Register top-level commands (alphabetical) -app.add_typer(commit.app, name="commit") -app.add_typer(context.app, name="context") -app.add_typer(incidents.app, name="incidents") -app.add_typer(insights.app, name="insights") -app.add_typer(jobs.app, name="jobs") -app.add_typer(local.app, name="local") -app.add_typer(operations.app, name="operations") -app.add_typer(posture.posture_app, name="posture") - - -# Note: test-auth command has been removed in favor of 'scm context test' -# Use 'scm context test' to test the current context -# Use 'scm context test ' to test a specific context without switching - _region_override: str | None = None _VALID_LOG_LEVELS = ("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG") @@ -333,9 +163,9 @@ def _configure_logging(debug: bool) -> None: def _version_callback(value: bool) -> None: if value: - from scm_cli import __version__ + from importlib.metadata import version - typer.echo(__version__) + typer.echo(version("pan-scm-cli")) raise typer.Exit() diff --git a/src/scm_cli/utils/bulk.py b/src/scm_cli/utils/bulk.py new file mode 100644 index 0000000..f4d4d90 --- /dev/null +++ b/src/scm_cli/utils/bulk.py @@ -0,0 +1,53 @@ +"""Bounded-concurrency runner for bulk operations (load commands). + +Bulk YAML loads previously issued API calls one object at a time; with the +upsert pattern (fetch + create/update) that meant 2N sequential roundtrips +for N objects. `run_bulk` runs the per-item work in a small thread pool — +results keep input order and per-item exceptions are captured, so callers +report outcomes exactly as before. + +Worker count: min(SCM_BULK_WORKERS (default 5), number of items). +""" + +import os +from collections.abc import Callable, Sequence +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any, TypeVar + +T = TypeVar("T") + +DEFAULT_WORKERS = 5 + + +def _worker_count(item_count: int, max_workers: int | None) -> int: + if max_workers is None: + try: + max_workers = int(os.environ.get("SCM_BULK_WORKERS", str(DEFAULT_WORKERS))) + except ValueError: + max_workers = DEFAULT_WORKERS + return max(1, min(max_workers, item_count)) + + +def run_bulk( + items: Sequence[T], + worker: Callable[[T], Any], + max_workers: int | None = None, +) -> list[tuple[T, Any, Exception | None]]: + """Run worker(item) for every item with bounded concurrency. + + Returns a list aligned with the input order: ``(item, result, exception)`` + per item, where exactly one of result/exception is set. + """ + if not items: + return [] + + results: list[tuple[T, Any, Exception | None]] = [None] * len(items) # type: ignore[list-item] + with ThreadPoolExecutor(max_workers=_worker_count(len(items), max_workers)) as executor: + futures = {executor.submit(worker, item): index for index, item in enumerate(items)} + for future in as_completed(futures): + index = futures[future] + try: + results[index] = (items[index], future.result(), None) + except Exception as e: # per-item failures are data, not control flow + results[index] = (items[index], None, e) + return results diff --git a/src/scm_cli/utils/sdk_client.py b/src/scm_cli/utils/sdk_client.py index 84f43e1..c7c1be0 100644 --- a/src/scm_cli/utils/sdk_client.py +++ b/src/scm_cli/utils/sdk_client.py @@ -20,6 +20,7 @@ from scm.client import Scm from scm.exceptions import APIError, AuthenticationError, ClientError, GatewayTimeoutError, NotFoundError, ObjectNotPresentError +from . import token_cache from .config import get_credentials, settings from .context import get_current_context @@ -88,6 +89,7 @@ def __init__(self): self.logger.info("No context set, using environment variables or default settings") self._bearer_token_mode = False + self._cached_token_mode = False if mock_mode_requested(): # Explicit mock mode: no API client, methods return mock data. @@ -159,23 +161,37 @@ def __init__(self): region_override = None resolved_region = region_override or credentials.get("region", "americas") - scm_kwargs: dict[str, Any] = { - "client_id": self.client_id, - "client_secret": self.client_secret, - "tsg_id": self.tsg_id, - "log_level": settings.get("log_level", "INFO"), - } scm_params = inspect.signature(Scm.__init__).parameters + common_kwargs: dict[str, Any] = {"log_level": settings.get("log_level", "INFO")} if "region" in scm_params: - scm_kwargs["region"] = resolved_region + common_kwargs["region"] = resolved_region # Endpoint overrides (env > context) — omitted when unset so SDK defaults apply api_base_url = credentials.get("api_base_url") or settings.get("api_base_url", None) if api_base_url and "api_base_url" in scm_params: - scm_kwargs["api_base_url"] = api_base_url + common_kwargs["api_base_url"] = api_base_url token_url = credentials.get("token_url") or settings.get("token_url", None) if token_url and "token_url" in scm_params: - scm_kwargs["token_url"] = token_url - self.client = Scm(**scm_kwargs) + common_kwargs["token_url"] = token_url + + # Reuse a cached OAuth token (bearer-mode session) when one is + # still valid for these exact credentials — skips the token + + # JWKS roundtrips on every invocation. + cached = token_cache.load_token(current_context) + if cached and str(cached.get("client_id")) == str(self.client_id) and str(cached.get("tsg_id")) == str(self.tsg_id): + self.client = Scm(access_token=cached["token"]["access_token"], **common_kwargs) + self._cached_token_mode = True + self.logger.debug("Using cached OAuth token (bearer-mode session)") + else: + self.client = Scm( + client_id=self.client_id, + client_secret=self.client_secret, + tsg_id=self.tsg_id, + **common_kwargs, + ) + oauth_client = getattr(self.client, "oauth_client", None) + token = getattr(getattr(oauth_client, "session", None), "token", None) + if token and token.get("access_token"): + token_cache.save_token(current_context, dict(token), client_id=self.client_id, tsg_id=self.tsg_id) self.logger.info(f"Successfully initialized SDK client for TSG ID: {self.tsg_id}") except (ValueError, AuthenticationError) as e: import sys @@ -311,6 +327,11 @@ def _handle_api_exception(self, operation: str, folder: str, resource_name: str, """ if isinstance(exception, AuthenticationError): self.logger.error(f"Authentication error during {operation} of {resource_name}: {str(exception)}") + if self._cached_token_mode: + # The cached token was rejected — clear it so the next + # invocation performs a fresh OAuth login. + token_cache.clear_token(get_current_context()) + self.logger.warning("Cached token rejected by the API; cache cleared — retry the command") elif isinstance(exception, NotFoundError): self.logger.error(f"Resource not found: {resource_name} in folder {folder}") elif isinstance(exception, ValidationError): @@ -10314,6 +10335,10 @@ def commit_config( # Pass admin parameter if specified (needed for bearer token auth) if admin: commit_kwargs["admin"] = [admin] + elif self._cached_token_mode: + # Cached-token sessions are bearer-mode to the SDK, so supply + # the admin identity the token was issued for. + commit_kwargs["admin"] = [self.client_id] # Always commit asynchronously, then use our own wait_for_job if sync async_kwargs = {k: v for k, v in commit_kwargs.items() if k not in ("sync", "timeout")} diff --git a/src/scm_cli/utils/token_cache.py b/src/scm_cli/utils/token_cache.py new file mode 100644 index 0000000..40c15f0 --- /dev/null +++ b/src/scm_cli/utils/token_cache.py @@ -0,0 +1,89 @@ +"""OAuth token file cache for scm-cli. + +Fetching an OAuth token (plus the JWKS signing key) adds network roundtrips +to every CLI invocation. Tokens are therefore cached per context under +``~/.scm-cli/cache/`` and reused until shortly before expiry, so consecutive +commands skip authentication entirely. + +Cache entries also record the client_id and tsg_id the token was issued for: +commit operations need an admin identity when running on a cached token +(bearer-mode session), and a context switch must not reuse another tenant's +token. + +Set ``SCM_NO_TOKEN_CACHE=1`` to disable the cache entirely. +""" + +import json +import logging +import os +import time +from typing import Any + +logger = logging.getLogger(__name__) + +CACHE_DIR = os.path.expanduser("~/.scm-cli/cache") + +# Refuse to reuse tokens within this many seconds of expiry (matches the +# SDK's own TOKEN_EXPIRY_BUFFER). +EXPIRY_BUFFER_SECONDS = 300 + + +def _cache_disabled() -> bool: + return os.environ.get("SCM_NO_TOKEN_CACHE", "").strip().lower() in ("1", "true", "yes", "on") + + +def _cache_path(context_name: str | None) -> str: + return os.path.join(CACHE_DIR, f"token-{context_name or 'default'}.json") + + +def save_token(context_name: str | None, token: dict[str, Any], client_id: str, tsg_id: str) -> None: + """Persist an OAuth token for a context (0600 file permissions).""" + if _cache_disabled(): + return + try: + os.makedirs(CACHE_DIR, mode=0o700, exist_ok=True) + path = _cache_path(context_name) + entry = {"token": token, "client_id": client_id, "tsg_id": tsg_id} + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as f: + json.dump(entry, f) + logger.debug(f"Cached OAuth token for context '{context_name or 'default'}'") + except OSError as e: + # Cache failures must never break the CLI. + logger.debug(f"Could not write token cache: {e}") + + +def load_token(context_name: str | None) -> dict[str, Any] | None: + """Return the cached entry for a context, or None if absent/expired/corrupt. + + The returned dict has keys ``token`` (the OAuth token dict), ``client_id``, + and ``tsg_id``. + """ + if _cache_disabled(): + return None + path = _cache_path(context_name) + try: + with open(path) as f: + entry = json.load(f) + expires_at = float(entry["token"]["expires_at"]) + if expires_at - EXPIRY_BUFFER_SECONDS <= time.time(): + logger.debug(f"Cached token for '{context_name or 'default'}' expired") + clear_token(context_name) + return None + if not entry["token"].get("access_token"): + raise KeyError("access_token") + return entry + except FileNotFoundError: + return None + except (KeyError, TypeError, ValueError, OSError) as e: + logger.debug(f"Discarding unusable token cache ({e})") + clear_token(context_name) + return None + + +def clear_token(context_name: str | None) -> None: + """Delete the cached token for a context (no-op if absent).""" + import contextlib + + with contextlib.suppress(OSError): + os.remove(_cache_path(context_name)) diff --git a/tests/test_bulk.py b/tests/test_bulk.py new file mode 100644 index 0000000..d870e43 --- /dev/null +++ b/tests/test_bulk.py @@ -0,0 +1,65 @@ +"""Tests for the bounded-concurrency bulk runner (scm_cli.utils.bulk).""" + +import threading +import time + +from scm_cli.utils.bulk import run_bulk + + +class TestRunBulk: + def test_results_preserve_input_order(self): + def worker(n): + time.sleep(0.01 if n % 2 else 0) + return n * 10 + + results = run_bulk([1, 2, 3, 4], worker) + + assert [(item, value) for item, value, _ in results] == [(1, 10), (2, 20), (3, 30), (4, 40)] + assert all(exc is None for _, _, exc in results) + + def test_exceptions_captured_per_item(self): + def worker(n): + if n == 2: + raise ValueError("bad item") + return n + + results = run_bulk([1, 2, 3], worker) + + assert results[0][2] is None + assert isinstance(results[1][2], ValueError) + assert results[1][1] is None + assert results[2][2] is None + + def test_runs_concurrently(self): + active = {"now": 0, "max": 0} + lock = threading.Lock() + + def worker(n): + with lock: + active["now"] += 1 + active["max"] = max(active["max"], active["now"]) + time.sleep(0.05) + with lock: + active["now"] -= 1 + return n + + start = time.monotonic() + run_bulk(list(range(10)), worker, max_workers=5) + elapsed = time.monotonic() - start + + assert active["max"] > 1, "workers never overlapped" + assert elapsed < 0.45, f"took {elapsed:.2f}s — looks sequential (10 x 0.05s)" + + def test_empty_items(self): + assert run_bulk([], lambda x: x) == [] + + def test_worker_env_override(self, monkeypatch): + monkeypatch.setenv("SCM_BULK_WORKERS", "1") + order = [] + + def worker(n): + order.append(n) + return n + + run_bulk([1, 2, 3], worker) + assert order == [1, 2, 3] # single worker executes in submission order diff --git a/tests/test_deployment_commands.py b/tests/test_deployment_commands.py index 11e0981..87bb651 100644 --- a/tests/test_deployment_commands.py +++ b/tests/test_deployment_commands.py @@ -836,3 +836,43 @@ def test_show_network_location_empty_json(self, runner, monkeypatch): assert result.exit_code == 0 assert json.loads(result.stdout) == [] + + +class TestDeploymentBulkLoadConcurrency: + """Bulk loads issue create calls concurrently (bounded thread pool).""" + + def test_load_internal_dns_server_runs_concurrently(self, runner, monkeypatch, tmp_path): + import threading + import time + + from scm_cli.utils.sdk_client import scm_client + + yaml_content = "internal_dns_servers:\n" + "".join(f" - name: dns-{i}\n domain_name: [corp{i}.example.com]\n primary: 10.0.0.{i + 1}\n" for i in range(4)) + test_file = tmp_path / "internal_dns_servers.yml" + test_file.write_text(yaml_content) + + active = {"now": 0, "max": 0} + lock = threading.Lock() + created = [] + + def mock_create(**kwargs): + with lock: + active["now"] += 1 + active["max"] = max(active["max"], active["now"]) + time.sleep(0.05) + with lock: + active["now"] -= 1 + created.append(kwargs.get("name")) + return {"name": kwargs.get("name"), "__action__": "created"} + + monkeypatch.setattr(scm_client, "create_internal_dns_server", mock_create) + + test_app = typer.Typer() + test_app.command()(load_internal_dns_server) + + result = runner.invoke(test_app, ["--file", str(test_file)]) + + assert result.exit_code == 0, result.output + assert active["max"] > 1, "create calls never overlapped" + assert sorted(created) == [f"dns-{i}" for i in range(4)] + assert "Loaded 4 internal DNS server(s)" in result.output diff --git a/tests/test_identity_commands.py b/tests/test_identity_commands.py index ef84041..f7bd633 100644 --- a/tests/test_identity_commands.py +++ b/tests/test_identity_commands.py @@ -343,3 +343,40 @@ def test_show_authentication_profile_empty_list_output_json(self, runner, monkey assert result.exit_code == 0, result.output assert json.loads(result.stdout) == [] + + +class TestIdentityBulkLoadConcurrency: + """Bulk loads issue create calls concurrently (bounded thread pool).""" + + def test_load_authentication_profile_runs_concurrently(self, runner, monkeypatch, tmp_path): + import threading + import time + + from scm_cli.utils.sdk_client import scm_client + + yaml_content = "authentication_profiles:\n" + "".join(f" - name: auth-{i}\n folder: Texas\n" for i in range(4)) + test_file = tmp_path / "auth_profiles.yaml" + test_file.write_text(yaml_content) + + active = {"now": 0, "max": 0} + lock = threading.Lock() + created = [] + + def mock_create(**kwargs): + with lock: + active["now"] += 1 + active["max"] = max(active["max"], active["now"]) + time.sleep(0.05) + with lock: + active["now"] -= 1 + created.append(kwargs.get("name")) + return {"name": kwargs.get("name"), "__action__": "created"} + + monkeypatch.setattr(scm_client, "create_authentication_profile", mock_create) + + result = runner.invoke(load_app, ["authentication-profile", "--file", str(test_file)]) + + assert result.exit_code == 0, result.output + assert active["max"] > 1, "create calls never overlapped" + assert sorted(created) == [f"auth-{i}" for i in range(4)] + assert "Processed 4 authentication profiles" in result.output diff --git a/tests/test_lazy_startup.py b/tests/test_lazy_startup.py new file mode 100644 index 0000000..6c00268 --- /dev/null +++ b/tests/test_lazy_startup.py @@ -0,0 +1,87 @@ +"""Tests for lazy command loading (startup performance). + +Importing scm_cli.main must not import the command modules or the heavy +SDK/validator stacks — they load on dispatch of the matching subcommand. +""" + +import subprocess +import sys + +from src.scm_cli.main import app + +HEAVY_MODULES = [ + "scm.client", + "requests", + "scm_cli.utils.sdk_client", + "scm_cli.utils.validators", + "scm_cli.commands.objects", + "scm_cli.commands.network", +] + + +class TestImportLaziness: + def test_importing_main_does_not_import_heavy_modules(self): + code = ( + "import sys\n" + "import scm_cli.main\n" + f"loaded = [m for m in {HEAVY_MODULES!r} if m in sys.modules]\n" + "print(','.join(loaded))\n" + ) + result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True) + + assert result.stdout.strip() == "", f"heavy modules imported at startup: {result.stdout.strip()}" + + def test_help_does_not_import_heavy_modules(self): + code = ( + "import sys\n" + "from typer.testing import CliRunner\n" + "from scm_cli.main import app\n" + "r = CliRunner().invoke(app, ['--help'])\n" + "assert r.exit_code == 0, r.output\n" + f"loaded = [m for m in {HEAVY_MODULES!r} if m in sys.modules]\n" + "print(','.join(loaded))\n" + ) + result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True) + + assert result.stdout.strip() == "", f"--help imported heavy modules: {result.stdout.strip()}" + + +class TestHelpListings: + def test_top_level_help_lists_actions_and_standalones(self, runner): + result = runner.invoke(app, ["--help"]) + + assert result.exit_code == 0 + for name in ["set", "delete", "show", "load", "backup", "move", "commit", "context", "jobs", "insights", "incidents", "local", "operations", "posture"]: + assert name in result.output + + def test_action_help_lists_categories(self, runner): + result = runner.invoke(app, ["set", "--help"]) + + assert result.exit_code == 0 + for name in ["identity", "mobile-agent", "network", "object", "sase", "security", "setup"]: + assert name in result.output + + +class TestDispatchStillWorks: + def test_config_command_dispatches(self, runner, monkeypatch): + monkeypatch.setenv("SCM_MOCK", "1") + result = runner.invoke(app, ["show", "object", "address", "--folder", "Texas"]) + + assert result.exit_code == 0 + + def test_standalone_command_dispatches(self, runner, monkeypatch): + monkeypatch.setenv("SCM_MOCK", "1") + result = runner.invoke(app, ["jobs", "list"]) + + assert result.exit_code == 0 + + def test_subgroup_help_dispatches(self, runner): + result = runner.invoke(app, ["set", "object", "--help"]) + + assert result.exit_code == 0 + assert "address" in result.output + + def test_unknown_command_still_errors(self, runner): + result = runner.invoke(app, ["set", "nonsense"]) + + assert result.exit_code != 0 diff --git a/tests/test_mobile_agent_commands.py b/tests/test_mobile_agent_commands.py index 5d8fca0..51b8ec7 100644 --- a/tests/test_mobile_agent_commands.py +++ b/tests/test_mobile_agent_commands.py @@ -175,10 +175,14 @@ def mock_create(*args, **kwargs): result = runner.invoke( test_app, [ - "--folder", "Mobile Users", - "--name", "saml-auth", - "--authentication-profile", "best-practice", - "--os", "Any", + "--folder", + "Mobile Users", + "--name", + "saml-auth", + "--authentication-profile", + "best-practice", + "--os", + "Any", ], ) @@ -206,9 +210,12 @@ def mock_create(*args, **kwargs): result = runner.invoke( test_app, [ - "--folder", "Mobile Users", - "--name", "saml-auth", - "--authentication-profile", "best-practice", + "--folder", + "Mobile Users", + "--name", + "saml-auth", + "--authentication-profile", + "best-practice", ], ) @@ -235,8 +242,10 @@ def mock_create(*args, **kwargs): result = runner.invoke( test_app, [ - "--folder", "Mobile Users", - "--name", "saml-auth", + "--folder", + "Mobile Users", + "--name", + "saml-auth", ], ) @@ -474,8 +483,10 @@ def mock_error(*args, **kwargs): result = runner.invoke( test_app, [ - "--folder", "Mobile Users", - "--name", "fail-auth", + "--folder", + "Mobile Users", + "--name", + "fail-auth", ], ) @@ -527,3 +538,43 @@ def test_show_auth_setting_detail_json(self, runner, monkeypatch): assert result.exit_code == 0 assert json.loads(result.stdout) == setting + + +class TestMobileAgentBulkLoadConcurrency: + """Bulk loads issue create calls concurrently (bounded thread pool).""" + + def test_load_auth_setting_runs_concurrently(self, runner, monkeypatch, tmp_path): + import threading + import time + + from scm_cli.utils.sdk_client import scm_client + + yaml_content = "auth_settings:\n" + "".join(f' - name: auth-{i}\n folder: "Mobile Users"\n authentication_profile: best-practice\n os: Any\n' for i in range(4)) + test_file = tmp_path / "auth_settings.yml" + test_file.write_text(yaml_content) + + active = {"now": 0, "max": 0} + lock = threading.Lock() + created = [] + + def mock_create(**kwargs): + with lock: + active["now"] += 1 + active["max"] = max(active["max"], active["now"]) + time.sleep(0.05) + with lock: + active["now"] -= 1 + created.append(kwargs.get("name")) + return {"name": kwargs.get("name"), "__action__": "created"} + + monkeypatch.setattr(scm_client, "create_auth_setting", mock_create) + + test_app = typer.Typer() + test_app.command()(load_auth_setting) + + result = runner.invoke(test_app, ["--file", str(test_file)]) + + assert result.exit_code == 0, result.output + assert active["max"] > 1, "create calls never overlapped" + assert sorted(created) == [f"auth-{i}" for i in range(4)] + assert "4 created" in result.output diff --git a/tests/test_network_commands.py b/tests/test_network_commands.py index 3f2cf53..fd421b6 100644 --- a/tests/test_network_commands.py +++ b/tests/test_network_commands.py @@ -3111,3 +3111,46 @@ def test_show_single_not_found_errors(self, runner, monkeypatch): result = runner.invoke(test_app, ["--folder", "test-folder", "--name", "missing"]) assert result.exit_code == 1 assert "not found" in result.output + + +class TestNetworkLoadConcurrency: + """Load commands issue their per-item API calls concurrently via run_bulk.""" + + def test_load_ethernet_interface_runs_concurrently(self, runner, monkeypatch, tmp_path): + """Loading >=4 ethernet interfaces overlaps create calls in the thread pool.""" + import threading + import time + + import yaml + + from scm_cli.utils.sdk_client import scm_client + + yaml_data = {"ethernet_interfaces": [{"name": f"$eth{i}", "folder": "test-folder", "layer3": {"mtu": 1500}} for i in range(1, 5)]} + yaml_file = tmp_path / "ethernet-interfaces.yaml" + with yaml_file.open("w") as f: + yaml.dump(yaml_data, f) + + active = {"now": 0, "max": 0} + created = [] + lock = threading.Lock() + + def mock_create(sdk_data): + with lock: + active["now"] += 1 + active["max"] = max(active["max"], active["now"]) + time.sleep(0.05) + with lock: + active["now"] -= 1 + created.append(sdk_data["name"]) + return {**sdk_data, "id": "eth-12345", "__action__": "created"} + + monkeypatch.setattr(scm_client, "create_ethernet_interface", mock_create) + test_app = typer.Typer() + test_app.command()(load_ethernet_interface) + result = runner.invoke(test_app, ["--file", str(yaml_file)]) + + assert result.exit_code == 0 + assert active["max"] > 1, "create calls never overlapped — load looks sequential" + assert sorted(created) == ["$eth1", "$eth2", "$eth3", "$eth4"] + assert result.stdout.count("Created ethernet interface") == 4 + assert "Summary: Processed 4 ethernet interfaces" in result.stdout diff --git a/tests/test_objects_commands.py b/tests/test_objects_commands.py index 887c61b..300e38e 100644 --- a/tests/test_objects_commands.py +++ b/tests/test_objects_commands.py @@ -1495,3 +1495,45 @@ def mock_create(*args, **kwargs): result = runner.invoke(test_app, ["--folder", "Texas", "--name", "test-sg", "--members", "HTTP, HTTPS, SSH"]) assert result.exit_code == 0 assert captured.get("members") == ["HTTP", "HTTPS", "SSH"] + + +class TestLoadConcurrency: + """Bulk load commands must apply items through the bounded thread pool (run_bulk).""" + + def test_load_address_runs_concurrently(self, runner, monkeypatch, tmp_path): + """Load with >=4 items should overlap create calls and still create every item.""" + import threading + import time + + from scm_cli.commands.objects import load_address + from scm_cli.utils.sdk_client import scm_client + + yaml_content = "addresses:\n" + "".join(f" - name: addr-{i}\n folder: Texas\n ip_netmask: 10.0.0.{i}/32\n" for i in range(6)) + test_file = tmp_path / "addresses.yml" + test_file.write_text(yaml_content) + + active = {"now": 0, "max": 0} + lock = threading.Lock() + created = [] + + def mock_create(*args, **kwargs): + with lock: + active["now"] += 1 + active["max"] = max(active["max"], active["now"]) + time.sleep(0.05) + with lock: + active["now"] -= 1 + created.append(kwargs.get("name")) + return {"id": "addr-1", "name": kwargs.get("name"), "folder": kwargs.get("folder"), "created": True} + + monkeypatch.setattr(scm_client, "create_address", mock_create) + + test_app = typer.Typer() + test_app.command()(load_address) + + result = runner.invoke(test_app, ["--file", str(test_file)]) + + assert result.exit_code == 0 + assert "Successfully processed 6 address(es):" in result.stdout + assert sorted(created) == [f"addr-{i}" for i in range(6)] + assert active["max"] > 1, "create calls never overlapped — load is still sequential" diff --git a/tests/test_posture_commands.py b/tests/test_posture_commands.py index 48effb0..3cbf98b 100644 --- a/tests/test_posture_commands.py +++ b/tests/test_posture_commands.py @@ -914,12 +914,10 @@ def test_format_empty_checks(self): class TestPostureRegistration: """Test posture command is registered in main app.""" - def test_posture_registered(self): + def test_posture_registered(self, runner): """Test that posture is registered as a top-level command.""" from scm_cli.main import app - group_names = [] - for group in app.registered_groups: - if hasattr(group, "typer_instance") and group.typer_instance: - group_names.append(group.name) - assert "posture" in group_names + result = runner.invoke(app, ["posture", "--help"]) + assert result.exit_code == 0 + assert "assess" in result.output diff --git a/tests/test_security_commands.py b/tests/test_security_commands.py index eb0d13e..75068fc 100644 --- a/tests/test_security_commands.py +++ b/tests/test_security_commands.py @@ -585,6 +585,54 @@ def test_load_wildfire_antivirus_profile_dry_run(self, runner, monkeypatch, tmp_ assert "Dry run mode" in result.stdout mock_client.create_wildfire_antivirus_profile.assert_not_called() + def test_load_wildfire_antivirus_profile_runs_concurrently(self, runner, monkeypatch, tmp_path): + """Bulk load issues create calls concurrently via run_bulk.""" + import threading + import time + + mock_client = self._mock_scm_client(monkeypatch) + active = {"now": 0, "max": 0} + lock = threading.Lock() + created = [] + + def mock_create(*args, **kwargs): + with lock: + active["now"] += 1 + active["max"] = max(active["max"], active["now"]) + time.sleep(0.05) + with lock: + active["now"] -= 1 + created.append(kwargs.get("name")) + return {"id": f"wfav-{kwargs.get('name')}", "name": kwargs.get("name"), "folder": kwargs.get("folder")} + + mock_client.create_wildfire_antivirus_profile.side_effect = mock_create + + entries = "\n".join( + f""" - name: wf-{i} + folder: Texas + rules: + - name: Forward All + direction: both + analysis: public-cloud + application: + - any + file_type: + - any""" + for i in range(4) + ) + test_file = tmp_path / "wf_profiles.yml" + test_file.write_text("wildfire_antivirus_profiles:\n" + entries + "\n") + + test_app = typer.Typer() + test_app.command()(load_wildfire_antivirus_profile) + + result = runner.invoke(test_app, ["--file", str(test_file)]) + + assert result.exit_code == 0 + assert active["max"] > 1, "create calls never overlapped — load looks sequential" + assert sorted(created) == [f"wf-{i}" for i in range(4)] + assert "Successfully processed 4 WildFire antivirus profile(s)" in result.stdout + def test_set_wildfire_antivirus_profile_updated(self, runner, monkeypatch): mock_client = self._mock_scm_client(monkeypatch) mock_client.create_wildfire_antivirus_profile.return_value = { diff --git a/tests/test_setup_commands.py b/tests/test_setup_commands.py index 97978a7..63127d0 100644 --- a/tests/test_setup_commands.py +++ b/tests/test_setup_commands.py @@ -836,3 +836,40 @@ def test_show_label_empty_list_output_json(self, runner, monkeypatch): assert result.exit_code == 0, result.output assert json.loads(result.stdout) == [] + + +class TestSetupBulkLoadConcurrency: + """Bulk loads issue create calls concurrently (bounded thread pool).""" + + def test_load_label_runs_concurrently(self, runner, monkeypatch, tmp_path): + import threading + import time + + from scm_cli.utils.sdk_client import scm_client + + yaml_content = "labels:\n" + "".join(f" - name: label-{i}\n" for i in range(4)) + test_file = tmp_path / "labels.yaml" + test_file.write_text(yaml_content) + + active = {"now": 0, "max": 0} + lock = threading.Lock() + created = [] + + def mock_create(**kwargs): + with lock: + active["now"] += 1 + active["max"] = max(active["max"], active["now"]) + time.sleep(0.05) + with lock: + active["now"] -= 1 + created.append(kwargs.get("name")) + return {"name": kwargs.get("name"), "__action__": "created"} + + monkeypatch.setattr(scm_client, "create_label", mock_create) + + result = runner.invoke(load_app, ["label", "--file", str(test_file)]) + + assert result.exit_code == 0, result.output + assert active["max"] > 1, "create calls never overlapped" + assert sorted(created) == [f"label-{i}" for i in range(4)] + assert "Processed 4 labels" in result.output diff --git a/tests/test_token_cache.py b/tests/test_token_cache.py new file mode 100644 index 0000000..84f08de --- /dev/null +++ b/tests/test_token_cache.py @@ -0,0 +1,164 @@ +"""Tests for the OAuth token file cache (scm_cli.utils.token_cache). + +Tokens fetched during OAuth login are cached per context so consecutive CLI +invocations skip the token + JWKS roundtrips. Cache entries carry the +client_id (needed for commit admin under cached-token bearer mode). +""" + +import os +import time + +import pytest + +from scm_cli.utils import token_cache + + +@pytest.fixture(autouse=True) +def cache_dir(tmp_path, monkeypatch): + """Point the cache at a temp directory for every test.""" + monkeypatch.setattr(token_cache, "CACHE_DIR", str(tmp_path / "cache")) + return tmp_path / "cache" + + +def _token(expires_in: float = 3600.0) -> dict: + return {"access_token": "tok-abc", "token_type": "Bearer", "expires_at": time.time() + expires_in} + + +class TestSaveAndLoad: + def test_round_trip(self): + token_cache.save_token("prod", _token(), client_id="cid@x.com", tsg_id="123") + entry = token_cache.load_token("prod") + + assert entry is not None + assert entry["token"]["access_token"] == "tok-abc" + assert entry["client_id"] == "cid@x.com" + assert entry["tsg_id"] == "123" + + def test_no_context_uses_default_name(self): + token_cache.save_token(None, _token(), client_id="cid", tsg_id="1") + assert token_cache.load_token(None) is not None + + def test_file_permissions_0600(self, cache_dir): + token_cache.save_token("prod", _token(), client_id="cid", tsg_id="1") + files = list(cache_dir.iterdir()) + assert len(files) == 1 + assert oct(files[0].stat().st_mode & 0o777) == "0o600" + + def test_contexts_are_isolated(self): + token_cache.save_token("a", _token(), client_id="cid-a", tsg_id="1") + token_cache.save_token("b", _token(), client_id="cid-b", tsg_id="2") + + assert token_cache.load_token("a")["client_id"] == "cid-a" + assert token_cache.load_token("b")["client_id"] == "cid-b" + + +class TestExpiryAndCorruption: + def test_expired_token_returns_none(self): + token_cache.save_token("prod", _token(expires_in=10), client_id="cid", tsg_id="1") + # within the 5-minute safety buffer -> treated as expired + assert token_cache.load_token("prod") is None + + def test_missing_cache_returns_none(self): + assert token_cache.load_token("nope") is None + + def test_corrupt_cache_tolerated_and_cleared(self, cache_dir): + os.makedirs(cache_dir, exist_ok=True) + path = cache_dir / "token-prod.json" + path.write_text("{not json") + + assert token_cache.load_token("prod") is None + assert not path.exists() + + def test_token_without_expiry_returns_none(self): + token_cache.save_token("prod", {"access_token": "x"}, client_id="cid", tsg_id="1") + assert token_cache.load_token("prod") is None + + +class TestClear: + def test_clear_removes_entry(self): + token_cache.save_token("prod", _token(), client_id="cid", tsg_id="1") + token_cache.clear_token("prod") + assert token_cache.load_token("prod") is None + + def test_clear_missing_is_noop(self): + token_cache.clear_token("never-existed") + + +class TestDisableSwitch: + def test_env_var_disables_cache(self, monkeypatch): + token_cache.save_token("prod", _token(), client_id="cid", tsg_id="1") + monkeypatch.setenv("SCM_NO_TOKEN_CACHE", "1") + + assert token_cache.load_token("prod") is None + + def test_env_var_disables_save(self, monkeypatch, cache_dir): + monkeypatch.setenv("SCM_NO_TOKEN_CACHE", "1") + token_cache.save_token("prod", _token(), client_id="cid", tsg_id="1") + + assert not (cache_dir / "token-prod.json").exists() + + +class TestClientIntegration: + """SCMClient uses the cache: hit -> bearer-mode init (no OAuth), miss -> OAuth + save.""" + + def _fresh_client(self, monkeypatch, fake_scm): + import scm_cli.utils.sdk_client as sdk_module + from scm_cli.utils.context import get_context_aware_settings + + monkeypatch.delenv("SCM_MOCK", raising=False) + monkeypatch.setenv("SCM_SCM_CLIENT_ID", "cid@x.com") + monkeypatch.setenv("SCM_SCM_CLIENT_SECRET", "sec") + monkeypatch.setenv("SCM_SCM_TSG_ID", "123") + monkeypatch.setattr(sdk_module, "Scm", fake_scm) + monkeypatch.setattr(sdk_module, "get_current_context", lambda: None) + monkeypatch.setattr(sdk_module, "settings", get_context_aware_settings()) + import scm_cli.utils.config as cfg_mod + + monkeypatch.setattr(cfg_mod, "settings", get_context_aware_settings()) + return sdk_module.SCMClient() + + def test_cache_hit_uses_bearer_mode(self, monkeypatch): + token_cache.save_token(None, _token(), client_id="cid@x.com", tsg_id="123") + captured = {} + + class FakeScm: + def __init__(self, **kwargs): + captured.update(kwargs) + + client = self._fresh_client(monkeypatch, FakeScm) + + assert captured.get("access_token") == "tok-abc" + assert "client_secret" not in captured + assert client._cached_token_mode is True + + def test_cache_hit_for_other_client_id_ignored(self, monkeypatch): + token_cache.save_token(None, _token(), client_id="someone-else", tsg_id="123") + captured = {} + + class FakeScm: + oauth_client = None + + def __init__(self, **kwargs): + captured.update(kwargs) + + client = self._fresh_client(monkeypatch, FakeScm) + + assert "access_token" not in captured + assert captured.get("client_id") == "cid@x.com" + assert client._cached_token_mode is False + + def test_cache_miss_saves_token_after_oauth(self, monkeypatch): + from types import SimpleNamespace + + fresh_token = {"access_token": "new-tok", "expires_at": time.time() + 900} + + class FakeScm: + def __init__(self, **kwargs): + self.oauth_client = SimpleNamespace(session=SimpleNamespace(token=fresh_token)) + + self._fresh_client(monkeypatch, FakeScm) + + entry = token_cache.load_token(None) + assert entry is not None + assert entry["token"]["access_token"] == "new-tok" + assert entry["client_id"] == "cid@x.com"