Feature/discovery job

.github/workflows/keyfactor-starter-workflow.yml

@@ -11,10 +11,17 @@ on:
 
 jobs:
   call-starter-workflow:
-    uses: keyfactor/actions/.github/workflows/starter.yml@3.1.2
+    uses: keyfactor/actions/.github/workflows/starter.yml@v4
+    with:
+      command_token_url: ${{ vars.COMMAND_TOKEN_URL }}
+      command_hostname: ${{ vars.COMMAND_HOSTNAME }}
+      command_base_api_path: ${{ vars.COMMAND_API_PATH }}
     secrets:
       token: ${{ secrets.V2BUILDTOKEN}}
-      APPROVE_README_PUSH: ${{ secrets.APPROVE_README_PUSH}}
       gpg_key: ${{ secrets.KF_GPG_PRIVATE_KEY }}
       gpg_pass: ${{ secrets.KF_GPG_PASSPHRASE }}
       scan_token: ${{ secrets.SAST_TOKEN }}
+      entra_username: ${{ secrets.DOCTOOL_ENTRA_USERNAME }}
+      entra_password: ${{ secrets.DOCTOOL_ENTRA_PASSWD }}
+      command_client_id: ${{ secrets.COMMAND_CLIENT_ID }}
+      command_client_secret: ${{ secrets.COMMAND_CLIENT_SECRET }}

.gitignore

@@ -348,3 +348,11 @@ MigrationBackup/
 
 # Ionide (cross platform F# VS Code tools) working folder
 .ionide/
+
+# Claude Code per-machine IDE state
+.claude/
+
+# Secrets - never commit credentials, PATs, or environment files
+.secrets/
+*.env
+

CHANGELOG.md

@@ -1,16 +1,58 @@
-1.1.2
+## 1.2.0 - unreleased
+
+### Added
+* Discovery job: automatically enumerates all application domains on a DataPower
+  appliance via `GET /mgmt/domains/config/`, then queries each domain's filestore
+  via `GET /mgmt/filestore/{domain}` to surface certificate store directories
+  (`cert`, `pubcert`, `sharedcert`). Eliminates manual creation of certificate
+  stores per domain in environments with many domains.
+* `FlowLogger` step-oriented breadcrumb logger; every job (Inventory, Management,
+  Discovery) wraps its `ProcessJob` body in a `using (var flow = new FlowLogger(...))`
+  block. The step summary is appended to `JobResult.FailureMessage` on both success
+  and failure so operators can scan job-history without enabling trace logs.
+* `DataPowerApiException`: typed API error carrying HTTP status code and trimmed
+  response body. `Find()` walker unwraps `AggregateException` chains so
+  `JobBase.DescribeException` can surface the underlying HTTP detail to operators.
+* `JobBase`: shared plumbing for PAM resolution (warn-on-empty fallback), JobResult
+  helpers that auto-append flow summaries, and exception unwrapping.
+* Spec documentation for the Discovery feature: `docs/discovery-overview.md`,
+  `docs/discovery-overview.pdf`, and the original HTML version.
+
+### Changed
+* `DataPowerClient.ApiRequestString` now throws the typed `DataPowerApiException`
+  on `WebException`, capturing the HTTP status and response body from the failed
+  response. Previously errors were re-thrown as raw `WebException`.
+* Trace logging masks sensitive payload fields (`content`, `Password`,
+  `PasswordAlias`) before serializing the request body to the log.
+* Request and response streams are now wrapped in `using` blocks for deterministic
+  disposal.
+* `Inventory`, `Management`, and `Discovery` jobs all derive from `JobBase` and
+  perform null-argument validation at the public boundary before any work begins.
+
+### Documentation
+* `docsource/content.md` and `readme_source.md` now have a dedicated "Store Path
+  Format" section explaining `<domain>\<directory>`, the three certificate
+  directory types (`cert`, `pubcert`, `sharedcert`), and their scoping.
+* Discovery section added to both source docs.
+* `.gitignore` updated to exclude `.claude/` (per-machine IDE state),
+  `.secrets/`, and `*.env` files.
+
+## 1.1.2
+
 * Added Support for new version of Data Power and Backwards for Old Versions After Data Power API Breaking Changes
 
-1.1.1
+## 1.1.1
+
 * Dual Build .Net 6 and .Net 8 support
 * Test Tool Modifications
 * Readme Updates
 
-1.1.0
+## 1.1.0
+
 * Convert to Universal Orchestrator Framework
 * Added Support for .cer files during inventory
 * Added PAM Support
 
-1.0.0
-* Windows Orchestrator with Add, Remove and Inventory Capabilities
+## 1.0.0
 
+* Windows Orchestrator with Add, Remove and Inventory Capabilities

DataPower/Client/DataPowerApiException.cs

@@ -0,0 +1,72 @@
+// Copyright 2024 Keyfactor
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Net;
+
+namespace Keyfactor.Extensions.Orchestrator.DataPower.Client
+{
+    /// <summary>
+    /// Thrown by <see cref="DataPowerClient"/> when the DataPower REST Management Interface
+    /// returns a non-success status. Carries the HTTP status code and trimmed response body
+    /// so callers can branch on specific conditions and operators can see what the appliance
+    /// actually said.
+    /// </summary>
+    public class DataPowerApiException : Exception
+    {
+        public HttpStatusCode StatusCode { get; }
+        public string Operation { get; }
+        public string ResponseBody { get; }
+
+        public DataPowerApiException(string message, HttpStatusCode statusCode, string operation, string responseBody)
+            : base(message)
+        {
+            StatusCode = statusCode;
+            Operation = operation;
+            ResponseBody = responseBody;
+        }
+
+        public DataPowerApiException(string message, HttpStatusCode statusCode, string operation, string responseBody, Exception inner)
+            : base(message, inner)
+        {
+            StatusCode = statusCode;
+            Operation = operation;
+            ResponseBody = responseBody;
+        }
+
+        /// <summary>
+        /// Walks an exception chain (including <see cref="AggregateException"/>) and returns the
+        /// first <see cref="DataPowerApiException"/> found, or <c>null</c> if none is present.
+        /// </summary>
+        public static DataPowerApiException Find(Exception ex)
+        {
+            while (ex != null)
+            {
+                if (ex is DataPowerApiException api)
+                    return api;
+                if (ex is AggregateException agg)
+                {
+                    foreach (var inner in agg.InnerExceptions)
+                    {
+                        var found = Find(inner);
+                        if (found != null) return found;
+                    }
+                    return null;
+                }
+                ex = ex.InnerException;
+            }
+            return null;
+        }
+    }
+}

DataPower/Client/DataPowerClient.cs

@@ -17,6 +17,8 @@
 using System.Net;
 using System.Net.Http;
 using System.Text;
+using System.Collections.Generic;
+using System.Linq;
 using Keyfactor.Extensions.Orchestrator.DataPower.Models.Requests;
 using Keyfactor.Extensions.Orchestrator.DataPower.Models.Responses;
 using Keyfactor.Extensions.Orchestrator.DataPower.Models.SupportingObjects;
@@ -72,6 +74,71 @@
 
         #region Class Methods
 
+        public List<DomainInfo> ListDomains()
+        {
+            try
+            {
+                var request = new ListDomainsRequest();
+                var strResponse = ApiRequestString("ListDomains", request.GetResource(), request.Method,
+                    string.Empty, false, true);
+
+                var containerName = "domain";
+
+                // DataPower returns a single object instead of array when only one domain exists
+                if (strResponse.Contains($"\"{containerName}\"") &&
+                    !strResponse.Contains($"\"{containerName}\" : [") &&
+                    !strResponse.Contains($"\"{containerName}\":["))
+                {
+                    // Wrap single domain object in array for proper deserialization
+                    var singleResponse = JsonConvert.DeserializeObject<ListDomainsSingleResponse>(strResponse);
+                    return singleResponse?.Domain != null
+                        ? new List<DomainInfo> { singleResponse.Domain }
+                        : new List<DomainInfo>();
+                }
+
+                var response = JsonConvert.DeserializeObject<ListDomainsResponse>(strResponse);
+                return response?.Domains?.ToList() ?? new List<DomainInfo>();
+            }
+            catch (Exception e)
+            {
+                _logger.LogError($"Error In DataPowerClient.ListDomains: {LogHandler.FlattenException(e)}");
+                throw;
+            }
+        }
+
+        public List<string> ListFileStoreDirectories(string domain)
+        {
+            try
+            {
+                var request = new ListFileStoreRequest(domain);
+                var strResponse = ApiRequestString("ListFileStoreDirectories", request.GetResource(), request.Method,
+                    string.Empty, false, true);
+
+                var containerName = "location";
+
+                // DataPower returns a single object instead of an array when only one location exists
+                if (strResponse.Contains($"\"{containerName}\"") &&
+                    !strResponse.Contains($"\"{containerName}\" : [") &&
+                    !strResponse.Contains($"\"{containerName}\":["))
+                {
+                    strResponse = FixDataPowerBadJson(strResponse, containerName);
+                }
+
+                var response = JsonConvert.DeserializeObject<ListFileStoreResponse>(strResponse);
+                if (response?.FileStore?.Locations == null)
+                    return new List<string>();
+
+                return response.FileStore.Locations
+                    .Select(d => d.Name)
+                    .ToList();
+            }
+            catch (Exception e)
+            {
+                _logger.LogError($"Error In DataPowerClient.ListFileStoreDirectories: {LogHandler.FlattenException(e)}");
+                throw;
+            }
+        }
+
         public bool SaveConfig()
         {
             try
@@ -405,13 +472,15 @@
             _logger.LogTrace($"BaseUrl: {BaseUrl}");
             _logger.LogTrace($"url: {strPostUrl}");
             _logger.LogTrace($"strMethod: {strMethod}");
-            _logger.LogTrace($"strQueryString: {strQueryString}");
+            // Mask sensitive payload fields (cert/key material, passwords) before logging
+            _logger.LogTrace($"strQueryString: {MaskSensitivePayload(strQueryString)}");
 
+            HttpWebResponse objResponse = null;
             try
             {
                 ServicePointManager.ServerCertificateValidationCallback = delegate { return true; };
 
                 var objRequest = (HttpWebRequest) WebRequest.Create(BaseUrl + strPostUrl);
                 objRequest.Method = strMethod;
                 objRequest.ContentType = "application/json";
                 var encoded =
@@ -424,25 +493,102 @@
                     var postBytes = Encoding.UTF8.GetBytes(strQueryString);
                     objRequest.ContentLength = postBytes.Length;
 
-                    var requestStream = objRequest.GetRequestStream();
-                    requestStream.Write(postBytes, 0, postBytes.Length);
-                    requestStream.Close();
+                    using (var requestStream = objRequest.GetRequestStream())
+                    {
+                        requestStream.Write(postBytes, 0, postBytes.Length);
+                    }
                 }
 
-                var objResponse = (HttpWebResponse) objRequest.GetResponse();
-                var strResponse =
-                    new StreamReader(objResponse.GetResponseStream() ?? throw new InvalidOperationException())
-                        .ReadToEnd();
-                _logger.LogTrace($"strResponse: {strResponse}");
-                _logger.LogTrace("END APIRequestString");
+                objResponse = (HttpWebResponse) objRequest.GetResponse();
+                using (var stream = objResponse.GetResponseStream() ?? throw new InvalidOperationException("Response stream is null."))
+                using (var reader = new StreamReader(stream))
+                {
+                    var strResponse = reader.ReadToEnd();
+                    _logger.LogTrace($"strResponse: {strResponse}");
+                    _logger.LogTrace("END APIRequestString");
+                    return strResponse;
+                }
+            }
+            catch (WebException webEx)
+            {
+                // Extract status code and response body from the failed response so the
+                // typed DataPowerApiException can carry both back to the operator.
+                var statusCode = HttpStatusCode.InternalServerError;
+                var responseBody = string.Empty;
+
+                if (webEx.Response is HttpWebResponse errorResponse)
+                {
+                    statusCode = errorResponse.StatusCode;
+                    try
+                    {
+                        using (var stream = errorResponse.GetResponseStream())
+                        {
+                            if (stream != null)
+                            {
+                                using (var reader = new StreamReader(stream))
+                                {
+                                    responseBody = reader.ReadToEnd();
+                                }
+                            }
+                        }
+                    }
+                    catch (Exception readEx)
+                    {
+                        _logger.LogWarning(readEx, "Failed to read error response body for {Operation}.", strCall);
+                    }
+                    finally
+                    {
+                        errorResponse.Dispose();
+                    }
+                }
 
-                return strResponse;
+                _logger.LogError(webEx,
+                    "END APIRequestString error for {Operation}: HTTP {Status} {StatusName}, body: {Body}",
+                    strCall, (int)statusCode, statusCode, responseBody);
+                throw new DataPowerApiException(
+                    $"DataPower API call '{strCall}' failed with HTTP {(int)statusCode} {statusCode}. Body: {responseBody}",
+                    statusCode, strCall, responseBody, webEx);
+            }
+            catch (DataPowerApiException)
+            {
+                // Already typed - just rethrow
+                throw;
             }
             catch (Exception ex)
             {
-                _logger.LogError($"END APIRequestString error: {LogHandler.FlattenException(ex)}");
+                _logger.LogError(ex, "END APIRequestString error: {ErrorMessage}", LogHandler.FlattenException(ex));
                 throw;
             }
+            finally
+            {
+                objResponse?.Dispose();
+            }
+        }
+
+        /// <summary>
+        /// Masks sensitive fields (cert content, private keys, passwords) in a JSON payload
+        /// before it goes into trace logs. Only the value portion is replaced - keys remain
+        /// readable so operators can see the request shape.
+        /// </summary>
+        private static string MaskSensitivePayload(string payload)
+        {
+            if (string.IsNullOrEmpty(payload)) return payload;
+
+            // Quick check: if no sensitive markers exist, skip the regex work
+            if (payload.IndexOf("content", StringComparison.OrdinalIgnoreCase) < 0
+                && payload.IndexOf("Password", StringComparison.OrdinalIgnoreCase) < 0
+                && payload.IndexOf("BEGIN", StringComparison.Ordinal) < 0)
+            {
+                return payload;
+            }
+
+            // Replace string values for known-sensitive keys: "content", "Password", "PasswordAlias"
+            var masked = System.Text.RegularExpressions.Regex.Replace(
+                payload,
+                "\"(content|Password|PasswordAlias)\"\\s*:\\s*\"[^\"]*\"",
+                "\"$1\":\"***MASKED***\"",
+                System.Text.RegularExpressions.RegexOptions.IgnoreCase);
+            return masked;
         }
 
         #endregion