feat: emit OTel log signals on unrouted requests — GH#4705

apiml/src/main/java/org/zowe/apiml/filter/OtelRequestFilter.java

@@ -24,7 +24,10 @@
 import org.springframework.web.server.WebFilter;
 import org.springframework.web.server.WebFilterChain;
 import org.zowe.apiml.product.opentelemetry.OtelRequestContext;
+import org.zowe.apiml.security.common.error.ServiceNotAccessibleException;
+import org.springframework.web.reactive.resource.NoResourceFoundException;
 import reactor.core.publisher.Mono;
+import reactor.core.publisher.SignalType;
 
 import java.util.*;
 import java.util.function.Function;
@@ -147,15 +150,32 @@ private Mono<Void> filterInternal(ServerWebExchange exchange, Function<ServerWeb
         // define default values from request perspective. they could be overwritten then
         setDefaults(exchange, otelContext);
 
+        // capture attempted service for error messages
+        var pathElements = exchange.getRequest().getPath().elements();
+        var attemptedService = pathElements.size() > 1 ? pathElements.get(1).value() : SERVICE_GATEWAY;
+
         return filter.apply(exchange)
+            // downstream chain: route matching → routing → service call
+            .doOnError(NoResourceFoundException.class, e -> {
+                otelContext.statusCode(404);
+                otelContext.errorType("Service not onboarded");
+                otelContext.errorMessage("Service " + attemptedService + " is not registered in the API ML");
+            })
+            .doOnError(ServiceNotAccessibleException.class, e -> {
+                otelContext.statusCode(503);
+                otelContext.errorType("Service instance not available");
+                otelContext.errorMessage(Objects.toString(e.getMessage(), "No available instances"));
+            })
             // in all cases (success / error) issue the log message
-            .doFinally(signalType -> otelContext.issue())
-            // update response codes
-            .then(Mono.fromRunnable(() -> Optional.ofNullable(exchange.getResponse())
-                .map(ServerHttpResponse::getStatusCode)
-                .map(HttpStatusCode::value)
-                .ifPresent(otelContext::responseCode)
-            ));
+            .doFinally(signalType -> {
+                if (signalType == SignalType.ON_COMPLETE) {
+                    Optional.ofNullable(exchange.getResponse())
+                        .map(ServerHttpResponse::getStatusCode)
+                        .map(HttpStatusCode::value)
+                        .ifPresent(otelContext::responseCode);
+                }
+                otelContext.issue();
+            });
     }
 
     @Override

apiml/src/test/java/org/zowe/apiml/filter/OtelRequestFilterTest.java

@@ -20,8 +20,10 @@
 import org.springframework.mock.http.server.reactive.MockServerHttpRequest;
 import org.springframework.mock.web.server.MockServerWebExchange;
 import org.springframework.test.util.ReflectionTestUtils;
+import org.springframework.web.reactive.resource.NoResourceFoundException;
 import org.springframework.web.server.WebFilterChain;
 import org.zowe.apiml.product.opentelemetry.OtelRequestContext;
+import org.zowe.apiml.security.common.error.ServiceNotAccessibleException;
 import reactor.core.publisher.Mono;
 import reactor.test.StepVerifier;
 
@@ -149,4 +151,50 @@ void givenRequest_whenFailed_thenIssueLog() {
         verify(otelContext, times(1)).issue();
     }
 
+    @Test
+    void givenUnknownService_whenNoResourceFound_thenSet404AndErrorType() {
+        var filter = new OtelRequestFilter();
+
+        var request = MockServerHttpRequest.get("http://localhost/unknownservice/api/v1/data")
+            .localAddress(InetSocketAddress.createUnresolved("localhost", 10010)).build();
+        var exchange = MockServerWebExchange.from(request);
+        var otelContext = spy(OtelRequestContext.of(exchange));
+        exchange.getAttributes().put(OTEL_CONTEXT, otelContext);
+
+        var chain = (GatewayFilterChain) e -> Mono.error(
+            new NoResourceFoundException(exchange.getRequest().getURI().getPath()));
+
+        StepVerifier.create(filter.filter(exchange, chain))
+            .expectError(NoResourceFoundException.class)
+            .verify();
+
+        verify(otelContext, times(1)).statusCode(404);
+        verify(otelContext, times(1)).errorType("Service not onboarded");
+        verify(otelContext, times(1)).errorMessage("Service unknownservice is not registered in the API ML");
+        verify(otelContext, times(1)).issue();
+    }
+
+    @Test
+    void givenServiceNotAccessible_whenCalled_thenSet503AndErrorType() {
+        var filter = new OtelRequestFilter();
+
+        var request = MockServerHttpRequest.get("http://localhost/discoverable-service/api/v1/data")
+            .localAddress(InetSocketAddress.createUnresolved("localhost", 10010)).build();
+        var exchange = MockServerWebExchange.from(request);
+        var otelContext = spy(OtelRequestContext.of(exchange));
+        exchange.getAttributes().put(OTEL_CONTEXT, otelContext);
+
+        var chain = (GatewayFilterChain) e -> Mono.error(
+            new ServiceNotAccessibleException("Service discoverable-service has no available instances"));
+
+        StepVerifier.create(filter.filter(exchange, chain))
+            .expectError(ServiceNotAccessibleException.class)
+            .verify();
+
+        verify(otelContext, times(1)).statusCode(503);
+        verify(otelContext, times(1)).errorType("Service instance not available");
+        verify(otelContext, times(1)).errorMessage("Service discoverable-service has no available instances");
+        verify(otelContext, times(1)).issue();
+    }
+
 }

gateway-service/src/main/java/org/zowe/apiml/product/opentelemetry/OtelRequestContext.java

@@ -48,6 +48,9 @@ public final class OtelRequestContext {
     private static final String OTEL_ATTRIBUTE_AUTH_ERROR_MESSAGE = "auth.error.message";
     private static final String OTEL_ATTRIBUTE_USER_ID = "user.id";
     private static final String OTEL_ATTRIBUTE_DISTRIBUTED_USER_ID = "user.distributed.id";
+    private static final String OTEL_ATTRIBUTE_STATUS_CODE = "http.response.status_code";
+    private static final String OTEL_ATTRIBUTE_ERROR_TYPE = "error.type";
+    private static final String OTEL_ATTRIBUTE_ERROR_MESSAGE = "error.message";
 
     private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
 
@@ -88,6 +91,18 @@ public OtelRequestContext responseCode(int status) {
         return put(OTEL_ATTRIBUTE_RESPONSE_CODE, String.valueOf(status));
     }
 
+    public OtelRequestContext statusCode(int status) {
+        return put(OTEL_ATTRIBUTE_STATUS_CODE, String.valueOf(status));
+    }
+
+    public OtelRequestContext errorType(String errorType) {
+        return put(OTEL_ATTRIBUTE_ERROR_TYPE, errorType);
+    }
+
+    public OtelRequestContext errorMessage(String errorMessage) {
+        return put(OTEL_ATTRIBUTE_ERROR_MESSAGE, errorMessage);
+    }
+
     public OtelRequestContext serviceId(String serviceId) {
         return put(OTEL_ATTRIBUTE_SERVICE_ID, StringUtils.lowerCase(serviceId));
     }

gateway-service/src/test/java/org/zowe/apiml/product/opentelemetry/OtelRequestContextTest.java

@@ -86,6 +86,24 @@ void givenOtelContext_whenSetResponseCode_thenTransformToString() {
         assertEquals("204", getValue("service.response_code"));
     }
 
+    @Test
+    void givenOtelContext_whenSetStatusCode_thenTransformToString() {
+        OtelRequestContext.of(exchange).statusCode(503);
+        assertEquals("503", getValue("http.response.status_code"));
+    }
+
+    @Test
+    void givenOtelContext_whenSetErrorType_thenStoreIt() {
+        OtelRequestContext.of(exchange).errorType("Service not onboarded");
+        assertEquals("Service not onboarded", getValue("error.type"));
+    }
+
+    @Test
+    void givenOtelContext_whenSetErrorMessage_thenStoreIt() {
+        OtelRequestContext.of(exchange).errorMessage("Service instance not available");
+        assertEquals("Service instance not available", getValue("error.message"));
+    }
+
     @Test
     void givenOtelContext_whenSetServiceId_thenStoreLowerCase() {
         OtelRequestContext.of(exchange).serviceId("serviceID");

otel/README.md

@@ -93,3 +93,49 @@ Note that `docker compose` cli arguments override the `command` value in the doc
 ## Local run for development
 
 To run the docker containers locally with the same setup as used in the integration tests, just run `docker compose up` (optionally with `-d`), or the scripts in the [sh](sh) directory, and then start the APIML modulith with the OpenTelemetry enabled. The signals received and exported by the collector are saved to the [otel-golden](otel-golden) folder. The Golden Tester exits after timeout reporting the result of validation in the container console/log. The timeout can be set in the [docker-compose.yml](docker-compose.yml) file.
+
+## HTTP Error Attributes
+
+Starting with issue [#4705](https://github.com/zowe/api-layer/issues/4705), the Gateway's `OtelRequestFilter` emits OpenTelemetry log signals with the following attributes when a routed request results in an error:
+
+| Attribute | Key | Type | Description |
+|-----------|-----|------|-------------|
+| HTTP response status code | `http.response.status_code` | int | The actual HTTP status code returned to the client (e.g., 404, 503) |
+| Error type | `error.type` | string | A short machine-readable string identifying the error category |
+| Error message | `error.message` | string | A human-readable description of the error |
+
+These attributes complement the existing [`service.response_code`] attribute — `http.response.status_code` captures the HTTP status sent to the client, while `service.response_code` reflects the HTTP status from the downstream service call.
+
+### Error Scenarios
+
+Two error paths produce OTel log signals:
+
+1. **Unknown Service ID (404):** When a request targets a Service ID that is not registered in the Discovery Service, the Gateway returns a 404. The OTel signal includes:
+
+   - `http.response.status_code` = `404`
+   - `error.type` = `"Service not onboarded"`
+   - `error.message` = `"Service <serviceId> is not registered in the API ML"`
+
+2. **Service Instances Down (503):** When a request targets a registered Service ID that has no available instances, the Gateway returns a 503. The OTel signal includes:
+
+   - `http.response.status_code` = `503`
+   - `error.type` = `"Service instance not available"`
+   - `error.message` = the exception message from `ServiceNotAccessibleException`
+
+### Monitoring Use
+
+Operators can use these attributes in observability dashboards and alerting rules. Example Prometheus / Alertmanager alert expressions:
+
+```yaml
+# Alert when an unknown service is requested (404)
+- alert: UnknownServiceRequested
+  expr: increase(otel_log_count{error_type="Service not onboarded"}[5m]) > 0
+  annotations:
+    summary: "Requests targeting unknown service ID"
+
+# Alert when a registered service has no available instances (503)
+- alert: ServiceInstancesDown
+  expr: increase(otel_log_count{error_type="Service instance not available"}[5m]) > 0
+  annotations:
+    summary: "Service has no available instances"
+```