feat: include a startup message for full HA setup

apiml-utility/src/main/resources/utility-log-messages.yml

@@ -14,7 +14,14 @@ messages:
       number: ZWEAM001
       type: INFO
       text: "API Mediation Layer started"
-      reason: "All key API Mediation Layer services started."
+      reason: "All key API Mediation Layer services started. At least one instance of API ML is available"
+      action: "No action required."
+
+    - key: org.zowe.apiml.common.mediationLayerStartedHA
+      number: ZWEAM002
+      type: INFO
+      text: "High Availability initialization complete"
+      reason: "All key API Mediation Layer services started. Full configured redundancy achieved"
       action: "No action required."
 
     - key: org.zowe.apiml.cache.errorOpeningCachingFiles

apiml/src/main/java/org/zowe/apiml/GatewayHealthIndicator.java

@@ -13,7 +13,7 @@
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.actuate.health.AbstractHealthIndicator;
 import org.springframework.boot.actuate.health.Health.Builder;
@@ -32,7 +32,9 @@
 import org.zowe.apiml.product.service.ServiceStartupEventHandler;
 import org.zowe.apiml.zaas.ZaasServiceAvailableEvent;
 
+import java.util.Optional;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.springframework.boot.actuate.health.Status.DOWN;
 import static org.springframework.boot.actuate.health.Status.UP;
@@ -46,14 +48,16 @@
 @Component
 @RequiredArgsConstructor
 @Slf4j
-public class GatewayHealthIndicator extends AbstractHealthIndicator {
+public class GatewayHealthIndicator extends AbstractHealthIndicator implements InitializingBean {
 
     private final ApplicationContext applicationContext;
     private final ServiceStartupEventHandler serviceStartupEventHandler;
 
     @InjectApimlLogger
     private final ApimlLogger apimlLog = ApimlLogger.empty();
 
+    private DiscoveryClient discoveryClient;
+
     @Value("${apiml.catalog.serviceId:}")
     private String apiCatalogServiceId;
 
@@ -62,23 +66,30 @@ public class GatewayHealthIndicator extends AbstractHealthIndicator {
     private AtomicBoolean catalogAvailable = new AtomicBoolean(false);
 
     private AtomicBoolean startedInformationPublished = new AtomicBoolean(false);
+    private AtomicBoolean startedHaInformationPublished = new AtomicBoolean(false);
+
+    private AtomicInteger gatewayCount = new AtomicInteger(0);
+    private AtomicInteger zaasCount = new AtomicInteger(0);
+
+    private Integer expectedInstanceCount;
+
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        expectedInstanceCount = Optional.ofNullable(System.getenv("ZWE_DISCOVERY_SERVICES_LIST"))
+            .map(discoveryServicesList -> discoveryServicesList.split(","))
+            .map(i -> i.length)
+            .orElse(1);
+
+        discoveryClient = applicationContext.getBean(DiscoveryClient.class);
+    }
 
     @Override
     protected void doHealthCheck(Builder builder) throws Exception {
         var anyCatalogIsAvailable = StringUtils.isNotBlank(apiCatalogServiceId);
-        DiscoveryClient discoveryClient;
-        try {
-            discoveryClient = applicationContext.getBean(DiscoveryClient.class);
-        } catch (BeansException e) {
-            log.debug("DiscoveryClient is not available", e);
-            return;
-        }
 
         catalogAvailable.set(anyCatalogIsAvailable && !discoveryClient.getInstances(apiCatalogServiceId).isEmpty());
 
-        // Keeping for backwards compatibility, in modulith the amount of gateways is the amount of authentication services available
-        var gatewayCount = discoveryClient.getInstances(CoreService.GATEWAY.getServiceId()).size();
-        var zaasCount = gatewayCount;
+        refreshInstanceCounts();
 
         builder.status(toStatus(discoveryAvailable.get() && zaasAvailable.get()))
             .withDetail(CoreService.DISCOVERY.getServiceId(), toStatus(discoveryAvailable.get()).getCode())
@@ -93,6 +104,15 @@ protected void doHealthCheck(Builder builder) throws Exception {
         if (isFullyUp()) {
             onFullyUp();
         }
+        if (isFullyHaUp()) {
+            onFullyHaUp();
+        }
+    }
+
+    private void refreshInstanceCounts() {
+        // Keeping for backwards compatibility, in modulith the amount of gateways is the amount of authentication services available
+        gatewayCount.compareAndSet(expectedInstanceCount, this.discoveryClient.getInstances(CoreService.GATEWAY.getServiceId()).size());
+        zaasCount.set(gatewayCount.get());
     }
 
     private boolean isFullyUp() {
@@ -105,12 +125,29 @@ private void onFullyUp() {
         }
     }
 
+    private boolean isFullyHaUp() {
+        if (expectedInstanceCount > 1) {
+            refreshInstanceCounts();
+            return expectedInstanceCount == gatewayCount.get();
+        }
+        return false;
+    }
+
+    private void onFullyHaUp() {
+        if (startedHaInformationPublished.compareAndSet(false, true)) {
+            apimlLog.log("org.zowe.apiml.common.mediationLayerStartedHA");
+        }
+    }
+
     @EventListener
     public void onApplicationEvent(ZaasServiceAvailableEvent event) {
         zaasAvailable.set(true);
         if (isFullyUp()) {
             onFullyUp();
         }
+        if (isFullyHaUp()) {
+            onFullyHaUp();
+        }
     }
 
     @EventListener
@@ -119,6 +156,9 @@ public void onApplicationEvent(EurekaRegistryAvailableEvent event) {
         if (isFullyUp()) {
             onFullyUp();
         }
+        if (isFullyHaUp()) {
+            onFullyHaUp();
+        }
     }
 
     @EventListener
@@ -130,6 +170,9 @@ public void onApplicationEvent(EurekaInstanceRegisteredEvent event) {
         if (isFullyUp()) {
             onFullyUp();
         }
+        if (isFullyHaUp()) {
+            onFullyHaUp();
+        }
     }
 
     @EventListener
@@ -140,6 +183,9 @@ public void onApplicationEvent(ApiCatalogServiceAvailableEvent event) {
         if (isFullyUp()) {
             onFullyUp();
         }
+        if (isFullyHaUp()) {
+            onFullyHaUp();
+        }
     }
 
     boolean isStartedInformationPublished() {

apiml/src/test/java/org/zowe/apiml/GatewayHealthIndicatorTest.java

@@ -18,7 +18,6 @@
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
-import org.springframework.beans.factory.NoSuchBeanDefinitionException;
 import org.springframework.boot.actuate.health.Health;
 import org.springframework.boot.actuate.health.Status;
 import org.springframework.cloud.client.DefaultServiceInstance;
@@ -41,23 +40,29 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoInteractions;
 import static org.mockito.Mockito.when;
 
 @ExtendWith(MockitoExtension.class)
 class GatewayHealthIndicatorTest {
 
-    @Mock private DiscoveryClient discoveryClient;
-    @Mock private ApplicationContext applicationContext;
-    @Mock private ServiceStartupEventHandler serviceStartupEventHandler;
+    @Mock
+    private DiscoveryClient discoveryClient;
+
+    @Mock
+    private ApplicationContext applicationContext;
+
+    @Mock
+    private ServiceStartupEventHandler serviceStartupEventHandler;
 
     private GatewayHealthIndicator healthIndicator;
 
     @BeforeEach
-    void setUp() {
+    void setUp() throws Exception {
         healthIndicator = new GatewayHealthIndicator(applicationContext, serviceStartupEventHandler);
         ReflectionTestUtils.setField(healthIndicator, "apiCatalogServiceId", CoreService.API_CATALOG.getServiceId());
+        ReflectionTestUtils.setField(healthIndicator, "expectedInstanceCount", 1);
         lenient().when(applicationContext.getBean(DiscoveryClient.class)).thenReturn(discoveryClient);
+        healthIndicator.afterPropertiesSet();
     }
 
     private DefaultServiceInstance getDefaultServiceInstance(String serviceId, String hostname, int port) {
@@ -111,17 +116,6 @@ void thenStatusIsDown() throws Exception {
             assertEquals(Status.DOWN, builder.build().getStatus());
         }
 
-        @Test
-        void whenClientNotAvailable_thenDoNothing() throws Exception {
-            when(applicationContext.getBean(DiscoveryClient.class)).thenThrow(new NoSuchBeanDefinitionException(DiscoveryClient.class));
-
-            Health.Builder builder = new Health.Builder();
-            healthIndicator.doHealthCheck(builder);
-
-            verifyNoInteractions(serviceStartupEventHandler);
-            verifyNoInteractions(discoveryClient);
-        }
-
     }
 
     @Nested
@@ -195,11 +189,14 @@ void whenHealthRequested_onceLogMessageAboutStartup() throws Exception {
     @Nested
     class OnCatalogRegistration {
 
+        @Mock
+        private EurekaInstanceRegisteredEvent registeredEvent;
+
+        @Mock
+        private InstanceInfo instanceInfo;
+
         @Test
         void whenBothEvents_thenOneMessage() {
-            var registeredEvent = mock(EurekaInstanceRegisteredEvent.class);
-
-            var instanceInfo = mock(InstanceInfo.class);
             when(registeredEvent.getInstanceInfo()).thenReturn(instanceInfo);
             when(instanceInfo.getAppName()).thenReturn("apicatalog");
 
@@ -213,9 +210,6 @@ void whenBothEvents_thenOneMessage() {
 
         @Test
         void whenBothEventsReverse_thenOneMessage() {
-            var registeredEvent = mock(EurekaInstanceRegisteredEvent.class);
-
-            var instanceInfo = mock(InstanceInfo.class);
             when(registeredEvent.getInstanceInfo()).thenReturn(instanceInfo);
             when(instanceInfo.getAppName()).thenReturn("apicatalog");
 

gateway-service/src/main/java/org/zowe/apiml/gateway/config/GatewayHealthIndicator.java

@@ -10,7 +10,9 @@
 
 package org.zowe.apiml.gateway.config;
 
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.actuate.health.AbstractHealthIndicator;
 import org.springframework.boot.actuate.health.Health;
@@ -25,7 +27,9 @@
 import org.zowe.apiml.product.constants.CoreService;
 import org.zowe.apiml.product.logging.annotations.InjectApimlLogger;
 
+import java.util.Optional;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.springframework.boot.actuate.health.Status.DOWN;
 import static org.springframework.boot.actuate.health.Status.UP;
@@ -37,22 +41,40 @@
  */
 @Component
 @ConditionalOnMissingBean(name = "modulithConfig")
-public class GatewayHealthIndicator extends AbstractHealthIndicator {
+@Slf4j
+public class GatewayHealthIndicator extends AbstractHealthIndicator implements InitializingBean {
+
+    private static final String ZWE_DISCOVERY_SERVICES_LIST = "ZWE_DISCOVERY_SERVICES_LIST";
 
     protected final DiscoveryClient discoveryClient;
     private final String apiCatalogServiceId;
     @InjectApimlLogger
     private final ApimlLogger apimlLog = ApimlLogger.empty();
 
     private AtomicBoolean startedInformationPublished = new AtomicBoolean(false);
+    private AtomicBoolean startedHaInformationPublished = new AtomicBoolean(false);
     private AtomicBoolean applicationReady = new AtomicBoolean(false);
 
+    private AtomicInteger gatewayCount = new AtomicInteger(0);
+    private AtomicInteger zaasCount = new AtomicInteger(0);
+    private AtomicInteger discoveryCount = new AtomicInteger(0);
+
+    private int expectedInstanceCount;
+
     public GatewayHealthIndicator(DiscoveryClient discoveryClient,
                                   @Value("${apiml.catalog.serviceId:}") String apiCatalogServiceId) {
         this.discoveryClient = discoveryClient;
         this.apiCatalogServiceId = apiCatalogServiceId;
     }
 
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        this.expectedInstanceCount = Optional.ofNullable(System.getenv(ZWE_DISCOVERY_SERVICES_LIST))
+            .map(discoveryServicesList -> discoveryServicesList.split(","))
+            .map(i -> i.length)
+            .orElse(1);
+    }
+
     @Override
     protected void doHealthCheck(Health.Builder builder) {
         var anyCatalogIsAvailable = StringUtils.isNotBlank(apiCatalogServiceId);
@@ -63,8 +85,7 @@ protected void doHealthCheck(Health.Builder builder) {
         var discoveryUp = !this.discoveryClient.getInstances(CoreService.DISCOVERY.getServiceId()).isEmpty();
         var zaasUp = !this.discoveryClient.getInstances(CoreService.ZAAS.getServiceId()).isEmpty();
 
-        var gatewayCount = this.discoveryClient.getInstances(CoreService.GATEWAY.getServiceId()).size();
-        var zaasCount = this.discoveryClient.getInstances(CoreService.ZAAS.getServiceId()).size();
+        refreshInstanceCounts();
 
         builder.status(toStatus(discoveryUp))
             .withDetail(CoreService.DISCOVERY.getServiceId(), toStatus(discoveryUp).getCode())
@@ -76,9 +97,19 @@ protected void doHealthCheck(Health.Builder builder) {
             builder.withDetail(CoreService.API_CATALOG.getServiceId(), toStatus(apiCatalogUp).getCode());
         }
 
+        // check number of instances (non-modulith)
         if (discoveryUp && apiCatalogUp && zaasUp && applicationReady.get()) {
             onFullyUp();
         }
+        if (isFullyHaUp()) {
+            onFullyHaUp();
+        }
+    }
+
+    private void refreshInstanceCounts() {
+        gatewayCount.compareAndSet(expectedInstanceCount, this.discoveryClient.getInstances(CoreService.GATEWAY.getServiceId()).size());
+        discoveryCount.compareAndSet(expectedInstanceCount, this.discoveryClient.getInstances(CoreService.DISCOVERY.getServiceId()).size());
+        zaasCount.compareAndSet(expectedInstanceCount, this.discoveryClient.getInstances(CoreService.ZAAS.getServiceId()).size());
     }
 
     @EventListener(ApplicationReadyEvent.class)
@@ -92,11 +123,28 @@ private void onFullyUp() {
         }
     }
 
+    private boolean isFullyHaUp() {
+        if (expectedInstanceCount > 1) {
+            refreshInstanceCounts();
+            return expectedInstanceCount == gatewayCount.get()
+                && expectedInstanceCount == zaasCount.get()
+                && expectedInstanceCount == discoveryCount.get();
+        }
+        return false;
+    }
+
+    private void onFullyHaUp() {
+        if (startedHaInformationPublished.compareAndSet(false, true)) {
+            apimlLog.log("null");
+        }
+    }
+
     boolean isStartedInformationPublished() {
         return startedInformationPublished.get();
     }
 
     private Status toStatus(boolean up) {
         return up ? UP : DOWN;
     }
+
 }