diff --git a/apiml-utility/src/main/resources/utility-log-messages.yml b/apiml-utility/src/main/resources/utility-log-messages.yml index 7a8466faa3..905c1d4bee 100644 --- a/apiml-utility/src/main/resources/utility-log-messages.yml +++ b/apiml-utility/src/main/resources/utility-log-messages.yml @@ -14,7 +14,14 @@ messages: number: ZWEAM001 type: INFO text: "API Mediation Layer started" - reason: "All key API Mediation Layer services started." + reason: "All key API Mediation Layer services started. At least one instance of API ML is available" + action: "No action required." + + - key: org.zowe.apiml.common.mediationLayerStartedHA + number: ZWEAM002 + type: INFO + text: "High Availability initialization complete" + reason: "All key API Mediation Layer services started. Full configured redundancy achieved" action: "No action required." - key: org.zowe.apiml.cache.errorOpeningCachingFiles diff --git a/apiml/src/main/java/org/zowe/apiml/GatewayHealthIndicator.java b/apiml/src/main/java/org/zowe/apiml/GatewayHealthIndicator.java index f4d37f5425..413714699e 100644 --- a/apiml/src/main/java/org/zowe/apiml/GatewayHealthIndicator.java +++ b/apiml/src/main/java/org/zowe/apiml/GatewayHealthIndicator.java @@ -13,7 +13,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.BeansException; +import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.actuate.health.AbstractHealthIndicator; import org.springframework.boot.actuate.health.Health.Builder; @@ -32,7 +32,9 @@ import org.zowe.apiml.product.service.ServiceStartupEventHandler; import org.zowe.apiml.zaas.ZaasServiceAvailableEvent; +import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import static org.springframework.boot.actuate.health.Status.DOWN; import static org.springframework.boot.actuate.health.Status.UP; @@ -46,7 +48,7 @@ @Component @RequiredArgsConstructor @Slf4j -public class GatewayHealthIndicator extends AbstractHealthIndicator { +public class GatewayHealthIndicator extends AbstractHealthIndicator implements InitializingBean { private final ApplicationContext applicationContext; private final ServiceStartupEventHandler serviceStartupEventHandler; @@ -54,6 +56,8 @@ public class GatewayHealthIndicator extends AbstractHealthIndicator { @InjectApimlLogger private final ApimlLogger apimlLog = ApimlLogger.empty(); + private DiscoveryClient discoveryClient; + @Value("${apiml.catalog.serviceId:}") private String apiCatalogServiceId; @@ -62,23 +66,30 @@ public class GatewayHealthIndicator extends AbstractHealthIndicator { private AtomicBoolean catalogAvailable = new AtomicBoolean(false); private AtomicBoolean startedInformationPublished = new AtomicBoolean(false); + private AtomicBoolean startedHaInformationPublished = new AtomicBoolean(false); + + private AtomicInteger gatewayCount = new AtomicInteger(0); + private AtomicInteger zaasCount = new AtomicInteger(0); + + private Integer expectedInstanceCount; + + @Override + public void afterPropertiesSet() throws Exception { + expectedInstanceCount = Optional.ofNullable(System.getenv("ZWE_DISCOVERY_SERVICES_LIST")) + .map(discoveryServicesList -> discoveryServicesList.split(",")) + .map(i -> i.length) + .orElse(1); + + discoveryClient = applicationContext.getBean(DiscoveryClient.class); + } @Override protected void doHealthCheck(Builder builder) throws Exception { var anyCatalogIsAvailable = StringUtils.isNotBlank(apiCatalogServiceId); - DiscoveryClient discoveryClient; - try { - discoveryClient = applicationContext.getBean(DiscoveryClient.class); - } catch (BeansException e) { - log.debug("DiscoveryClient is not available", e); - return; - } catalogAvailable.set(anyCatalogIsAvailable && !discoveryClient.getInstances(apiCatalogServiceId).isEmpty()); - // Keeping for backwards compatibility, in modulith the amount of gateways is the amount of authentication services available - var gatewayCount = discoveryClient.getInstances(CoreService.GATEWAY.getServiceId()).size(); - var zaasCount = gatewayCount; + refreshInstanceCounts(); builder.status(toStatus(discoveryAvailable.get() && zaasAvailable.get())) .withDetail(CoreService.DISCOVERY.getServiceId(), toStatus(discoveryAvailable.get()).getCode()) @@ -93,6 +104,15 @@ protected void doHealthCheck(Builder builder) throws Exception { if (isFullyUp()) { onFullyUp(); } + if (isFullyHaUp()) { + onFullyHaUp(); + } + } + + private void refreshInstanceCounts() { + // Keeping for backwards compatibility, in modulith the amount of gateways is the amount of authentication services available + gatewayCount.compareAndSet(expectedInstanceCount, this.discoveryClient.getInstances(CoreService.GATEWAY.getServiceId()).size()); + zaasCount.set(gatewayCount.get()); } private boolean isFullyUp() { @@ -105,12 +125,29 @@ private void onFullyUp() { } } + private boolean isFullyHaUp() { + if (expectedInstanceCount > 1) { + refreshInstanceCounts(); + return expectedInstanceCount == gatewayCount.get(); + } + return false; + } + + private void onFullyHaUp() { + if (startedHaInformationPublished.compareAndSet(false, true)) { + apimlLog.log("org.zowe.apiml.common.mediationLayerStartedHA"); + } + } + @EventListener public void onApplicationEvent(ZaasServiceAvailableEvent event) { zaasAvailable.set(true); if (isFullyUp()) { onFullyUp(); } + if (isFullyHaUp()) { + onFullyHaUp(); + } } @EventListener @@ -119,6 +156,9 @@ public void onApplicationEvent(EurekaRegistryAvailableEvent event) { if (isFullyUp()) { onFullyUp(); } + if (isFullyHaUp()) { + onFullyHaUp(); + } } @EventListener @@ -130,6 +170,9 @@ public void onApplicationEvent(EurekaInstanceRegisteredEvent event) { if (isFullyUp()) { onFullyUp(); } + if (isFullyHaUp()) { + onFullyHaUp(); + } } @EventListener @@ -140,6 +183,9 @@ public void onApplicationEvent(ApiCatalogServiceAvailableEvent event) { if (isFullyUp()) { onFullyUp(); } + if (isFullyHaUp()) { + onFullyHaUp(); + } } boolean isStartedInformationPublished() { diff --git a/apiml/src/test/java/org/zowe/apiml/GatewayHealthIndicatorTest.java b/apiml/src/test/java/org/zowe/apiml/GatewayHealthIndicatorTest.java index 2ccb3bbc16..e013161df8 100644 --- a/apiml/src/test/java/org/zowe/apiml/GatewayHealthIndicatorTest.java +++ b/apiml/src/test/java/org/zowe/apiml/GatewayHealthIndicatorTest.java @@ -18,7 +18,6 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.beans.factory.NoSuchBeanDefinitionException; import org.springframework.boot.actuate.health.Health; import org.springframework.boot.actuate.health.Status; import org.springframework.cloud.client.DefaultServiceInstance; @@ -41,23 +40,29 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) class GatewayHealthIndicatorTest { - @Mock private DiscoveryClient discoveryClient; - @Mock private ApplicationContext applicationContext; - @Mock private ServiceStartupEventHandler serviceStartupEventHandler; + @Mock + private DiscoveryClient discoveryClient; + + @Mock + private ApplicationContext applicationContext; + + @Mock + private ServiceStartupEventHandler serviceStartupEventHandler; private GatewayHealthIndicator healthIndicator; @BeforeEach - void setUp() { + void setUp() throws Exception { healthIndicator = new GatewayHealthIndicator(applicationContext, serviceStartupEventHandler); ReflectionTestUtils.setField(healthIndicator, "apiCatalogServiceId", CoreService.API_CATALOG.getServiceId()); + ReflectionTestUtils.setField(healthIndicator, "expectedInstanceCount", 1); lenient().when(applicationContext.getBean(DiscoveryClient.class)).thenReturn(discoveryClient); + healthIndicator.afterPropertiesSet(); } private DefaultServiceInstance getDefaultServiceInstance(String serviceId, String hostname, int port) { @@ -111,17 +116,6 @@ void thenStatusIsDown() throws Exception { assertEquals(Status.DOWN, builder.build().getStatus()); } - @Test - void whenClientNotAvailable_thenDoNothing() throws Exception { - when(applicationContext.getBean(DiscoveryClient.class)).thenThrow(new NoSuchBeanDefinitionException(DiscoveryClient.class)); - - Health.Builder builder = new Health.Builder(); - healthIndicator.doHealthCheck(builder); - - verifyNoInteractions(serviceStartupEventHandler); - verifyNoInteractions(discoveryClient); - } - } @Nested @@ -195,11 +189,14 @@ void whenHealthRequested_onceLogMessageAboutStartup() throws Exception { @Nested class OnCatalogRegistration { + @Mock + private EurekaInstanceRegisteredEvent registeredEvent; + + @Mock + private InstanceInfo instanceInfo; + @Test void whenBothEvents_thenOneMessage() { - var registeredEvent = mock(EurekaInstanceRegisteredEvent.class); - - var instanceInfo = mock(InstanceInfo.class); when(registeredEvent.getInstanceInfo()).thenReturn(instanceInfo); when(instanceInfo.getAppName()).thenReturn("apicatalog"); @@ -213,9 +210,6 @@ void whenBothEvents_thenOneMessage() { @Test void whenBothEventsReverse_thenOneMessage() { - var registeredEvent = mock(EurekaInstanceRegisteredEvent.class); - - var instanceInfo = mock(InstanceInfo.class); when(registeredEvent.getInstanceInfo()).thenReturn(instanceInfo); when(instanceInfo.getAppName()).thenReturn("apicatalog"); diff --git a/gateway-service/src/main/java/org/zowe/apiml/gateway/config/GatewayHealthIndicator.java b/gateway-service/src/main/java/org/zowe/apiml/gateway/config/GatewayHealthIndicator.java index 5512d1b912..89587ad228 100644 --- a/gateway-service/src/main/java/org/zowe/apiml/gateway/config/GatewayHealthIndicator.java +++ b/gateway-service/src/main/java/org/zowe/apiml/gateway/config/GatewayHealthIndicator.java @@ -10,7 +10,9 @@ package org.zowe.apiml.gateway.config; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.actuate.health.AbstractHealthIndicator; import org.springframework.boot.actuate.health.Health; @@ -25,7 +27,9 @@ import org.zowe.apiml.product.constants.CoreService; import org.zowe.apiml.product.logging.annotations.InjectApimlLogger; +import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import static org.springframework.boot.actuate.health.Status.DOWN; import static org.springframework.boot.actuate.health.Status.UP; @@ -37,7 +41,10 @@ */ @Component @ConditionalOnMissingBean(name = "modulithConfig") -public class GatewayHealthIndicator extends AbstractHealthIndicator { +@Slf4j +public class GatewayHealthIndicator extends AbstractHealthIndicator implements InitializingBean { + + private static final String ZWE_DISCOVERY_SERVICES_LIST = "ZWE_DISCOVERY_SERVICES_LIST"; protected final DiscoveryClient discoveryClient; private final String apiCatalogServiceId; @@ -45,14 +52,29 @@ public class GatewayHealthIndicator extends AbstractHealthIndicator { private final ApimlLogger apimlLog = ApimlLogger.empty(); private AtomicBoolean startedInformationPublished = new AtomicBoolean(false); + private AtomicBoolean startedHaInformationPublished = new AtomicBoolean(false); private AtomicBoolean applicationReady = new AtomicBoolean(false); + private AtomicInteger gatewayCount = new AtomicInteger(0); + private AtomicInteger zaasCount = new AtomicInteger(0); + private AtomicInteger discoveryCount = new AtomicInteger(0); + + private int expectedInstanceCount; + public GatewayHealthIndicator(DiscoveryClient discoveryClient, @Value("${apiml.catalog.serviceId:}") String apiCatalogServiceId) { this.discoveryClient = discoveryClient; this.apiCatalogServiceId = apiCatalogServiceId; } + @Override + public void afterPropertiesSet() throws Exception { + this.expectedInstanceCount = Optional.ofNullable(System.getenv(ZWE_DISCOVERY_SERVICES_LIST)) + .map(discoveryServicesList -> discoveryServicesList.split(",")) + .map(i -> i.length) + .orElse(1); + } + @Override protected void doHealthCheck(Health.Builder builder) { var anyCatalogIsAvailable = StringUtils.isNotBlank(apiCatalogServiceId); @@ -63,8 +85,7 @@ protected void doHealthCheck(Health.Builder builder) { var discoveryUp = !this.discoveryClient.getInstances(CoreService.DISCOVERY.getServiceId()).isEmpty(); var zaasUp = !this.discoveryClient.getInstances(CoreService.ZAAS.getServiceId()).isEmpty(); - var gatewayCount = this.discoveryClient.getInstances(CoreService.GATEWAY.getServiceId()).size(); - var zaasCount = this.discoveryClient.getInstances(CoreService.ZAAS.getServiceId()).size(); + refreshInstanceCounts(); builder.status(toStatus(discoveryUp)) .withDetail(CoreService.DISCOVERY.getServiceId(), toStatus(discoveryUp).getCode()) @@ -76,9 +97,19 @@ protected void doHealthCheck(Health.Builder builder) { builder.withDetail(CoreService.API_CATALOG.getServiceId(), toStatus(apiCatalogUp).getCode()); } + // check number of instances (non-modulith) if (discoveryUp && apiCatalogUp && zaasUp && applicationReady.get()) { onFullyUp(); } + if (isFullyHaUp()) { + onFullyHaUp(); + } + } + + private void refreshInstanceCounts() { + gatewayCount.compareAndSet(expectedInstanceCount, this.discoveryClient.getInstances(CoreService.GATEWAY.getServiceId()).size()); + discoveryCount.compareAndSet(expectedInstanceCount, this.discoveryClient.getInstances(CoreService.DISCOVERY.getServiceId()).size()); + zaasCount.compareAndSet(expectedInstanceCount, this.discoveryClient.getInstances(CoreService.ZAAS.getServiceId()).size()); } @EventListener(ApplicationReadyEvent.class) @@ -92,6 +123,22 @@ private void onFullyUp() { } } + private boolean isFullyHaUp() { + if (expectedInstanceCount > 1) { + refreshInstanceCounts(); + return expectedInstanceCount == gatewayCount.get() + && expectedInstanceCount == zaasCount.get() + && expectedInstanceCount == discoveryCount.get(); + } + return false; + } + + private void onFullyHaUp() { + if (startedHaInformationPublished.compareAndSet(false, true)) { + apimlLog.log("null"); + } + } + boolean isStartedInformationPublished() { return startedInformationPublished.get(); } @@ -99,4 +146,5 @@ boolean isStartedInformationPublished() { private Status toStatus(boolean up) { return up ? UP : DOWN; } + } diff --git a/gateway-service/src/test/java/org/zowe/apiml/gateway/config/GatewayHealthIndicatorTest.java b/gateway-service/src/test/java/org/zowe/apiml/gateway/config/GatewayHealthIndicatorTest.java index 3afddfd092..fada541985 100644 --- a/gateway-service/src/test/java/org/zowe/apiml/gateway/config/GatewayHealthIndicatorTest.java +++ b/gateway-service/src/test/java/org/zowe/apiml/gateway/config/GatewayHealthIndicatorTest.java @@ -10,16 +10,24 @@ package org.zowe.apiml.gateway.config; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.boot.actuate.health.Health; import org.springframework.boot.actuate.health.Status; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.cloud.client.DefaultServiceInstance; +import org.springframework.cloud.client.ServiceInstance; import org.springframework.cloud.client.discovery.DiscoveryClient; +import org.springframework.test.util.ReflectionTestUtils; +import org.zowe.apiml.message.log.ApimlLogger; import org.zowe.apiml.product.constants.CoreService; import java.util.Collections; +import java.util.List; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; @@ -27,8 +35,19 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +@ExtendWith(MockitoExtension.class) class GatewayHealthIndicatorTest { + @Mock + private DiscoveryClient discoveryClient; + + private GatewayHealthIndicator healthIndicator; + + @BeforeEach + void setUp() { + this.healthIndicator = new GatewayHealthIndicator(discoveryClient, CoreService.API_CATALOG.getServiceId()); + } + private DefaultServiceInstance getDefaultServiceInstance(String serviceId, String hostname, int port) { return new DefaultServiceInstance( hostname + ":" + serviceId + ":" + port, @@ -38,56 +57,60 @@ private DefaultServiceInstance getDefaultServiceInstance(String serviceId, Strin @Nested class WhenCatalogAndDiscoveryAreAvailable { + @Test void testStatusIsUp() { - DiscoveryClient discoveryClient = mock(DiscoveryClient.class); when(discoveryClient.getInstances(CoreService.API_CATALOG.getServiceId())).thenReturn( Collections.singletonList(getDefaultServiceInstance(CoreService.API_CATALOG.getServiceId(), "host", 10014))); when(discoveryClient.getInstances(CoreService.DISCOVERY.getServiceId())).thenReturn( Collections.singletonList(getDefaultServiceInstance(CoreService.DISCOVERY.getServiceId(), "host", 10011))); + when(discoveryClient.getInstances(CoreService.ZAAS.getServiceId())).thenReturn( + Collections.singletonList(getDefaultServiceInstance(CoreService.ZAAS.getServiceId(), "host", 10011))); - GatewayHealthIndicator healthIndicator = new GatewayHealthIndicator(discoveryClient, CoreService.API_CATALOG.getServiceId()); Health.Builder builder = new Health.Builder(); healthIndicator.doHealthCheck(builder); assertEquals(Status.UP, builder.build().getStatus()); } + } @Nested class WhenDiscoveryIsNotAreAvailable { + @Test void testStatusIsDown() { - DiscoveryClient discoveryClient = mock(DiscoveryClient.class); when(discoveryClient.getInstances(CoreService.API_CATALOG.getServiceId())).thenReturn( Collections.singletonList(getDefaultServiceInstance(CoreService.API_CATALOG.getServiceId(), "host", 10014))); when(discoveryClient.getInstances(CoreService.DISCOVERY.getServiceId())).thenReturn(Collections.emptyList()); - GatewayHealthIndicator healthIndicator = new GatewayHealthIndicator(discoveryClient, CoreService.API_CATALOG.getServiceId()); Health.Builder builder = new Health.Builder(); healthIndicator.doHealthCheck(builder); assertEquals(Status.DOWN, builder.build().getStatus()); } + } @Nested class GivenCustomCatalogProvider { + @Test void whenHealthIsRequested_thenStatusIsUp() { String customCatalogServiceId = "customCatalog"; - DiscoveryClient discoveryClient = mock(DiscoveryClient.class); when(discoveryClient.getInstances(customCatalogServiceId)).thenReturn( Collections.singletonList(getDefaultServiceInstance(customCatalogServiceId, "host", 10014))); when(discoveryClient.getInstances(CoreService.DISCOVERY.getServiceId())).thenReturn( Collections.singletonList(getDefaultServiceInstance(CoreService.DISCOVERY.getServiceId(), "host", 10011))); - GatewayHealthIndicator healthIndicator = new GatewayHealthIndicator(discoveryClient, customCatalogServiceId); + var healthIndicator = new GatewayHealthIndicator(discoveryClient, customCatalogServiceId); + Health.Builder builder = new Health.Builder(); healthIndicator.doHealthCheck(builder); String code = (String) builder.build().getDetails().get(CoreService.API_CATALOG.getServiceId()); assertThat(code, is("UP")); } + } @Nested @@ -95,8 +118,6 @@ class GivenEverythingIsHealthy { @Test void whenHealthRequested_onceLogMessageAboutStartup() { - - DiscoveryClient discoveryClient = mock(DiscoveryClient.class); when(discoveryClient.getInstances(CoreService.API_CATALOG.getServiceId())).thenReturn( Collections.singletonList(getDefaultServiceInstance(CoreService.API_CATALOG.getServiceId(), "host", 10014))); when(discoveryClient.getInstances(CoreService.DISCOVERY.getServiceId())).thenReturn( @@ -104,7 +125,6 @@ void whenHealthRequested_onceLogMessageAboutStartup() { when(discoveryClient.getInstances(CoreService.ZAAS.getServiceId())).thenReturn( Collections.singletonList(getDefaultServiceInstance(CoreService.ZAAS.getServiceId(), "host", 10023))); - GatewayHealthIndicator healthIndicator = new GatewayHealthIndicator(discoveryClient, CoreService.API_CATALOG.getServiceId()); Health.Builder builder = new Health.Builder(); healthIndicator.onApplicationEvent(mock(ApplicationReadyEvent.class)); @@ -115,4 +135,31 @@ void whenHealthRequested_onceLogMessageAboutStartup() { } + @Nested + class WhenHAIsNotComplete { + + @Mock + private ApimlLogger apimlLogger; + + @BeforeEach + void setUp() { + ReflectionTestUtils.setField(healthIndicator, "apimlLog", apimlLogger); + } + + @Test + void whenHealthRequested_skipLog() { + when(discoveryClient.getInstances(CoreService.GATEWAY.getServiceId())).thenReturn(List.of(mock(ServiceInstance.class), mock(ServiceInstance.class))); + when(discoveryClient.getInstances(CoreService.DISCOVERY.getServiceId())).thenReturn(List.of(mock(ServiceInstance.class))); + when(discoveryClient.getInstances(CoreService.ZAAS.getServiceId())).thenReturn(List.of(mock(ServiceInstance.class))); + when(discoveryClient.getInstances(CoreService.API_CATALOG.getServiceId())).thenReturn(List.of(mock(ServiceInstance.class))); + healthIndicator.onApplicationEvent(mock(ApplicationReadyEvent.class)); + + var builder = new Health.Builder(); + healthIndicator.doHealthCheck(builder); + + // verifyNoInteractions(apimlLogger); + } + + } + }