From 31fc54b451ab2d17bd2b4e2e19c2325219633dde Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Tue, 17 Feb 2026 09:16:47 +0530 Subject: [PATCH 1/8] feat(health,version): add health and version endpoints without auth --- .../controller/health/HealthController.java | 90 ++++++ .../controller/version/VersionController.java | 73 ++--- .../bengen/service/health/HealthService.java | 274 ++++++++++++++++++ .../utils/JwtUserIdValidationFilter.java | 3 +- 4 files changed, 405 insertions(+), 35 deletions(-) create mode 100644 src/main/java/com/iemr/common/bengen/controller/health/HealthController.java create mode 100644 src/main/java/com/iemr/common/bengen/service/health/HealthService.java diff --git a/src/main/java/com/iemr/common/bengen/controller/health/HealthController.java b/src/main/java/com/iemr/common/bengen/controller/health/HealthController.java new file mode 100644 index 0000000..fd423a8 --- /dev/null +++ b/src/main/java/com/iemr/common/bengen/controller/health/HealthController.java @@ -0,0 +1,90 @@ +package com.iemr.common.bengen.controller.health; + +import java.time.Instant; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import jakarta.servlet.http.HttpServletRequest; +import com.iemr.common.bengen.service.health.HealthService; +import com.iemr.common.bengen.utils.JwtAuthenticationUtil; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; + + +@RestController +@RequestMapping("/health") +@Tag(name = "Health Check", description = "APIs for checking infrastructure health status") +public class HealthController { + + private static final Logger logger = LoggerFactory.getLogger(HealthController.class); + + private final HealthService healthService; + private final JwtAuthenticationUtil jwtAuthenticationUtil; + + public HealthController(HealthService healthService, JwtAuthenticationUtil jwtAuthenticationUtil) { + this.healthService = healthService; + this.jwtAuthenticationUtil = jwtAuthenticationUtil; + } + @GetMapping + @Operation(summary = "Check infrastructure health", + description = "Returns the health status of MySQL, Redis, and other configured services") + @ApiResponses({ + @ApiResponse(responseCode = "200", description = "All checked components are UP"), + @ApiResponse(responseCode = "503", description = "One or more critical services are DOWN") + }) + public ResponseEntity> checkHealth(HttpServletRequest request) { + logger.info("Health check endpoint called"); + + try { + // Check if user is authenticated by verifying Authorization header + boolean isAuthenticated = isUserAuthenticated(request); + Map healthStatus = healthService.checkHealth(isAuthenticated); + String overallStatus = (String) healthStatus.get("status"); + + // Return 200 if overall status is UP, 503 if DOWN + HttpStatus httpStatus = "UP".equals(overallStatus) ? HttpStatus.OK : HttpStatus.SERVICE_UNAVAILABLE; + + logger.debug("Health check completed with status: {}", overallStatus); + return new ResponseEntity<>(healthStatus, httpStatus); + + } catch (Exception e) { + logger.error("Unexpected error during health check", e); + + // Return sanitized error response + Map errorResponse = Map.of( + "status", "DOWN", + "error", "Health check service unavailable", + "timestamp", Instant.now().toString() + ); + + return new ResponseEntity<>(errorResponse, HttpStatus.SERVICE_UNAVAILABLE); + } + } + + private boolean isUserAuthenticated(HttpServletRequest request) { + String authHeader = request.getHeader("Authorization"); + if (authHeader == null || authHeader.trim().isEmpty()) { + return false; + } + + // Extract token from "Bearer " format + String token = authHeader.startsWith("Bearer ") + ? authHeader.substring(7) + : authHeader; + + try { + // Validate JWT token - returns true if valid, throws exception if invalid + return jwtAuthenticationUtil.validateUserIdAndJwtToken(token); + } catch (Exception e) { + logger.debug("JWT token validation failed: {}", e.getMessage()); + return false; + } + } +} diff --git a/src/main/java/com/iemr/common/bengen/controller/version/VersionController.java b/src/main/java/com/iemr/common/bengen/controller/version/VersionController.java index 83a5a45..53cb6fc 100644 --- a/src/main/java/com/iemr/common/bengen/controller/version/VersionController.java +++ b/src/main/java/com/iemr/common/bengen/controller/version/VersionController.java @@ -1,8 +1,8 @@ /* -* AMRIT - Accessible Medical Records via Integrated Technologies -* Integrated EHR (Electronic Health Records) Solution +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution * -* Copyright (C) "Piramal Swasthya Management and Research Institute" +* Copyright (C) "Piramal Swasthya Management and Research Institute" * * This file is part of AMRIT. * @@ -21,54 +21,59 @@ */ package com.iemr.common.bengen.controller.version; -import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; -import com.iemr.common.bengen.utils.response.OutputResponse; - import io.swagger.v3.oas.annotations.Operation; @RestController public class VersionController { - private Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + private final Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); - @Operation(summary = "Version Information") - @GetMapping(value = "/version") - public String versionInformation() { - OutputResponse output = new OutputResponse(); + private static final String UNKNOWN_VALUE = "unknown"; + + @Operation(summary = "Get version information") + @GetMapping(value = "/version", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity> versionInformation() { + Map response = new LinkedHashMap<>(); try { logger.info("version Controller Start"); - output.setResponse(readGitProperties()); - } catch (Exception e) { - output.setError(e); - } - + Properties gitProperties = loadGitProperties(); + response.put("buildTimestamp", gitProperties.getProperty("git.build.time", UNKNOWN_VALUE)); + response.put("version", gitProperties.getProperty("git.build.version", UNKNOWN_VALUE)); + response.put("branch", gitProperties.getProperty("git.branch", UNKNOWN_VALUE)); + response.put("commitHash", gitProperties.getProperty("git.commit.id.abbrev", UNKNOWN_VALUE)); + } catch (Exception e) { + logger.error("Failed to load version information", e); + response.put("buildTimestamp", UNKNOWN_VALUE); + response.put("version", UNKNOWN_VALUE); + response.put("branch", UNKNOWN_VALUE); + response.put("commitHash", UNKNOWN_VALUE); + } logger.info("version Controller End"); - return output.toString(); + return ResponseEntity.ok(response); } - private String readGitProperties() throws Exception { - ClassLoader classLoader = getClass().getClassLoader(); - InputStream inputStream = classLoader.getResourceAsStream("git.properties"); - - return readFromInputStream(inputStream); - } - private String readFromInputStream(InputStream inputStream) - throws IOException { - StringBuilder resultStringBuilder = new StringBuilder(); - try (BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) { - String line; - while ((line = br.readLine()) != null) { - resultStringBuilder.append(line).append("\n"); - } - } - return resultStringBuilder.toString(); + + private Properties loadGitProperties() throws IOException { + Properties properties = new Properties(); + try (InputStream input = getClass().getClassLoader() + .getResourceAsStream("git.properties")) { + if (input != null) { + properties.load(input); + } + } + return properties; } } diff --git a/src/main/java/com/iemr/common/bengen/service/health/HealthService.java b/src/main/java/com/iemr/common/bengen/service/health/HealthService.java new file mode 100644 index 0000000..a45efca --- /dev/null +++ b/src/main/java/com/iemr/common/bengen/service/health/HealthService.java @@ -0,0 +1,274 @@ +package com.iemr.common.bengen.service.health; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; +import java.util.function.Supplier; +import javax.sql.DataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.redis.core.RedisCallback; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.stereotype.Service; + +@Service +public class HealthService { + + private static final Logger logger = LoggerFactory.getLogger(HealthService.class); + private static final String STATUS_KEY = "status"; + private static final String DB_HEALTH_CHECK_QUERY = "SELECT 1 as health_check"; + private static final String DB_VERSION_QUERY = "SELECT VERSION()"; + private static final String STATUS_UP = "UP"; + private static final String STATUS_DOWN = "DOWN"; + private static final String UNKNOWN_VALUE = "unknown"; + + private final DataSource dataSource; + private final RedisTemplate redisTemplate; + private final String dbUrl; + private final String redisHost; + private final int redisPort; + + public HealthService(DataSource dataSource, + @Autowired(required = false) RedisTemplate redisTemplate, + @Value("${spring.datasource.url:unknown}") String dbUrl, + @Value("${spring.redis.host:localhost}") String redisHost, + @Value("${spring.redis.port:6379}") int redisPort) { + this.dataSource = dataSource; + this.redisTemplate = redisTemplate; + this.dbUrl = dbUrl; + this.redisHost = redisHost; + this.redisPort = redisPort; + } + + public Map checkHealth(boolean includeDetails) { + Map healthStatus = new LinkedHashMap<>(); + Map components = new LinkedHashMap<>(); + boolean overallHealth = true; + + Map mysqlStatus = checkMySQLHealth(includeDetails); + components.put("mysql", mysqlStatus); + if (!isHealthy(mysqlStatus)) { + overallHealth = false; + } + + if (redisTemplate != null) { + Map redisStatus = checkRedisHealth(includeDetails); + components.put("redis", redisStatus); + if (!isHealthy(redisStatus)) { + overallHealth = false; + } + } + + healthStatus.put(STATUS_KEY, overallHealth ? STATUS_UP : STATUS_DOWN); + healthStatus.put("timestamp", Instant.now().toString()); + healthStatus.put("components", components); + logger.info("Health check completed - Overall status: {}", overallHealth ? STATUS_UP : STATUS_DOWN); + + return healthStatus; + } + + public Map checkHealth() { + return checkHealth(true); + } + + private Map checkMySQLHealth(boolean includeDetails) { + Map details = new LinkedHashMap<>(); + details.put("type", "MySQL"); + + if (includeDetails) { + details.put("host", extractHost(dbUrl)); + details.put("port", extractPort(dbUrl)); + details.put("database", extractDatabaseName(dbUrl)); + } + + return performHealthCheck("MySQL", details, () -> { + try { + try (Connection connection = dataSource.getConnection()) { + if (connection.isValid(2)) { + try (PreparedStatement stmt = connection.prepareStatement(DB_HEALTH_CHECK_QUERY)) { + stmt.setQueryTimeout(3); + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next() && rs.getInt(1) == 1) { + String version = includeDetails ? getMySQLVersion(connection) : null; + return new HealthCheckResult(true, version, null); + } + } + } + } + return new HealthCheckResult(false, null, "Connection validation failed"); + } + } catch (Exception e) { + throw new IllegalStateException("Failed to perform MySQL health check", e); + } + }); + } + + private Map checkRedisHealth(boolean includeDetails) { + Map details = new LinkedHashMap<>(); + details.put("type", "Redis"); + + if (includeDetails) { + details.put("host", redisHost); + details.put("port", redisPort); + } + + return performHealthCheck("Redis", details, () -> { + String pong = redisTemplate.execute((RedisCallback) connection -> + connection.ping() + ); + if ("PONG".equals(pong)) { + String version = includeDetails ? getRedisVersion() : null; + return new HealthCheckResult(true, version, null); + } + return new HealthCheckResult(false, null, "Ping returned unexpected response"); + }); + } + + private Map performHealthCheck(String componentName, + Map details, + Supplier checker) { + Map status = new LinkedHashMap<>(); + long startTime = System.currentTimeMillis(); + + try { + HealthCheckResult result = checker.get(); + long responseTime = System.currentTimeMillis() - startTime; + + details.put("responseTimeMs", responseTime); + + if (result.isHealthy) { + logger.debug("{} health check: UP ({}ms)", componentName, responseTime); + status.put(STATUS_KEY, STATUS_UP); + if (result.version != null) { + details.put("version", result.version); + } + } else { + String safeError = result.error != null ? result.error : "Health check failed"; + logger.warn("{} health check failed: {}", componentName, safeError); + status.put(STATUS_KEY, STATUS_DOWN); + details.put("error", safeError); + details.put("errorType", "CheckFailed"); + } + + status.put("details", details); + return status; + + } catch (Exception e) { + long responseTime = System.currentTimeMillis() - startTime; + + logger.error("{} health check failed", componentName, e); + + status.put(STATUS_KEY, STATUS_DOWN); + details.put("responseTimeMs", responseTime); + details.put("error", "Health check failed"); + details.put("errorType", "InternalError"); + status.put("details", details); + + return status; + } + } + + private boolean isHealthy(Map componentStatus) { + return STATUS_UP.equals(componentStatus.get(STATUS_KEY)); + } + + private String getMySQLVersion(Connection connection) { + try (PreparedStatement stmt = connection.prepareStatement(DB_VERSION_QUERY); + ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + return rs.getString(1); + } + } catch (Exception e) { + logger.debug("Could not retrieve MySQL version", e); + } + return null; + } + + private String getRedisVersion() { + try { + Properties info = redisTemplate.execute((RedisCallback) connection -> + connection.serverCommands().info("server") + ); + if (info != null && info.containsKey("redis_version")) { + return info.getProperty("redis_version"); + } + } catch (Exception e) { + logger.debug("Could not retrieve Redis version", e); + } + return null; + } + + private String extractHost(String jdbcUrl) { + if (jdbcUrl == null || UNKNOWN_VALUE.equals(jdbcUrl)) { + return UNKNOWN_VALUE; + } + try { + String withoutPrefix = jdbcUrl.replaceFirst("jdbc:mysql://", ""); + int slashIndex = withoutPrefix.indexOf('/'); + String hostPort = slashIndex > 0 + ? withoutPrefix.substring(0, slashIndex) + : withoutPrefix; + int colonIndex = hostPort.indexOf(':'); + return colonIndex > 0 ? hostPort.substring(0, colonIndex) : hostPort; + } catch (Exception e) { + logger.debug("Could not extract host from URL", e); + } + return UNKNOWN_VALUE; + } + + private String extractPort(String jdbcUrl) { + if (jdbcUrl == null || UNKNOWN_VALUE.equals(jdbcUrl)) { + return UNKNOWN_VALUE; + } + try { + String withoutPrefix = jdbcUrl.replaceFirst("jdbc:mysql://", ""); + int slashIndex = withoutPrefix.indexOf('/'); + String hostPort = slashIndex > 0 + ? withoutPrefix.substring(0, slashIndex) + : withoutPrefix; + int colonIndex = hostPort.indexOf(':'); + return colonIndex > 0 ? hostPort.substring(colonIndex + 1) : "3306"; + } catch (Exception e) { + logger.debug("Could not extract port from URL", e); + } + return "3306"; + } + + private String extractDatabaseName(String jdbcUrl) { + if (jdbcUrl == null || UNKNOWN_VALUE.equals(jdbcUrl)) { + return UNKNOWN_VALUE; + } + try { + int lastSlash = jdbcUrl.lastIndexOf('/'); + if (lastSlash >= 0 && lastSlash < jdbcUrl.length() - 1) { + String afterSlash = jdbcUrl.substring(lastSlash + 1); + int queryStart = afterSlash.indexOf('?'); + if (queryStart > 0) { + return afterSlash.substring(0, queryStart); + } + return afterSlash; + } + } catch (Exception e) { + logger.debug("Could not extract database name from URL", e); + } + return UNKNOWN_VALUE; + } + + private static class HealthCheckResult { + final boolean isHealthy; + final String version; + final String error; + + HealthCheckResult(boolean isHealthy, String version, String error) { + this.isHealthy = isHealthy; + this.version = version; + this.error = error; + } + } +} diff --git a/src/main/java/com/iemr/common/bengen/utils/JwtUserIdValidationFilter.java b/src/main/java/com/iemr/common/bengen/utils/JwtUserIdValidationFilter.java index b6b57c0..c43bcb4 100644 --- a/src/main/java/com/iemr/common/bengen/utils/JwtUserIdValidationFilter.java +++ b/src/main/java/com/iemr/common/bengen/utils/JwtUserIdValidationFilter.java @@ -110,7 +110,8 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo // Skip login and public endpoints if (path.equals(contextPath + "/user/userAuthenticate") || path.equalsIgnoreCase(contextPath + "/user/logOutUserFromConcurrentSession") - || path.startsWith(contextPath + "/public")) { + || path.startsWith(contextPath + "/public") + || path.equals(contextPath + "/health") || path.equals(contextPath + "/version")) { logger.info("Skipping filter for path: " + path); filterChain.doFilter(servletRequest, servletResponse); return; From 6807bd47c5960d0ac4ebde9c81dff3e133098315 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Tue, 17 Feb 2026 09:34:41 +0530 Subject: [PATCH 2/8] fix(redis): fix redis timeout handling issue --- .../bengen/service/health/HealthService.java | 64 ++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/iemr/common/bengen/service/health/HealthService.java b/src/main/java/com/iemr/common/bengen/service/health/HealthService.java index a45efca..34867e7 100644 --- a/src/main/java/com/iemr/common/bengen/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/bengen/service/health/HealthService.java @@ -3,10 +3,16 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.SQLException; import java.time.Instant; import java.util.LinkedHashMap; import java.util.Map; import java.util.Properties; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Supplier; import javax.sql.DataSource; import org.slf4j.Logger; @@ -16,6 +22,7 @@ import org.springframework.data.redis.core.RedisCallback; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; +import com.zaxxer.hikari.HikariDataSource; @Service public class HealthService { @@ -27,6 +34,12 @@ public class HealthService { private static final String STATUS_UP = "UP"; private static final String STATUS_DOWN = "DOWN"; private static final String UNKNOWN_VALUE = "unknown"; + private static final int CONNECTION_ACQUISITION_TIMEOUT_SECONDS = 2; + private static final ExecutorService executorService = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "HealthCheckConnectionPool"); + t.setDaemon(true); + return t; + }); private final DataSource dataSource; private final RedisTemplate redisTemplate; @@ -89,7 +102,9 @@ private Map checkMySQLHealth(boolean includeDetails) { return performHealthCheck("MySQL", details, () -> { try { - try (Connection connection = dataSource.getConnection()) { + + Connection connection = getConnectionWithTimeout(); + try { if (connection.isValid(2)) { try (PreparedStatement stmt = connection.prepareStatement(DB_HEALTH_CHECK_QUERY)) { stmt.setQueryTimeout(3); @@ -102,12 +117,59 @@ private Map checkMySQLHealth(boolean includeDetails) { } } return new HealthCheckResult(false, null, "Connection validation failed"); + } finally { + if (connection != null) { + connection.close(); + } } + } catch (TimeoutException e) { + logger.warn("MySQL health check timed out - connection pool may be exhausted", e); + throw new IllegalStateException( + "Failed to acquire database connection within " + + CONNECTION_ACQUISITION_TIMEOUT_SECONDS + " seconds", e); } catch (Exception e) { + logger.error("MySQL health check failed with exception", e); throw new IllegalStateException("Failed to perform MySQL health check", e); } }); } + + + private Connection getConnectionWithTimeout() throws TimeoutException { + // If datasource is HikariDataSource, verify its connectionTimeout is set appropriately + if (dataSource instanceof HikariDataSource) { + HikariDataSource hikariDs = (HikariDataSource) dataSource; + long connectionTimeout = hikariDs.getConnectionTimeout(); + if (connectionTimeout > 15000) { + logger.warn("HikariDataSource connectionTimeout is {}ms; consider reducing for health checks", + connectionTimeout); + } + // HikariDataSource already has built-in timeout; just acquire normally + try { + return hikariDs.getConnection(); + } catch (SQLException e) { + throw new TimeoutException("Failed to acquire connection from HikariDataSource: " + e.getMessage()); + } + } + + // Fallback for non-Hikari datasources: wrap in CompletableFuture with timeout + try { + return CompletableFuture.supplyAsync(() -> { + try { + return dataSource.getConnection(); + } catch (SQLException e) { + throw new IllegalStateException("Error acquiring connection: " + e.getMessage(), e); + } + }, executorService).get(CONNECTION_ACQUISITION_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } catch (TimeoutException e) { + throw e; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new TimeoutException("Connection acquisition was interrupted: " + e.getMessage()); + } catch (Exception e) { + throw new TimeoutException("Failed to acquire connection within timeout: " + e.getMessage()); + } + } private Map checkRedisHealth(boolean includeDetails) { Map details = new LinkedHashMap<>(); From f57a388c8c1b73a7d5c4cc97e502f2d8d4bd2b30 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Tue, 17 Feb 2026 09:41:09 +0530 Subject: [PATCH 3/8] fix(health): fix the exception logging and timeout issues --- .../bengen/service/health/HealthService.java | 64 +------------------ 1 file changed, 1 insertion(+), 63 deletions(-) diff --git a/src/main/java/com/iemr/common/bengen/service/health/HealthService.java b/src/main/java/com/iemr/common/bengen/service/health/HealthService.java index 34867e7..a45efca 100644 --- a/src/main/java/com/iemr/common/bengen/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/bengen/service/health/HealthService.java @@ -3,16 +3,10 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.sql.SQLException; import java.time.Instant; import java.util.LinkedHashMap; import java.util.Map; import java.util.Properties; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.function.Supplier; import javax.sql.DataSource; import org.slf4j.Logger; @@ -22,7 +16,6 @@ import org.springframework.data.redis.core.RedisCallback; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; -import com.zaxxer.hikari.HikariDataSource; @Service public class HealthService { @@ -34,12 +27,6 @@ public class HealthService { private static final String STATUS_UP = "UP"; private static final String STATUS_DOWN = "DOWN"; private static final String UNKNOWN_VALUE = "unknown"; - private static final int CONNECTION_ACQUISITION_TIMEOUT_SECONDS = 2; - private static final ExecutorService executorService = Executors.newSingleThreadExecutor(r -> { - Thread t = new Thread(r, "HealthCheckConnectionPool"); - t.setDaemon(true); - return t; - }); private final DataSource dataSource; private final RedisTemplate redisTemplate; @@ -102,9 +89,7 @@ private Map checkMySQLHealth(boolean includeDetails) { return performHealthCheck("MySQL", details, () -> { try { - - Connection connection = getConnectionWithTimeout(); - try { + try (Connection connection = dataSource.getConnection()) { if (connection.isValid(2)) { try (PreparedStatement stmt = connection.prepareStatement(DB_HEALTH_CHECK_QUERY)) { stmt.setQueryTimeout(3); @@ -117,59 +102,12 @@ private Map checkMySQLHealth(boolean includeDetails) { } } return new HealthCheckResult(false, null, "Connection validation failed"); - } finally { - if (connection != null) { - connection.close(); - } } - } catch (TimeoutException e) { - logger.warn("MySQL health check timed out - connection pool may be exhausted", e); - throw new IllegalStateException( - "Failed to acquire database connection within " + - CONNECTION_ACQUISITION_TIMEOUT_SECONDS + " seconds", e); } catch (Exception e) { - logger.error("MySQL health check failed with exception", e); throw new IllegalStateException("Failed to perform MySQL health check", e); } }); } - - - private Connection getConnectionWithTimeout() throws TimeoutException { - // If datasource is HikariDataSource, verify its connectionTimeout is set appropriately - if (dataSource instanceof HikariDataSource) { - HikariDataSource hikariDs = (HikariDataSource) dataSource; - long connectionTimeout = hikariDs.getConnectionTimeout(); - if (connectionTimeout > 15000) { - logger.warn("HikariDataSource connectionTimeout is {}ms; consider reducing for health checks", - connectionTimeout); - } - // HikariDataSource already has built-in timeout; just acquire normally - try { - return hikariDs.getConnection(); - } catch (SQLException e) { - throw new TimeoutException("Failed to acquire connection from HikariDataSource: " + e.getMessage()); - } - } - - // Fallback for non-Hikari datasources: wrap in CompletableFuture with timeout - try { - return CompletableFuture.supplyAsync(() -> { - try { - return dataSource.getConnection(); - } catch (SQLException e) { - throw new IllegalStateException("Error acquiring connection: " + e.getMessage(), e); - } - }, executorService).get(CONNECTION_ACQUISITION_TIMEOUT_SECONDS, TimeUnit.SECONDS); - } catch (TimeoutException e) { - throw e; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new TimeoutException("Connection acquisition was interrupted: " + e.getMessage()); - } catch (Exception e) { - throw new TimeoutException("Failed to acquire connection within timeout: " + e.getMessage()); - } - } private Map checkRedisHealth(boolean includeDetails) { Map details = new LinkedHashMap<>(); From 75a21d87cd560b2b7628797cafec55c047f8ab31 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Thu, 19 Feb 2026 11:39:02 +0530 Subject: [PATCH 4/8] refactor(health): simplify MySQL health check and remove sensitive details --- pom.xml | 26 +++ .../controller/health/HealthController.java | 37 +--- .../bengen/service/health/HealthService.java | 203 ++++-------------- 3 files changed, 76 insertions(+), 190 deletions(-) diff --git a/pom.xml b/pom.xml index 9ebc49c..9f92755 100644 --- a/pom.xml +++ b/pom.xml @@ -440,6 +440,32 @@ maven-jar-plugin 3.0.2 + + io.github.git-commit-id + git-commit-id-maven-plugin + 9.0.2 + + + get-the-git-infos + + revision + + initialize + + + + true + ${project.build.outputDirectory}/git.properties + + ^git.branch$ + ^git.commit.id.abbrev$ + ^git.build.version$ + ^git.build.time$ + + false + false + + org.apache.maven.plugins maven-resources-plugin diff --git a/src/main/java/com/iemr/common/bengen/controller/health/HealthController.java b/src/main/java/com/iemr/common/bengen/controller/health/HealthController.java index fd423a8..fb10a3c 100644 --- a/src/main/java/com/iemr/common/bengen/controller/health/HealthController.java +++ b/src/main/java/com/iemr/common/bengen/controller/health/HealthController.java @@ -9,15 +9,12 @@ import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import jakarta.servlet.http.HttpServletRequest; import com.iemr.common.bengen.service.health.HealthService; -import com.iemr.common.bengen.utils.JwtAuthenticationUtil; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.tags.Tag; - @RestController @RequestMapping("/health") @Tag(name = "Health Check", description = "APIs for checking infrastructure health status") @@ -26,12 +23,11 @@ public class HealthController { private static final Logger logger = LoggerFactory.getLogger(HealthController.class); private final HealthService healthService; - private final JwtAuthenticationUtil jwtAuthenticationUtil; - public HealthController(HealthService healthService, JwtAuthenticationUtil jwtAuthenticationUtil) { + public HealthController(HealthService healthService) { this.healthService = healthService; - this.jwtAuthenticationUtil = jwtAuthenticationUtil; } + @GetMapping @Operation(summary = "Check infrastructure health", description = "Returns the health status of MySQL, Redis, and other configured services") @@ -39,16 +35,13 @@ public HealthController(HealthService healthService, JwtAuthenticationUtil jwtAu @ApiResponse(responseCode = "200", description = "All checked components are UP"), @ApiResponse(responseCode = "503", description = "One or more critical services are DOWN") }) - public ResponseEntity> checkHealth(HttpServletRequest request) { + public ResponseEntity> checkHealth() { logger.info("Health check endpoint called"); try { - // Check if user is authenticated by verifying Authorization header - boolean isAuthenticated = isUserAuthenticated(request); - Map healthStatus = healthService.checkHealth(isAuthenticated); + Map healthStatus = healthService.checkHealth(); String overallStatus = (String) healthStatus.get("status"); - // Return 200 if overall status is UP, 503 if DOWN HttpStatus httpStatus = "UP".equals(overallStatus) ? HttpStatus.OK : HttpStatus.SERVICE_UNAVAILABLE; logger.debug("Health check completed with status: {}", overallStatus); @@ -57,34 +50,12 @@ public ResponseEntity> checkHealth(HttpServletRequest reques } catch (Exception e) { logger.error("Unexpected error during health check", e); - // Return sanitized error response Map errorResponse = Map.of( "status", "DOWN", - "error", "Health check service unavailable", "timestamp", Instant.now().toString() ); return new ResponseEntity<>(errorResponse, HttpStatus.SERVICE_UNAVAILABLE); } } - - private boolean isUserAuthenticated(HttpServletRequest request) { - String authHeader = request.getHeader("Authorization"); - if (authHeader == null || authHeader.trim().isEmpty()) { - return false; - } - - // Extract token from "Bearer " format - String token = authHeader.startsWith("Bearer ") - ? authHeader.substring(7) - : authHeader; - - try { - // Validate JWT token - returns true if valid, throws exception if invalid - return jwtAuthenticationUtil.validateUserIdAndJwtToken(token); - } catch (Exception e) { - logger.debug("JWT token validation failed: {}", e.getMessage()); - return false; - } - } } diff --git a/src/main/java/com/iemr/common/bengen/service/health/HealthService.java b/src/main/java/com/iemr/common/bengen/service/health/HealthService.java index a45efca..78db4af 100644 --- a/src/main/java/com/iemr/common/bengen/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/bengen/service/health/HealthService.java @@ -6,13 +6,16 @@ import java.time.Instant; import java.util.LinkedHashMap; import java.util.Map; -import java.util.Properties; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Supplier; import javax.sql.DataSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.data.redis.core.RedisCallback; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; @@ -23,42 +26,33 @@ public class HealthService { private static final Logger logger = LoggerFactory.getLogger(HealthService.class); private static final String STATUS_KEY = "status"; private static final String DB_HEALTH_CHECK_QUERY = "SELECT 1 as health_check"; - private static final String DB_VERSION_QUERY = "SELECT VERSION()"; private static final String STATUS_UP = "UP"; private static final String STATUS_DOWN = "DOWN"; - private static final String UNKNOWN_VALUE = "unknown"; + private static final int REDIS_TIMEOUT_SECONDS = 3; private final DataSource dataSource; + private final ExecutorService executorService = Executors.newFixedThreadPool(2); private final RedisTemplate redisTemplate; - private final String dbUrl; - private final String redisHost; - private final int redisPort; public HealthService(DataSource dataSource, - @Autowired(required = false) RedisTemplate redisTemplate, - @Value("${spring.datasource.url:unknown}") String dbUrl, - @Value("${spring.redis.host:localhost}") String redisHost, - @Value("${spring.redis.port:6379}") int redisPort) { + @Autowired(required = false) RedisTemplate redisTemplate) { this.dataSource = dataSource; this.redisTemplate = redisTemplate; - this.dbUrl = dbUrl; - this.redisHost = redisHost; - this.redisPort = redisPort; } - public Map checkHealth(boolean includeDetails) { + public Map checkHealth() { Map healthStatus = new LinkedHashMap<>(); Map components = new LinkedHashMap<>(); boolean overallHealth = true; - Map mysqlStatus = checkMySQLHealth(includeDetails); + Map mysqlStatus = checkMySQLHealth(); components.put("mysql", mysqlStatus); if (!isHealthy(mysqlStatus)) { overallHealth = false; } if (redisTemplate != null) { - Map redisStatus = checkRedisHealth(includeDetails); + Map redisStatus = checkRedisHealth(); components.put("redis", redisStatus); if (!isHealthy(redisStatus)) { overallHealth = false; @@ -73,102 +67,81 @@ public Map checkHealth(boolean includeDetails) { return healthStatus; } - public Map checkHealth() { - return checkHealth(true); - } - - private Map checkMySQLHealth(boolean includeDetails) { - Map details = new LinkedHashMap<>(); - details.put("type", "MySQL"); + private Map checkMySQLHealth() { + Map status = new LinkedHashMap<>(); - if (includeDetails) { - details.put("host", extractHost(dbUrl)); - details.put("port", extractPort(dbUrl)); - details.put("database", extractDatabaseName(dbUrl)); - } - - return performHealthCheck("MySQL", details, () -> { + return performHealthCheck("MySQL", status, () -> { try { try (Connection connection = dataSource.getConnection()) { - if (connection.isValid(2)) { - try (PreparedStatement stmt = connection.prepareStatement(DB_HEALTH_CHECK_QUERY)) { - stmt.setQueryTimeout(3); - try (ResultSet rs = stmt.executeQuery()) { - if (rs.next() && rs.getInt(1) == 1) { - String version = includeDetails ? getMySQLVersion(connection) : null; - return new HealthCheckResult(true, version, null); - } + try (PreparedStatement stmt = connection.prepareStatement(DB_HEALTH_CHECK_QUERY)) { + stmt.setQueryTimeout(3); + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next() && rs.getInt(1) == 1) { + return new HealthCheckResult(true, null); } } } - return new HealthCheckResult(false, null, "Connection validation failed"); + return new HealthCheckResult(false, "Query validation failed"); } } catch (Exception e) { - throw new IllegalStateException("Failed to perform MySQL health check", e); + throw new IllegalStateException("MySQL health check failed: " + e.getMessage(), e); } }); } - private Map checkRedisHealth(boolean includeDetails) { - Map details = new LinkedHashMap<>(); - details.put("type", "Redis"); - - if (includeDetails) { - details.put("host", redisHost); - details.put("port", redisPort); - } + private Map checkRedisHealth() { + Map status = new LinkedHashMap<>(); - return performHealthCheck("Redis", details, () -> { - String pong = redisTemplate.execute((RedisCallback) connection -> - connection.ping() - ); - if ("PONG".equals(pong)) { - String version = includeDetails ? getRedisVersion() : null; - return new HealthCheckResult(true, version, null); + return performHealthCheck("Redis", status, () -> { + try { + String pong = CompletableFuture.supplyAsync(() -> + redisTemplate.execute((RedisCallback) connection -> connection.ping()), + executorService + ).get(REDIS_TIMEOUT_SECONDS, TimeUnit.SECONDS); + + if ("PONG".equals(pong)) { + return new HealthCheckResult(true, null); + } + return new HealthCheckResult(false, "Ping returned unexpected response"); + } catch (TimeoutException e) { + return new HealthCheckResult(false, "Redis ping timed out after " + REDIS_TIMEOUT_SECONDS + " seconds"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return new HealthCheckResult(false, "Redis health check was interrupted"); + } catch (Exception e) { + throw new IllegalStateException("Redis health check failed", e); } - return new HealthCheckResult(false, null, "Ping returned unexpected response"); }); } private Map performHealthCheck(String componentName, - Map details, + Map status, Supplier checker) { - Map status = new LinkedHashMap<>(); long startTime = System.currentTimeMillis(); try { HealthCheckResult result = checker.get(); long responseTime = System.currentTimeMillis() - startTime; - details.put("responseTimeMs", responseTime); + status.put("responseTimeMs", responseTime); if (result.isHealthy) { logger.debug("{} health check: UP ({}ms)", componentName, responseTime); status.put(STATUS_KEY, STATUS_UP); - if (result.version != null) { - details.put("version", result.version); - } } else { String safeError = result.error != null ? result.error : "Health check failed"; logger.warn("{} health check failed: {}", componentName, safeError); status.put(STATUS_KEY, STATUS_DOWN); - details.put("error", safeError); - details.put("errorType", "CheckFailed"); } - status.put("details", details); return status; } catch (Exception e) { long responseTime = System.currentTimeMillis() - startTime; - - logger.error("{} health check failed", componentName, e); + logger.error("{} health check failed with exception: {}", componentName, e.getMessage(), e); status.put(STATUS_KEY, STATUS_DOWN); - details.put("responseTimeMs", responseTime); - details.put("error", "Health check failed"); - details.put("errorType", "InternalError"); - status.put("details", details); + status.put("responseTimeMs", responseTime); return status; } @@ -178,96 +151,12 @@ private boolean isHealthy(Map componentStatus) { return STATUS_UP.equals(componentStatus.get(STATUS_KEY)); } - private String getMySQLVersion(Connection connection) { - try (PreparedStatement stmt = connection.prepareStatement(DB_VERSION_QUERY); - ResultSet rs = stmt.executeQuery()) { - if (rs.next()) { - return rs.getString(1); - } - } catch (Exception e) { - logger.debug("Could not retrieve MySQL version", e); - } - return null; - } - - private String getRedisVersion() { - try { - Properties info = redisTemplate.execute((RedisCallback) connection -> - connection.serverCommands().info("server") - ); - if (info != null && info.containsKey("redis_version")) { - return info.getProperty("redis_version"); - } - } catch (Exception e) { - logger.debug("Could not retrieve Redis version", e); - } - return null; - } - - private String extractHost(String jdbcUrl) { - if (jdbcUrl == null || UNKNOWN_VALUE.equals(jdbcUrl)) { - return UNKNOWN_VALUE; - } - try { - String withoutPrefix = jdbcUrl.replaceFirst("jdbc:mysql://", ""); - int slashIndex = withoutPrefix.indexOf('/'); - String hostPort = slashIndex > 0 - ? withoutPrefix.substring(0, slashIndex) - : withoutPrefix; - int colonIndex = hostPort.indexOf(':'); - return colonIndex > 0 ? hostPort.substring(0, colonIndex) : hostPort; - } catch (Exception e) { - logger.debug("Could not extract host from URL", e); - } - return UNKNOWN_VALUE; - } - - private String extractPort(String jdbcUrl) { - if (jdbcUrl == null || UNKNOWN_VALUE.equals(jdbcUrl)) { - return UNKNOWN_VALUE; - } - try { - String withoutPrefix = jdbcUrl.replaceFirst("jdbc:mysql://", ""); - int slashIndex = withoutPrefix.indexOf('/'); - String hostPort = slashIndex > 0 - ? withoutPrefix.substring(0, slashIndex) - : withoutPrefix; - int colonIndex = hostPort.indexOf(':'); - return colonIndex > 0 ? hostPort.substring(colonIndex + 1) : "3306"; - } catch (Exception e) { - logger.debug("Could not extract port from URL", e); - } - return "3306"; - } - - private String extractDatabaseName(String jdbcUrl) { - if (jdbcUrl == null || UNKNOWN_VALUE.equals(jdbcUrl)) { - return UNKNOWN_VALUE; - } - try { - int lastSlash = jdbcUrl.lastIndexOf('/'); - if (lastSlash >= 0 && lastSlash < jdbcUrl.length() - 1) { - String afterSlash = jdbcUrl.substring(lastSlash + 1); - int queryStart = afterSlash.indexOf('?'); - if (queryStart > 0) { - return afterSlash.substring(0, queryStart); - } - return afterSlash; - } - } catch (Exception e) { - logger.debug("Could not extract database name from URL", e); - } - return UNKNOWN_VALUE; - } - private static class HealthCheckResult { final boolean isHealthy; - final String version; final String error; - HealthCheckResult(boolean isHealthy, String version, String error) { + HealthCheckResult(boolean isHealthy, String error) { this.isHealthy = isHealthy; - this.version = version; this.error = error; } } From 721e61c08452886ee75f8aa183a30aca8e7a6f7e Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sat, 21 Feb 2026 12:47:10 +0530 Subject: [PATCH 5/8] fix(health): scope PROCESSLIST lock-wait check to application DB user --- .../controller/health/HealthController.java | 5 +- .../bengen/service/health/HealthService.java | 505 +++++++++++++++--- 2 files changed, 431 insertions(+), 79 deletions(-) diff --git a/src/main/java/com/iemr/common/bengen/controller/health/HealthController.java b/src/main/java/com/iemr/common/bengen/controller/health/HealthController.java index fb10a3c..bf6976c 100644 --- a/src/main/java/com/iemr/common/bengen/controller/health/HealthController.java +++ b/src/main/java/com/iemr/common/bengen/controller/health/HealthController.java @@ -32,7 +32,7 @@ public HealthController(HealthService healthService) { @Operation(summary = "Check infrastructure health", description = "Returns the health status of MySQL, Redis, and other configured services") @ApiResponses({ - @ApiResponse(responseCode = "200", description = "All checked components are UP"), + @ApiResponse(responseCode = "200", description = "Services are UP or DEGRADED (operational with warnings)"), @ApiResponse(responseCode = "503", description = "One or more critical services are DOWN") }) public ResponseEntity> checkHealth() { @@ -42,7 +42,8 @@ public ResponseEntity> checkHealth() { Map healthStatus = healthService.checkHealth(); String overallStatus = (String) healthStatus.get("status"); - HttpStatus httpStatus = "UP".equals(overallStatus) ? HttpStatus.OK : HttpStatus.SERVICE_UNAVAILABLE; + // Return 503 only if DOWN; 200 for both UP and DEGRADED (DEGRADED = operational with warnings) + HttpStatus httpStatus = "DOWN".equals(overallStatus) ? HttpStatus.SERVICE_UNAVAILABLE : HttpStatus.OK; logger.debug("Health check completed with status: {}", overallStatus); return new ResponseEntity<>(healthStatus, httpStatus); diff --git a/src/main/java/com/iemr/common/bengen/service/health/HealthService.java b/src/main/java/com/iemr/common/bengen/service/health/HealthService.java index 78db4af..5c1058d 100644 --- a/src/main/java/com/iemr/common/bengen/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/bengen/service/health/HealthService.java @@ -6,17 +6,25 @@ import java.time.Instant; import java.util.LinkedHashMap; import java.util.Map; -import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.function.Supplier; +import jakarta.annotation.PreDestroy; import javax.sql.DataSource; +import com.zaxxer.hikari.HikariDataSource; +import com.zaxxer.hikari.HikariPoolMXBean; +import java.lang.management.ManagementFactory; +import javax.management.MBeanServer; +import javax.management.ObjectName; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.redis.core.RedisCallback; +import org.springframework.beans.factory.annotation.Value; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; @@ -24,94 +32,169 @@ public class HealthService { private static final Logger logger = LoggerFactory.getLogger(HealthService.class); + + // Status constants private static final String STATUS_KEY = "status"; - private static final String DB_HEALTH_CHECK_QUERY = "SELECT 1 as health_check"; private static final String STATUS_UP = "UP"; private static final String STATUS_DOWN = "DOWN"; - private static final int REDIS_TIMEOUT_SECONDS = 3; + private static final String STATUS_DEGRADED = "DEGRADED"; + private static final String SEVERITY_KEY = "severity"; + private static final String SEVERITY_OK = "OK"; + private static final String SEVERITY_WARNING = "WARNING"; + private static final String SEVERITY_CRITICAL = "CRITICAL"; + private static final String ERROR_KEY = "error"; + private static final String MESSAGE_KEY = "message"; + private static final String RESPONSE_TIME_KEY = "responseTimeMs"; + private static final long MYSQL_TIMEOUT_SECONDS = 3; + private static final long REDIS_TIMEOUT_SECONDS = 3; + private static final long ADVANCED_CHECKS_THROTTLE_SECONDS = 30; + private static final long RESPONSE_TIME_THRESHOLD_MS = 2000; + private static final String DIAGNOSTIC_LOCK_WAIT = "MYSQL_LOCK_WAIT"; + private static final String DIAGNOSTIC_DEADLOCK = "MYSQL_DEADLOCK"; + private static final String DIAGNOSTIC_SLOW_QUERIES = "MYSQL_SLOW_QUERIES"; + private static final String DIAGNOSTIC_POOL_EXHAUSTED = "MYSQL_POOL_EXHAUSTED"; + private static final String DIAGNOSTIC_LOG_TEMPLATE = "Diagnostic: {}"; private final DataSource dataSource; - private final ExecutorService executorService = Executors.newFixedThreadPool(2); private final RedisTemplate redisTemplate; + private final ExecutorService executorService; + + private volatile long lastAdvancedCheckTime = 0; + private volatile AdvancedCheckResult cachedAdvancedCheckResult = null; + private final ReentrantReadWriteLock advancedCheckLock = new ReentrantReadWriteLock(); + + // Deadlock check resilience - disable after first permission error + private volatile boolean deadlockCheckDisabled = false; + + @Value("${health.advanced.enabled:true}") + private boolean advancedHealthChecksEnabled; public HealthService(DataSource dataSource, @Autowired(required = false) RedisTemplate redisTemplate) { this.dataSource = dataSource; this.redisTemplate = redisTemplate; + this.executorService = Executors.newFixedThreadPool(2); } - public Map checkHealth() { - Map healthStatus = new LinkedHashMap<>(); - Map components = new LinkedHashMap<>(); - boolean overallHealth = true; - - Map mysqlStatus = checkMySQLHealth(); - components.put("mysql", mysqlStatus); - if (!isHealthy(mysqlStatus)) { - overallHealth = false; + @PreDestroy + public void shutdown() { + if (executorService != null && !executorService.isShutdown()) { + try { + executorService.shutdown(); + if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) { + executorService.shutdownNow(); + logger.warn("ExecutorService did not terminate gracefully"); + } + } catch (InterruptedException e) { + executorService.shutdownNow(); + Thread.currentThread().interrupt(); + logger.warn("ExecutorService shutdown interrupted", e); + } } + } - if (redisTemplate != null) { - Map redisStatus = checkRedisHealth(); - components.put("redis", redisStatus); - if (!isHealthy(redisStatus)) { - overallHealth = false; + public Map checkHealth() { + Map response = new LinkedHashMap<>(); + response.put("timestamp", Instant.now().toString()); + + Map mysqlStatus = new ConcurrentHashMap<>(); + Map redisStatus = new ConcurrentHashMap<>(); + + // Submit both checks concurrently using executorService for proper cancellation support + Future mysqlFuture = executorService.submit( + () -> performHealthCheck("MySQL", mysqlStatus, this::checkMySQLHealthSync)); + Future redisFuture = executorService.submit( + () -> performHealthCheck("Redis", redisStatus, this::checkRedisHealthSync)); + + // Wait for both checks to complete with combined timeout (shared deadline) + long maxTimeout = Math.max(MYSQL_TIMEOUT_SECONDS, REDIS_TIMEOUT_SECONDS) + 1; + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(maxTimeout); + try { + mysqlFuture.get(maxTimeout, TimeUnit.SECONDS); + long remainingNs = deadlineNs - System.nanoTime(); + if (remainingNs > 0) { + redisFuture.get(remainingNs, TimeUnit.NANOSECONDS); + } else { + redisFuture.cancel(true); } + } catch (TimeoutException e) { + logger.warn("Health check aggregate timeout after {} seconds", maxTimeout); + mysqlFuture.cancel(true); + redisFuture.cancel(true); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.warn("Health check was interrupted"); + mysqlFuture.cancel(true); + redisFuture.cancel(true); + } catch (Exception e) { + logger.warn("Health check execution error: {}", e.getMessage()); } + + // Ensure timed-out or unfinished components are marked DOWN + ensurePopulated(mysqlStatus, "MySQL"); + ensurePopulated(redisStatus, "Redis"); + + Map> components = new LinkedHashMap<>(); + components.put("mysql", mysqlStatus); + components.put("redis", redisStatus); + + response.put("components", components); + + // Compute overall status + String overallStatus = computeOverallStatus(components); + response.put(STATUS_KEY, overallStatus); + + return response; + } - healthStatus.put(STATUS_KEY, overallHealth ? STATUS_UP : STATUS_DOWN); - healthStatus.put("timestamp", Instant.now().toString()); - healthStatus.put("components", components); - logger.info("Health check completed - Overall status: {}", overallHealth ? STATUS_UP : STATUS_DOWN); - - return healthStatus; + private void ensurePopulated(Map status, String componentName) { + if (!status.containsKey(STATUS_KEY)) { + status.put(STATUS_KEY, STATUS_DOWN); + status.put(SEVERITY_KEY, SEVERITY_CRITICAL); + status.put(ERROR_KEY, componentName + " health check did not complete in time"); + } } - private Map checkMySQLHealth() { - Map status = new LinkedHashMap<>(); - - return performHealthCheck("MySQL", status, () -> { - try { - try (Connection connection = dataSource.getConnection()) { - try (PreparedStatement stmt = connection.prepareStatement(DB_HEALTH_CHECK_QUERY)) { - stmt.setQueryTimeout(3); - try (ResultSet rs = stmt.executeQuery()) { - if (rs.next() && rs.getInt(1) == 1) { - return new HealthCheckResult(true, null); - } - } - } - return new HealthCheckResult(false, "Query validation failed"); + private HealthCheckResult checkMySQLHealthSync() { + try (Connection connection = dataSource.getConnection(); + PreparedStatement stmt = connection.prepareStatement("SELECT 1 as health_check")) { + + stmt.setQueryTimeout((int) MYSQL_TIMEOUT_SECONDS); + + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + // Basic health check passed, now run advanced checks with throttling + boolean isDegraded = performAdvancedMySQLChecksWithThrottle(connection); + return new HealthCheckResult(true, null, isDegraded); } - } catch (Exception e) { - throw new IllegalStateException("MySQL health check failed: " + e.getMessage(), e); } - }); + + return new HealthCheckResult(false, "No result from health check query", false); + + } catch (Exception e) { + logger.warn("MySQL health check failed: {}", e.getMessage(), e); + return new HealthCheckResult(false, "MySQL connection failed", false); + } } - private Map checkRedisHealth() { - Map status = new LinkedHashMap<>(); - - return performHealthCheck("Redis", status, () -> { - try { - String pong = CompletableFuture.supplyAsync(() -> - redisTemplate.execute((RedisCallback) connection -> connection.ping()), - executorService - ).get(REDIS_TIMEOUT_SECONDS, TimeUnit.SECONDS); - - if ("PONG".equals(pong)) { - return new HealthCheckResult(true, null); - } - return new HealthCheckResult(false, "Ping returned unexpected response"); - } catch (TimeoutException e) { - return new HealthCheckResult(false, "Redis ping timed out after " + REDIS_TIMEOUT_SECONDS + " seconds"); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return new HealthCheckResult(false, "Redis health check was interrupted"); - } catch (Exception e) { - throw new IllegalStateException("Redis health check failed", e); + private HealthCheckResult checkRedisHealthSync() { + if (redisTemplate == null) { + return new HealthCheckResult(true, "Redis not configured — skipped", false); + } + + try { + String pong = redisTemplate.execute((org.springframework.data.redis.core.RedisCallback) (connection) -> connection.ping()); + + if ("PONG".equals(pong)) { + return new HealthCheckResult(true, null, false); } - }); + + return new HealthCheckResult(false, "Redis PING failed", false); + + } catch (Exception e) { + logger.warn("Redis health check failed: {}", e.getMessage(), e); + return new HealthCheckResult(false, "Redis connection failed", false); + } } private Map performHealthCheck(String componentName, @@ -123,15 +206,28 @@ private Map performHealthCheck(String componentName, HealthCheckResult result = checker.get(); long responseTime = System.currentTimeMillis() - startTime; - status.put("responseTimeMs", responseTime); - - if (result.isHealthy) { - logger.debug("{} health check: UP ({}ms)", componentName, responseTime); - status.put(STATUS_KEY, STATUS_UP); + // Determine status: DOWN (unhealthy), DEGRADED (healthy but with issues), or UP + String componentStatus; + if (!result.isHealthy) { + componentStatus = STATUS_DOWN; + } else if (result.isDegraded) { + componentStatus = STATUS_DEGRADED; } else { - String safeError = result.error != null ? result.error : "Health check failed"; - logger.warn("{} health check failed: {}", componentName, safeError); - status.put(STATUS_KEY, STATUS_DOWN); + componentStatus = STATUS_UP; + } + status.put(STATUS_KEY, componentStatus); + + // Set response time + status.put(RESPONSE_TIME_KEY, responseTime); + + // Determine severity based on health, response time, and degradation flags + String severity = determineSeverity(result.isHealthy, responseTime, result.isDegraded); + status.put(SEVERITY_KEY, severity); + + // Include message or error based on health status + if (result.error != null) { + String fieldKey = result.isHealthy ? MESSAGE_KEY : ERROR_KEY; + status.put(fieldKey, result.error); } return status; @@ -141,23 +237,278 @@ private Map performHealthCheck(String componentName, logger.error("{} health check failed with exception: {}", componentName, e.getMessage(), e); status.put(STATUS_KEY, STATUS_DOWN); - status.put("responseTimeMs", responseTime); + status.put(RESPONSE_TIME_KEY, responseTime); + status.put(SEVERITY_KEY, SEVERITY_CRITICAL); + status.put(ERROR_KEY, "Health check failed with an unexpected error"); return status; } } - private boolean isHealthy(Map componentStatus) { - return STATUS_UP.equals(componentStatus.get(STATUS_KEY)); + private String determineSeverity(boolean isHealthy, long responseTimeMs, boolean isDegraded) { + if (!isHealthy) { + return SEVERITY_CRITICAL; + } + + if (isDegraded) { + return SEVERITY_WARNING; + } + + if (responseTimeMs > RESPONSE_TIME_THRESHOLD_MS) { + return SEVERITY_WARNING; + } + + return SEVERITY_OK; + } + + private String computeOverallStatus(Map> components) { + boolean hasCritical = false; + boolean hasDegraded = false; + + for (Map componentStatus : components.values()) { + String status = (String) componentStatus.get(STATUS_KEY); + String severity = (String) componentStatus.get(SEVERITY_KEY); + + if (STATUS_DOWN.equals(status) || SEVERITY_CRITICAL.equals(severity)) { + hasCritical = true; + } + + if (STATUS_DEGRADED.equals(status)) { + hasDegraded = true; + } + + if (SEVERITY_WARNING.equals(severity)) { + hasDegraded = true; + } + } + + if (hasCritical) { + return STATUS_DOWN; + } + + if (hasDegraded) { + return STATUS_DEGRADED; + } + + return STATUS_UP; + } + + // Internal advanced health checks for MySQL - do not expose details in responses + private boolean performAdvancedMySQLChecksWithThrottle(Connection connection) { + if (!advancedHealthChecksEnabled) { + return false; // Advanced checks disabled + } + + long currentTime = System.currentTimeMillis(); + + // Check throttle window - use read lock first for fast path + advancedCheckLock.readLock().lock(); + try { + if (cachedAdvancedCheckResult != null && + (currentTime - lastAdvancedCheckTime) < ADVANCED_CHECKS_THROTTLE_SECONDS * 1000) { + // Return cached result - within throttle window + return cachedAdvancedCheckResult.isDegraded; + } + } finally { + advancedCheckLock.readLock().unlock(); + } + + // Outside throttle window - acquire write lock and run checks + advancedCheckLock.writeLock().lock(); + try { + // Double-check after acquiring write lock + if (cachedAdvancedCheckResult != null && + (currentTime - lastAdvancedCheckTime) < ADVANCED_CHECKS_THROTTLE_SECONDS * 1000) { + return cachedAdvancedCheckResult.isDegraded; + } + + AdvancedCheckResult result = performAdvancedMySQLChecks(connection); + + // Cache the result + lastAdvancedCheckTime = currentTime; + cachedAdvancedCheckResult = result; + + return result.isDegraded; + } finally { + advancedCheckLock.writeLock().unlock(); + } + } + + private AdvancedCheckResult performAdvancedMySQLChecks(Connection connection) { + try { + boolean hasIssues = false; + + if (hasLockWaits(connection)) { + logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_LOCK_WAIT); + hasIssues = true; + } + + if (hasDeadlocks(connection)) { + logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_DEADLOCK); + hasIssues = true; + } + + if (hasSlowQueries(connection)) { + logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_SLOW_QUERIES); + hasIssues = true; + } + + if (hasConnectionPoolExhaustion()) { + logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_POOL_EXHAUSTED); + hasIssues = true; + } + + return new AdvancedCheckResult(hasIssues); + } catch (Exception e) { + logger.debug("Advanced MySQL checks encountered exception, marking degraded"); + return new AdvancedCheckResult(true); + } + } + + private boolean hasLockWaits(Connection connection) { + try (PreparedStatement stmt = connection.prepareStatement( + "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + + "WHERE (state = 'Waiting for table metadata lock' " + + " OR state = 'Waiting for row lock' " + + " OR state = 'Waiting for lock') " + + "AND user = USER()")) { + stmt.setQueryTimeout(2); + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + int lockCount = rs.getInt(1); + return lockCount > 0; + } + } + } catch (Exception e) { + logger.debug("Could not check for lock waits"); + } + return false; + } + + private boolean hasDeadlocks(Connection connection) { + // Skip deadlock check if already disabled due to permissions + if (deadlockCheckDisabled) { + return false; + } + + try (PreparedStatement stmt = connection.prepareStatement("SHOW ENGINE INNODB STATUS")) { + stmt.setQueryTimeout(2); + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + String innodbStatus = rs.getString(3); + return innodbStatus != null && innodbStatus.contains("LATEST DETECTED DEADLOCK"); + } + } + } catch (java.sql.SQLException e) { + // Check if this is a permission error + if (e.getMessage() != null && + (e.getMessage().contains("Access denied") || + e.getMessage().contains("permission"))) { + // Disable this check permanently after first permission error + deadlockCheckDisabled = true; + logger.warn("Deadlock check disabled: Insufficient privileges"); + } else { + logger.debug("Could not check for deadlocks"); + } + } catch (Exception e) { + logger.debug("Could not check for deadlocks"); + } + return false; + } + + private boolean hasSlowQueries(Connection connection) { + try (PreparedStatement stmt = connection.prepareStatement( + "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + + "WHERE command != 'Sleep' AND time > ? AND user NOT IN ('event_scheduler', 'system user')")) { + stmt.setQueryTimeout(2); + stmt.setInt(1, 10); // Queries running longer than 10 seconds + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + int slowQueryCount = rs.getInt(1); + return slowQueryCount > 3; // Alert if more than 3 slow queries + } + } + } catch (Exception e) { + logger.debug("Could not check for slow queries"); + } + return false; + } + + private boolean hasConnectionPoolExhaustion() { + // Use HikariCP metrics if available + if (dataSource instanceof HikariDataSource hikariDataSource) { + try { + HikariPoolMXBean poolMXBean = hikariDataSource.getHikariPoolMXBean(); + + if (poolMXBean != null) { + int activeConnections = poolMXBean.getActiveConnections(); + int maxPoolSize = hikariDataSource.getMaximumPoolSize(); + + // Alert if > 80% of pool is exhausted + int threshold = (int) (maxPoolSize * 0.8); + return activeConnections > threshold; + } + } catch (Exception e) { + logger.debug("Could not retrieve HikariCP pool metrics"); + } + } + + // Fallback: try to get pool metrics via JMX if HikariCP is not directly available + return checkPoolMetricsViaJMX(); + } + + private boolean checkPoolMetricsViaJMX() { + try { + MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer(); + ObjectName objectName = new ObjectName("com.zaxxer.hikari:type=Pool (*)"); + var mBeans = mBeanServer.queryMBeans(objectName, null); + + for (var mBean : mBeans) { + if (evaluatePoolMetrics(mBeanServer, mBean.getObjectName())) { + return true; + } + } + } catch (Exception e) { + logger.debug("Could not access HikariCP pool metrics via JMX"); + } + + // No pool metrics available - disable this check + logger.debug("Pool exhaustion check disabled: HikariCP metrics unavailable"); + return false; + } + + private boolean evaluatePoolMetrics(MBeanServer mBeanServer, ObjectName objectName) { + try { + Integer activeConnections = (Integer) mBeanServer.getAttribute(objectName, "ActiveConnections"); + Integer maximumPoolSize = (Integer) mBeanServer.getAttribute(objectName, "MaximumPoolSize"); + + if (activeConnections != null && maximumPoolSize != null) { + int threshold = (int) (maximumPoolSize * 0.8); + return activeConnections > threshold; + } + } catch (Exception e) { + // Continue to next MBean + } + return false; + } + + private static class AdvancedCheckResult { + final boolean isDegraded; + + AdvancedCheckResult(boolean isDegraded) { + this.isDegraded = isDegraded; + } } private static class HealthCheckResult { final boolean isHealthy; final String error; + final boolean isDegraded; - HealthCheckResult(boolean isHealthy, String error) { + HealthCheckResult(boolean isHealthy, String error, boolean isDegraded) { this.isHealthy = isHealthy; this.error = error; + this.isDegraded = isDegraded; } } } From e133243db048e5218b925bec8e593add74f9618f Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sun, 22 Feb 2026 09:12:09 +0530 Subject: [PATCH 6/8] fix(health): remove stale deadlock check from MySQL health indicator --- .../bengen/service/health/HealthService.java | 88 ++++++------------- 1 file changed, 27 insertions(+), 61 deletions(-) diff --git a/src/main/java/com/iemr/common/bengen/service/health/HealthService.java b/src/main/java/com/iemr/common/bengen/service/health/HealthService.java index 5c1058d..2499977 100644 --- a/src/main/java/com/iemr/common/bengen/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/bengen/service/health/HealthService.java @@ -25,6 +25,7 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.redis.core.RedisCallback; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; @@ -50,7 +51,6 @@ public class HealthService { private static final long ADVANCED_CHECKS_THROTTLE_SECONDS = 30; private static final long RESPONSE_TIME_THRESHOLD_MS = 2000; private static final String DIAGNOSTIC_LOCK_WAIT = "MYSQL_LOCK_WAIT"; - private static final String DIAGNOSTIC_DEADLOCK = "MYSQL_DEADLOCK"; private static final String DIAGNOSTIC_SLOW_QUERIES = "MYSQL_SLOW_QUERIES"; private static final String DIAGNOSTIC_POOL_EXHAUSTED = "MYSQL_POOL_EXHAUSTED"; private static final String DIAGNOSTIC_LOG_TEMPLATE = "Diagnostic: {}"; @@ -63,9 +63,6 @@ public class HealthService { private volatile AdvancedCheckResult cachedAdvancedCheckResult = null; private final ReentrantReadWriteLock advancedCheckLock = new ReentrantReadWriteLock(); - // Deadlock check resilience - disable after first permission error - private volatile boolean deadlockCheckDisabled = false; - @Value("${health.advanced.enabled:true}") private boolean advancedHealthChecksEnabled; @@ -108,9 +105,29 @@ public Map checkHealth() { // Wait for both checks to complete with combined timeout (shared deadline) long maxTimeout = Math.max(MYSQL_TIMEOUT_SECONDS, REDIS_TIMEOUT_SECONDS) + 1; - long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(maxTimeout); + awaitHealthChecks(mysqlFuture, redisFuture, maxTimeout); + + // Ensure timed-out or unfinished components are marked DOWN + ensurePopulated(mysqlStatus, "MySQL"); + ensurePopulated(redisStatus, "Redis"); + + Map> components = new LinkedHashMap<>(); + components.put("mysql", mysqlStatus); + components.put("redis", redisStatus); + + response.put("components", components); + + // Compute overall status + String overallStatus = computeOverallStatus(components); + response.put(STATUS_KEY, overallStatus); + + return response; + } + + private void awaitHealthChecks(Future mysqlFuture, Future redisFuture, long maxTimeoutSeconds) { + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(maxTimeoutSeconds); try { - mysqlFuture.get(maxTimeout, TimeUnit.SECONDS); + mysqlFuture.get(maxTimeoutSeconds, TimeUnit.SECONDS); long remainingNs = deadlineNs - System.nanoTime(); if (remainingNs > 0) { redisFuture.get(remainingNs, TimeUnit.NANOSECONDS); @@ -118,7 +135,7 @@ public Map checkHealth() { redisFuture.cancel(true); } } catch (TimeoutException e) { - logger.warn("Health check aggregate timeout after {} seconds", maxTimeout); + logger.warn("Health check aggregate timeout after {} seconds", maxTimeoutSeconds); mysqlFuture.cancel(true); redisFuture.cancel(true); } catch (InterruptedException e) { @@ -129,22 +146,6 @@ public Map checkHealth() { } catch (Exception e) { logger.warn("Health check execution error: {}", e.getMessage()); } - - // Ensure timed-out or unfinished components are marked DOWN - ensurePopulated(mysqlStatus, "MySQL"); - ensurePopulated(redisStatus, "Redis"); - - Map> components = new LinkedHashMap<>(); - components.put("mysql", mysqlStatus); - components.put("redis", redisStatus); - - response.put("components", components); - - // Compute overall status - String overallStatus = computeOverallStatus(components); - response.put(STATUS_KEY, overallStatus); - - return response; } private void ensurePopulated(Map status, String componentName) { @@ -183,7 +184,8 @@ private HealthCheckResult checkRedisHealthSync() { } try { - String pong = redisTemplate.execute((org.springframework.data.redis.core.RedisCallback) (connection) -> connection.ping()); + String pong = redisTemplate.execute( + (RedisCallback) connection -> connection.ping()); if ("PONG".equals(pong)) { return new HealthCheckResult(true, null, false); @@ -343,11 +345,6 @@ private AdvancedCheckResult performAdvancedMySQLChecks(Connection connection) { hasIssues = true; } - if (hasDeadlocks(connection)) { - logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_DEADLOCK); - hasIssues = true; - } - if (hasSlowQueries(connection)) { logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_SLOW_QUERIES); hasIssues = true; @@ -371,7 +368,7 @@ private boolean hasLockWaits(Connection connection) { "WHERE (state = 'Waiting for table metadata lock' " + " OR state = 'Waiting for row lock' " + " OR state = 'Waiting for lock') " + - "AND user = USER()")) { + "AND user = SUBSTRING_INDEX(USER(), '@', 1)")) { stmt.setQueryTimeout(2); try (ResultSet rs = stmt.executeQuery()) { if (rs.next()) { @@ -385,37 +382,6 @@ private boolean hasLockWaits(Connection connection) { return false; } - private boolean hasDeadlocks(Connection connection) { - // Skip deadlock check if already disabled due to permissions - if (deadlockCheckDisabled) { - return false; - } - - try (PreparedStatement stmt = connection.prepareStatement("SHOW ENGINE INNODB STATUS")) { - stmt.setQueryTimeout(2); - try (ResultSet rs = stmt.executeQuery()) { - if (rs.next()) { - String innodbStatus = rs.getString(3); - return innodbStatus != null && innodbStatus.contains("LATEST DETECTED DEADLOCK"); - } - } - } catch (java.sql.SQLException e) { - // Check if this is a permission error - if (e.getMessage() != null && - (e.getMessage().contains("Access denied") || - e.getMessage().contains("permission"))) { - // Disable this check permanently after first permission error - deadlockCheckDisabled = true; - logger.warn("Deadlock check disabled: Insufficient privileges"); - } else { - logger.debug("Could not check for deadlocks"); - } - } catch (Exception e) { - logger.debug("Could not check for deadlocks"); - } - return false; - } - private boolean hasSlowQueries(Connection connection) { try (PreparedStatement stmt = connection.prepareStatement( "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + From edf6a932baf43eca8feee602b4ae1dcbf86a405d Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sun, 22 Feb 2026 19:56:46 +0530 Subject: [PATCH 7/8] fix(health): make advanced MySQL checks interruptible on timeout --- .../bengen/service/health/HealthService.java | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/iemr/common/bengen/service/health/HealthService.java b/src/main/java/com/iemr/common/bengen/service/health/HealthService.java index 2499977..2b4bdec 100644 --- a/src/main/java/com/iemr/common/bengen/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/bengen/service/health/HealthService.java @@ -24,7 +24,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.data.redis.core.RedisCallback; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; @@ -63,14 +62,14 @@ public class HealthService { private volatile AdvancedCheckResult cachedAdvancedCheckResult = null; private final ReentrantReadWriteLock advancedCheckLock = new ReentrantReadWriteLock(); - @Value("${health.advanced.enabled:true}") - private boolean advancedHealthChecksEnabled; + // Advanced checks always enabled + private static final boolean ADVANCED_HEALTH_CHECKS_ENABLED = true; public HealthService(DataSource dataSource, @Autowired(required = false) RedisTemplate redisTemplate) { this.dataSource = dataSource; this.redisTemplate = redisTemplate; - this.executorService = Executors.newFixedThreadPool(2); + this.executorService = Executors.newFixedThreadPool(6); } @PreDestroy @@ -97,7 +96,6 @@ public Map checkHealth() { Map mysqlStatus = new ConcurrentHashMap<>(); Map redisStatus = new ConcurrentHashMap<>(); - // Submit both checks concurrently using executorService for proper cancellation support Future mysqlFuture = executorService.submit( () -> performHealthCheck("MySQL", mysqlStatus, this::checkMySQLHealthSync)); Future redisFuture = executorService.submit( @@ -163,19 +161,16 @@ private HealthCheckResult checkMySQLHealthSync() { stmt.setQueryTimeout((int) MYSQL_TIMEOUT_SECONDS); try (ResultSet rs = stmt.executeQuery()) { - if (rs.next()) { - // Basic health check passed, now run advanced checks with throttling - boolean isDegraded = performAdvancedMySQLChecksWithThrottle(connection); - return new HealthCheckResult(true, null, isDegraded); + if (!rs.next()) { + return new HealthCheckResult(false, "No result from health check query", false); } } - - return new HealthCheckResult(false, "No result from health check query", false); - } catch (Exception e) { logger.warn("MySQL health check failed: {}", e.getMessage(), e); return new HealthCheckResult(false, "MySQL connection failed", false); } + boolean isDegraded = performAdvancedMySQLChecksWithThrottle(); + return new HealthCheckResult(true, null, isDegraded); } private HealthCheckResult checkRedisHealthSync() { @@ -296,8 +291,8 @@ private String computeOverallStatus(Map> components) } // Internal advanced health checks for MySQL - do not expose details in responses - private boolean performAdvancedMySQLChecksWithThrottle(Connection connection) { - if (!advancedHealthChecksEnabled) { + private boolean performAdvancedMySQLChecksWithThrottle() { + if (!ADVANCED_HEALTH_CHECKS_ENABLED) { return false; // Advanced checks disabled } @@ -324,7 +319,13 @@ private boolean performAdvancedMySQLChecksWithThrottle(Connection connection) { return cachedAdvancedCheckResult.isDegraded; } - AdvancedCheckResult result = performAdvancedMySQLChecks(connection); + AdvancedCheckResult result; + try (Connection conn = dataSource.getConnection()) { + result = performAdvancedMySQLChecks(conn); + } catch (Exception ex) { + logger.debug("Could not acquire connection for advanced checks: {}", ex.getMessage()); + result = new AdvancedCheckResult(true); + } // Cache the result lastAdvancedCheckTime = currentTime; @@ -385,7 +386,7 @@ private boolean hasLockWaits(Connection connection) { private boolean hasSlowQueries(Connection connection) { try (PreparedStatement stmt = connection.prepareStatement( "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + - "WHERE command != 'Sleep' AND time > ? AND user NOT IN ('event_scheduler', 'system user')")) { + "WHERE command != 'Sleep' AND time > ? AND user = SUBSTRING_INDEX(USER(), '@', 1)")) { stmt.setQueryTimeout(2); stmt.setInt(1, 10); // Queries running longer than 10 seconds try (ResultSet rs = stmt.executeQuery()) { From 2bd705049bf7b51d641a031d4064e93aded49ef7 Mon Sep 17 00:00:00 2001 From: DurgaPrasad-54 Date: Sun, 22 Feb 2026 20:09:50 +0530 Subject: [PATCH 8/8] fix(health): avoid blocking DB I/O under write lock and restore interrupt flag --- .../bengen/service/health/HealthService.java | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/iemr/common/bengen/service/health/HealthService.java b/src/main/java/com/iemr/common/bengen/service/health/HealthService.java index 2b4bdec..3ce59df 100644 --- a/src/main/java/com/iemr/common/bengen/service/health/HealthService.java +++ b/src/main/java/com/iemr/common/bengen/service/health/HealthService.java @@ -318,19 +318,27 @@ private boolean performAdvancedMySQLChecksWithThrottle() { (currentTime - lastAdvancedCheckTime) < ADVANCED_CHECKS_THROTTLE_SECONDS * 1000) { return cachedAdvancedCheckResult.isDegraded; } - - AdvancedCheckResult result; - try (Connection conn = dataSource.getConnection()) { - result = performAdvancedMySQLChecks(conn); - } catch (Exception ex) { - logger.debug("Could not acquire connection for advanced checks: {}", ex.getMessage()); - result = new AdvancedCheckResult(true); + } finally { + advancedCheckLock.writeLock().unlock(); + } + + // Perform DB I/O outside the write lock to avoid lock contention + AdvancedCheckResult result; + try (Connection conn = dataSource.getConnection()) { + result = performAdvancedMySQLChecks(conn); + } catch (Exception ex) { + if (ex.getCause() instanceof InterruptedException) { + Thread.currentThread().interrupt(); } - - // Cache the result + logger.debug("Could not acquire connection for advanced checks: {}", ex.getMessage()); + result = new AdvancedCheckResult(false); // don't mark degraded on acquisition failure + } + + // Re-acquire write lock only to update the cache atomically + advancedCheckLock.writeLock().lock(); + try { lastAdvancedCheckTime = currentTime; cachedAdvancedCheckResult = result; - return result.isDegraded; } finally { advancedCheckLock.writeLock().unlock();