From 84679719ca3ae6c71f5b65b4b09dacd20821c86f Mon Sep 17 00:00:00 2001 From: lyh Date: Tue, 18 Aug 2026 14:54:37 +0800 Subject: [PATCH 1/2] fix(metadata): recover from stale connections closed by server wait_timeout (#4570) --- .../metadata/driver/AbstractJdbcDriver.java | 53 +++++++++++++++---- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java index 3f5fa3e4c8..4e52d92fdf 100644 --- a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java +++ b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java @@ -130,37 +130,70 @@ protected void createDataSource(DruidDataSource ds, AbstractJdbcConfig connectCo ds.setPassword(connectConfig.getPassword()); ds.setValidationQuery(validationQuery); ds.setTestWhileIdle(true); - ds.setBreakAfterAcquireFailure(true); + // Allow the pool to recover after a connection failure instead of breaking permanently + ds.setBreakAfterAcquireFailure(false); ds.setFailFast(true); ds.setInitialSize(1); ds.setMaxActive(8); ds.setMinIdle(5); + // Keep idle connections alive via validation query to prevent server-side wait_timeout closure + ds.setKeepAlive(true); + // Evict connections idle for more than 5 minutes (min) / 15 minutes (max) + ds.setMinEvictableIdleTimeMillis(300000); + ds.setMaxEvictableIdleTimeMillis(900000); } @Override public Driver connect() { - if (Asserts.isNull(conn.get())) { - try { - Class.forName(getDriverClass()); - DruidPooledConnection connection = createDataSource().getConnection(); - conn.set(connection); - } catch (ClassNotFoundException | SQLException e) { - throw new RuntimeException(e); + try { + Connection currentConn = conn.get(); + if (Asserts.isNotNull(currentConn)) { + PreparedStatement preparedStatement = null; + try { + // Validate existing connection by executing the validation query + preparedStatement = currentConn.prepareStatement(validationQuery); + preparedStatement.executeQuery(); + return this; + } catch (Exception e) { + // Connection is invalid (closed, disabled, or timed out), reconnect + log.warn("Connection is invalid, reconnecting: {}", e.getMessage()); + try { + currentConn.close(); + } catch (Exception ignore) { + } + conn.remove(); + } finally { + close(preparedStatement, null); + } } + // Connection is null or invalid, create new connection + Class.forName(getDriverClass()); + DruidPooledConnection connection = createDataSource().getConnection(); + conn.set(connection); + } catch (ClassNotFoundException | SQLException e) { + throw new RuntimeException(e); } return this; } @Override public boolean isHealth() { + PreparedStatement preparedStatement = null; try { if (Asserts.isNotNull(conn.get())) { - return !conn.get().isClosed(); + // Use validation query to truly test connectivity instead of just isClosed(). + // isClosed() only checks the Druid wrapper state, not the underlying physical + // connection which may have been closed by the server due to wait_timeout. + preparedStatement = conn.get().prepareStatement(validationQuery); + preparedStatement.executeQuery(); + return true; } return false; } catch (Exception e) { - log.error("check is health errr:", e); + log.warn("Connection health check failed, will reconnect: {}", e.getMessage()); return false; + } finally { + close(preparedStatement, null); } } From 73b27c397e2624ca7ca1ef5b89268bfb7fcd61c9 Mon Sep 17 00:00:00 2001 From: lyh Date: Wed, 19 Aug 2026 17:22:45 +0800 Subject: [PATCH 2/2] refactor(metadata): enrich reconnect logs and clean up dead conn in isHealth (#4570) --- .../metadata/driver/AbstractJdbcDriver.java | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java index 4e52d92fdf..a8d9429908 100644 --- a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java +++ b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java @@ -155,11 +155,16 @@ public Driver connect() { preparedStatement.executeQuery(); return this; } catch (Exception e) { - // Connection is invalid (closed, disabled, or timed out), reconnect - log.warn("Connection is invalid, reconnecting: {}", e.getMessage()); + // Stale connection detected (e.g. closed by server due to wait_timeout), + // close and discard it, then rebuild via the pool below + log.warn( + "Stale connection detected, closing it and reconnecting, datasource: {}, reason: {}", + config.getName(), + e.getMessage()); try { currentConn.close(); } catch (Exception ignore) { + // connection already broken, close failure is expected } conn.remove(); } finally { @@ -179,18 +184,33 @@ public Driver connect() { @Override public boolean isHealth() { PreparedStatement preparedStatement = null; + Connection currentConn = conn.get(); try { if (Asserts.isNotNull(conn.get())) { // Use validation query to truly test connectivity instead of just isClosed(). // isClosed() only checks the Druid wrapper state, not the underlying physical // connection which may have been closed by the server due to wait_timeout. - preparedStatement = conn.get().prepareStatement(validationQuery); + preparedStatement = currentConn.prepareStatement(validationQuery); preparedStatement.executeQuery(); return true; } return false; } catch (Exception e) { - log.warn("Connection health check failed, will reconnect: {}", e.getMessage()); + log.warn( + "Connection health check failed, stale connection will be closed and reconnected, datasource: {}, reason: {}", + config.getName(), + e.getMessage()); + // The probe has already confirmed the connection is dead, close and remove + // it right here, so that connect() won't probe the same broken connection + // a second time (which would cost another round of timeout) + conn.remove(); + if (Asserts.isNotNull(currentConn)) { + try { + currentConn.close(); + } catch (Exception ignore) { + // connection already broken, close failure is expected + } + } return false; } finally { close(preparedStatement, null);