diff --git a/buildSrc/src/main/java/deltix/buildtools/SonatypeCentralPortalUploadRepositoryTask.java b/buildSrc/src/main/java/deltix/buildtools/SonatypeCentralPortalUploadRepositoryTask.java index 555125b5..fc0a8a92 100644 --- a/buildSrc/src/main/java/deltix/buildtools/SonatypeCentralPortalUploadRepositoryTask.java +++ b/buildSrc/src/main/java/deltix/buildtools/SonatypeCentralPortalUploadRepositoryTask.java @@ -32,9 +32,15 @@ import java.net.SocketTimeoutException; import java.net.URI; import java.net.URL; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.Base64; +import static java.nio.charset.StandardCharsets.US_ASCII; + /** * This task performs manual steps to publish artifacts to Central Portal via OSSRH Staging API. */ @@ -110,16 +116,29 @@ public void run() throws IOException, InterruptedException { return; } - String userNameAndPassword = portalUsername.get() + ":" + portalPassword.get(); - String bearer = Base64.getEncoder().encodeToString(userNameAndPassword.getBytes(StandardCharsets.US_ASCII)); + final HttpClient httpClient = HttpClient.newBuilder() + .connectTimeout(Duration.ofMillis(CONNECTION_TIMEOUT)) + .build(); + + final String userNameAndPassword = portalUsername.get() + ":" + portalPassword.get(); + final String bearer = new String( + Base64.getEncoder().encode(userNameAndPassword.getBytes(US_ASCII)), US_ASCII); + + final HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() + .header("Authorization", "Bearer " + bearer); + URI apiUri = URI.create(CENTRAL_PORTAL_OSSRH_API_URI); String repositoryKey = findOpenRepository(apiUri, bearer); System.out.println("Found published repository: " + repositoryKey); - int status = uploadRepository(apiUri, bearer, repositoryKey); + int status = uploadRepositoryToPortal(apiUri, httpClient, requestBuilder, repositoryKey); if (status == HttpURLConnection.HTTP_CLIENT_TIMEOUT) - uploadRepository(apiUri, bearer, repositoryKey); + uploadRepositoryToPortal(apiUri, httpClient, requestBuilder, repositoryKey); + +// int status = uploadRepository(apiUri, bearer, repositoryKey); +// if (status == HttpURLConnection.HTTP_CLIENT_TIMEOUT) +// uploadRepository(apiUri, bearer, repositoryKey); dropRepository(apiUri, bearer, repositoryKey); } @@ -189,6 +208,28 @@ private static int uploadRepository(URI apiUri, String bearer, String repository return status; } + private static int uploadRepositoryToPortal( + final URI apiUri, + final HttpClient httpClient, + final HttpRequest.Builder requestBuilder, + final String repositoryKey) throws IOException, InterruptedException { + + HttpRequest request = requestBuilder + .copy() + .POST(HttpRequest.BodyPublishers.noBody()) + .uri(apiUri.resolve("/manual/upload/repository/" + repositoryKey + "?publishing_type=automatic")) + .build(); + HttpResponse response = httpClient.send( + request, (HttpResponse.ResponseInfo responseInfo) -> HttpResponse.BodySubscribers.ofString(US_ASCII)); + + return response.statusCode(); + +// if (200 != response.statusCode()) { +// throw new IllegalStateException("Failed to upload repository: repository_key=" + repositoryKey + +// ", status=" + response.statusCode() + ", response=" + response.body()); +// } + } + private static void dropRepository(URI apiUri, String bearer, String repositoryKey) throws IOException { String endpoint = apiUri.resolve("/manual/drop/repository/" + repositoryKey).toString(); HttpURLConnection conn = (HttpURLConnection) new URL(endpoint).openConnection(); diff --git a/java/build.gradle b/java/build.gradle index 8b6461c9..6b64db71 100644 --- a/java/build.gradle +++ b/java/build.gradle @@ -97,8 +97,8 @@ configure(leafProjects) { compileTestJava.options.compilerArgs += '--add-opens=java.base/java.lang=ALL-UNNAMED' compileTestJava.options.compilerArgs += '--add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED' - sourceCompatibility = 11 - targetCompatibility = 11 + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 configurations.all { resolutionStrategy { @@ -301,7 +301,8 @@ configure(leafProjects) { dependency 'org.slf4j:slf4j-log4j12:1.7.10' dependency 'com.sun.xml.bind:jaxb-impl:2.3.0' - dependency 'junit:junit:4.13.1' + + dependency "junit:junit:4.13.2" dependency 'com.nimbusds:nimbus-jose-jwt:9.40' @@ -351,8 +352,17 @@ configure(leafProjects) { compileOnly 'com.google.code.findbugs:annotations' testCompileOnly 'com.google.code.findbugs:jsr305' - testCompile 'junit:junit:4.13.1' - testCompile 'org.mockito:mockito-core:1.10.19' + + testImplementation "org.mockito:mockito-core:5.14.2" + testImplementation 'org.mockito:mockito-junit-jupiter:5.14.2' + testImplementation 'org.mockito:mockito-inline:4.11.0' + + testImplementation 'junit:junit' + testImplementation 'org.junit.jupiter:junit-jupiter-api:5.12.2' + + testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.12.2' + testRuntimeOnly 'org.junit.vintage:junit-vintage-engine:5.12.2' + testRuntimeOnly 'org.junit.platform:junit-platform-launcher:1.12.2' } spotbugs { diff --git a/java/timebase/api/build.gradle b/java/timebase/api/build.gradle index e610ffaf..818fa226 100644 --- a/java/timebase/api/build.gradle +++ b/java/timebase/api/build.gradle @@ -33,4 +33,7 @@ dependencies { // TODO: Move to client and server? implementation ('io.aeron:aeron-client') implementation ('io.aeron:aeron-driver') + + // For simulation of connection loss scenarios + testImplementation 'com.github.netcrusherorg:netcrusher-core:0.10' } \ No newline at end of file diff --git a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/ChannelOutputStream.java b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/ChannelOutputStream.java index 6da705ba..fc8b35e6 100644 --- a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/ChannelOutputStream.java +++ b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/ChannelOutputStream.java @@ -19,6 +19,7 @@ import com.epam.deltix.util.concurrent.UncheckedInterruptedException; import com.epam.deltix.util.lang.Util; import net.jcip.annotations.GuardedBy; +import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; import java.io.IOException; @@ -28,7 +29,7 @@ * Date: Mar 25, 2010 */ public class ChannelOutputStream extends VSOutputStream { - //@ApiStatus.Experimental // Temporary option for testing performance effect of flushing single packet + @ApiStatus.Experimental // Temporary option for testing performance effect of flushing single packet private static final boolean SINGLE_SEND_ON_PARTIAL_FLUSH = Boolean.getBoolean("TimeBase.network.channel.singleSendOnPartialFlush"); private final int maxCapacity; @@ -87,12 +88,12 @@ public synchronized void enableFlushing() throws IOException { // However, if we are above 75% capacity, we should flush all data // and block till all accumulated data is sent. // Otherwise, if the consumer too slow, the buffer will start to grow indefinitely. - // See https://gitlab.deltixhub.com/Deltix/QuantServer/QuantServer/-/issues/1298 int buffer75percent = halfCapacity + (halfCapacity >> 1); boolean partialOk = size < buffer75percent; try { flushInternal(partialOk, false); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new UncheckedInterruptedException(e); } } @@ -269,6 +270,7 @@ else if (newSize <= buffer.length) { send (b, off, len); } } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new UncheckedInterruptedException (e); } } @@ -290,6 +292,7 @@ else if (size >= maxCapacity) buffer [size++] = (byte) b; } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new UncheckedInterruptedException (e); } } @@ -367,4 +370,4 @@ public String toString() { return "ChannelOutputStream@" + Integer.toHexString(hashCode()) + " for channel=" + channel; } -} \ No newline at end of file +} diff --git a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/ConnectionStateListener.java b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/ConnectionStateListener.java index 58ce3934..26e96da9 100644 --- a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/ConnectionStateListener.java +++ b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/ConnectionStateListener.java @@ -17,18 +17,31 @@ package com.epam.deltix.util.vsocket; abstract class ConnectionStateListener { - + /** + * Triggered when connection loss causes dispatcher to give up on recovery. + *

+ * Triggered only once per dispatcher lifecycle. + *

+ * Not triggered if dispatcher is stopped normally with {@link VSDispatcher#close()}. + */ abstract void onDisconnected(); - abstract void onReconnected(); + /** + * Triggered when the first connection is established. + */ + abstract void onConnected(); /** - * @return true if transport is already known to be unrecoverable + * Triggered when transport is stopped (e.g. connection lost) but may be recoverable. + * + * @return true if transport is already known to be unrecoverable (and recovery should be stopped right away) */ - abstract boolean onTransportStopped(VSocketRecoveryInfo recoveryInfo); + abstract boolean onTransportRecoveryStart(VSocketRecoveryInfo recoveryInfo); /** + * Triggered when transport recovery have to stop (because of timeout or dispatcher shutdown). + * * @return true if transport was permanently lost (can't be recovered anymore) */ - abstract boolean onTransportBroken(VSocketRecoveryInfo recoveryInfo); -} \ No newline at end of file + abstract boolean onTransportRecoveryStop(VSocketRecoveryInfo recoveryInfo); +} diff --git a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSChannelImpl.java b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSChannelImpl.java index eb89ef03..65365cf0 100644 --- a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSChannelImpl.java +++ b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSChannelImpl.java @@ -315,6 +315,7 @@ public void close (boolean terminate) { } catch (ConnectionAbortedException x) { LOGGER.log (Level.FINE, "Error sending disconnect.", x); } catch (InterruptedException x) { + Thread.currentThread().interrupt(); LOGGER.log (Level.FINE, "Sending disconnect interrupted.", x); } catch (Exception x) { LOGGER.log (Level.WARNING, "Error sending disconnect", x); diff --git a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSClient.java b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSClient.java index a5fcad63..65206c2a 100644 --- a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSClient.java +++ b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSClient.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 EPAM Systems, Inc + * Copyright 2026 EPAM Systems, Inc * * See the NOTICE file distributed with this work for additional information * regarding copyright ownership. Licensed under the Apache License, @@ -17,26 +17,29 @@ package com.epam.deltix.util.vsocket; import com.epam.deltix.util.ContextContainer; +import com.epam.deltix.util.concurrent.QuickExecutor; +import com.epam.deltix.qsrv.hf.spi.conn.DisconnectEventListener; import com.epam.deltix.util.io.GUID; import com.epam.deltix.util.io.IOUtil; -import com.epam.deltix.util.io.aeron.DXAeron; import com.epam.deltix.util.io.offheap.OffHeap; import com.epam.deltix.util.lang.Disposable; -import com.epam.deltix.util.concurrent.QuickExecutor; -import com.epam.deltix.qsrv.hf.spi.conn.DisconnectEventListener; import com.epam.deltix.util.lang.DisposableListener; import com.epam.deltix.util.time.GlobalTimer; import com.epam.deltix.util.time.TimeKeeper; +import com.epam.deltix.util.time.TimerRunner; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.VisibleForTesting; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLSocket; import java.io.*; -import java.net.*; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Socket; import java.util.Date; -import java.util.TimerTask; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.logging.Level; @@ -53,10 +56,14 @@ public class VSClient extends ConnectionStateListener implements Disposable, Dis public static final boolean SSL_TERMINATION = Boolean.getBoolean(SSL_TERMINATION_PROPERTY); //private static final int MAX_TRANSPORT_RECONNECT_ATTEMPTS = Integer.getInteger("TimeBase.network.VSClient.maxTransportReconnectAttempts", 5); - private static final int TRANSPORT_RECONNECT_ATTEMPT_INTERVAL = Integer.getInteger("TimeBase.network.VSClient.transportReconnectAttemptInterval", 1000); + private final int transportReconnectAttemptInterval; + private final int socketSendBufferSize; + private final int socketReceiveBufferSize; private static final int RE_ATTEMPT_EXTRA_DELAY = 10; // Extra delay to avoid situation when we re-schedule task due to timer jitter + private static final VSClientOptions DEFAULT_CLIENT_OPTIONS = new VSClientOptions(); + private String host; private int port; private int numTransportChannels = 3; @@ -66,19 +73,19 @@ public class VSClient extends ConnectionStateListener implements Disposable, Dis private final Object dispatcherLock = new Object(); private String clientId; - private long serverTime = -1; + private long serverTime = -1; private volatile DisconnectEventListener listener; private int reconnectInterval; private VSCompression serverCompression; - private int soTimeout = Integer.getInteger("TimeBase.network.VSClient.soTimeout", 5000); - private int timeout = Integer.getInteger("TimeBase.network.VSClient.timeout", 5000); + private int soTimeout; + private int timeout; private boolean enableSSL = false; private final boolean sslTermination; private int sslPort = 0; - private SSLContext sslContext; + private SSLContext sslContext; private final ContextContainer contextContainer; @@ -88,7 +95,8 @@ public class VSClient extends ConnectionStateListener implements Disposable, Dis private volatile boolean closed = false; // Elements should be sorted (when possible) by time of last reconnection attempt however there is no strict enforcement for this. - // New broken sockets should be added to the head of the queue + // New broken sockets should be added to the head of the queue. + // Broken sockets that failed to reconnect should be added to the tail of the queue. private final ConcurrentLinkedDeque broken = new ConcurrentLinkedDeque<>(); private final QuickExecutor.QuickTask reconnector; @@ -96,8 +104,8 @@ public class VSClient extends ConnectionStateListener implements Disposable, Dis private QuickExecutor.QuickTask createReconnectorTask(final QuickExecutor quickExecutor) { return new QuickExecutor.QuickTask(quickExecutor) { @Override - public void run() { - for (;;) { + public void run() { + for ( ;; ) { long currentTime = TimeKeeper.currentTime; VSocketRecoveryInfo socketRecovery = broken.peek(); @@ -106,9 +114,9 @@ public void run() { break; long lastReconnectAttemptTs = socketRecovery.getLastReconnectAttemptTs(); - if (lastReconnectAttemptTs > currentTime - TRANSPORT_RECONNECT_ATTEMPT_INTERVAL) { + if (lastReconnectAttemptTs > currentTime - transportReconnectAttemptInterval) { // It's too early to recover this socket - scheduleReconnectAttempt(lastReconnectAttemptTs + TRANSPORT_RECONNECT_ATTEMPT_INTERVAL + RE_ATTEMPT_EXTRA_DELAY); + scheduleReconnectAttempt(lastReconnectAttemptTs + transportReconnectAttemptInterval + RE_ATTEMPT_EXTRA_DELAY); return; } @@ -132,19 +140,22 @@ public void run() { VSocket socket = socketRecovery.getSocket(); // Start reconnect attempt - int attemptNumber = socketRecovery.addReconnectAttempt(currentTime); + int attemptNumber; + synchronized (socketRecovery) { + attemptNumber = socketRecovery.addReconnectAttempt(currentTime); + } boolean success = false; boolean transportLost = false; try { + // Try to reconnect - long operation VSocket vSocket = openTransport(socket); if (vSocket != null) { success = true; dispatcher.addTransportChannel(vSocket); synchronized (socketRecovery) { - if (!socketRecovery.isRecoveryEnded()) { - socketRecovery.markRecoverySucceeded(); + if (socketRecovery.tryMarkRecoverySucceeded()) { socketRecovery.notifyAll(); } else { VSProtocol.LOGGER.log(Level.WARNING, "Reconnect succeeded but recovery process is already cancelled"); @@ -165,7 +176,7 @@ public void run() { } if (!success) { - if (currentTime > socketRecovery.getDisconnectTs() + reconnectInterval) { + if (currentTime >= socketRecovery.getRecoveryDeadlineTs()) { // At this time socket is discarded on the server side so we should give up now VSProtocol.LOGGER.log(Level.WARNING, "Transport " + socket.getSocketIdStr() + " was not recovered after " + attemptNumber + " attempts (timeout reached)"); transportLost = true; @@ -174,6 +185,7 @@ public void run() { // We failed to recover the connection so we have to disconnect entire transport because we might loss some data synchronized (socketRecovery) { socketRecovery.stopRecoveryAttempt(); + assert !socketRecovery.isRecoverySucceeded(); socketRecovery.markRecoveryFailed(); socketRecovery.notifyAll(); } @@ -208,35 +220,40 @@ protected boolean killSupported() { } private void scheduleReconnectAttempt(long nextAttemptTimestamp) { - GlobalTimer.INSTANCE.schedule(new TimerTask() { + GlobalTimer.INSTANCE.schedule(new TimerRunner() { @Override - public void run() { + public void runInternal() { reconnector.submit(); } }, new Date(nextAttemptTimestamp)); } - @org.jetbrains.annotations.VisibleForTesting // Should by used in tests ONLY. TODO: Delete? + @VisibleForTesting // Should by used in tests ONLY. TODO: Delete? public VSClient (String host, int port, String ownerID) throws IOException { this(host, port, ownerID, false, ContextContainer.getContextContainerForClientTests()); } - @VisibleForTesting - // Should by used in tests ONLY. TODO: Create a factory method with name like "createClientForTests" + @VisibleForTesting // Should by used in tests ONLY. TODO: Create a factory method with name like "createClientForTests" public VSClient (String host, int port) throws IOException { this(host, port, null, false, ContextContainer.getContextContainerForClientTests()); } - public VSClient(String host, int port, String ownerID, boolean enableSSL, ContextContainer contextContainer) throws IOException { + public VSClient(String host, int port, @Nullable String ownerID, boolean enableSSL, ContextContainer contextContainer) throws IOException { this(host, port, ownerID, enableSSL, SSL_TERMINATION, contextContainer); } - public VSClient(String host, int port, String ownerID, boolean enableSSL, boolean sslTermination, + public VSClient(String host, int port, @Nullable String ownerID, boolean enableSSL, boolean sslTermination, ContextContainer contextContainer) throws IOException { + this(host, port, ownerID, enableSSL, contextContainer, withSslTermination(sslTermination)); + } + + @ApiStatus.Experimental + public VSClient(String host, int port, @Nullable String ownerID, boolean enableSSL, + ContextContainer contextContainer, VSClientOptions clientOptions) throws IOException { this.host = host; this.port = port; this.enableSSL = enableSSL; - this.sslTermination = sslTermination; + this.sslTermination = clientOptions.isSslTermination(); this.contextContainer = contextContainer; this.reconnector = createReconnectorTask(contextContainer.getQuickExecutor()); @@ -244,6 +261,12 @@ public VSClient(String host, int port, String ownerID, boolean enableSSL, boolea this.clientId = new GUID().toStringWithPrefix (InetAddress.getLocalHost().getHostAddress() + ":"); else this.clientId = new GUID().toStringWithPrefix(InetAddress.getLocalHost().getHostAddress() + ":" + ownerID + ":"); + + this.transportReconnectAttemptInterval = clientOptions.getTransportReconnectAttemptInterval(); + this.soTimeout = clientOptions.getHandshakeSocketTimeout(); + this.timeout = clientOptions.getSocketConnectTimeout(); + this.socketSendBufferSize = clientOptions.getSocketSendBufferSize(); + this.socketReceiveBufferSize = clientOptions.getSocketReceiveBufferSize(); } public void setClientAddress(String address, String ownerID) { @@ -302,14 +325,35 @@ public int getReconnectInterval() { return reconnectInterval; } + /** + * Checks if client is connected. + * Will not wait but will return true even if reconnecting and there is no immediately available transports. + * + *

It returns true during reconnecting phase because it would be inconsistent to return false, + * considering that reconnecting state does not trigger "disconnected" event. + * + *

In most cases you should use {@link #tryGetConnectionStatus()} instead. + * + * @return true if connected or reconnecting, false otherwise + */ public boolean isConnected() { - return dispatcher != null && dispatcher.hasAvailableTransport(); + return dispatcher != null && dispatcher.isConnectedOrReconnecting(); + } + + /** + * Checks if client is fully connected right now. + * Will not wait and will return false if reconnecting. + * + * @return true if connected and NOT reconnecting, false otherwise + */ + public boolean isConnectedAndNotReconnecting() { + return dispatcher != null && dispatcher.isConnectedAndNotReconnecting(); } /** * Return true, if it has CONNECTED state. - * Return false, if it has DISCONNECTED state. - * Otherwise, waits at least {@link #reconnectInterval} until status gets CONNECTED or DISCONNECTED. + * Return false, if it has DISCONNECTED/DISCONNECTING state. + * Otherwise, waits until status gets CONNECTED or DISCONNECTED. * * @return true if connected, false if disconnected */ @@ -376,12 +420,11 @@ private Socket setupSocket() throws IOException { socket.setTcpNoDelay(true); // Sets socket buffer sizes. - // Please note that later socket also will be additionally configured in VSocketImpl.setUpSocket() method. - // However, that happens only after socket gets connected. // It's important to configure receive buffer size before connection is established // to allow it to use TCP window size greater than 64kb. // That's why we have to do that here. - VSocketImpl.configureBufferSizes(socket); + socket.setReceiveBufferSize(this.socketReceiveBufferSize); + socket.setSendBufferSize(this.socketSendBufferSize); // Connect socket.connect(socketAddress, timeout); @@ -578,7 +621,7 @@ VSocket openTransport () throws IOException { assert numBytesRecieved == 0; // new connections should have = 0; this.serverCompression = Enum.valueOf(VSCompression.class, compression); } - + ok = true; } finally { if (!ok) @@ -614,6 +657,7 @@ else if (serverCompression == VSCompression.ON) try { vsc.sendConnect (); } catch (InterruptedException x) { + Thread.currentThread().interrupt(); throw new InterruptedIOException (); } @@ -633,7 +677,11 @@ private void close(boolean waitForChannelsToFinish) { VSDispatcher d = dispatcher; // If dispatcher is null, then we already disconnected or even never were connected. - triggerDisconnectEvent = d != null; + // If dispatcher is in shutdown state, then disconnect event already was triggered. + // Note that this check does not give 100% guarantee that disconnect event will be triggered no more than once + // because of race between checking isShutdownState() and calling d.setStateListener(null). + // However, in practice this should be sufficient. + triggerDisconnectEvent = d != null && !d.isShutdownState(); if (d != null) { d.setStateListener(null); @@ -645,9 +693,14 @@ private void close(boolean waitForChannelsToFinish) { dispatcher = null; } - - // https://gitlab.deltixhub.com/Deltix/QuantServer/QuantServer/-/issues/1269 // Trigger a disconnect event, so any disconnect listeners can be notified. + // https://gitlab.deltixhub.com/Deltix/QuantServer/QuantServer/-/issues/1269 + // However, this also results that onDisconnect event will be triggered even if no "unexpected disconnect" actually happened. + // So while VSDispatcher does not trigger disconnect event if it shut down gracefully, VSClient.close() will still trigger it. + // TODO: Decide if we want to call .onDisconnected() in case of normal shutdown. + // TODO: This should be reviewed after TickDBClient refactor. We may want to completely remove this call + // as updated VSDispatcher already triggers disconnect event on unexpected disconnects + // and state change that is caused by TickDBClient closing the connection may be handled in TickDBClient itself. if (triggerDisconnectEvent) { DisconnectEventListener listenerRef = listener; if (listenerRef != null) { @@ -661,7 +714,7 @@ public void setDisconnectedListener(DisconnectEventListener } @Override - boolean onTransportStopped(VSocketRecoveryInfo recoveryInfo) { + boolean onTransportRecoveryStart(VSocketRecoveryInfo recoveryInfo) { if (dispatcher != null) { // TODO: Ensure that we can't get duplicate instance of socket in the broken list broken.addFirst(recoveryInfo); @@ -674,7 +727,7 @@ boolean onTransportStopped(VSocketRecoveryInfo recoveryInfo) { } @Override - boolean onTransportBroken(VSocketRecoveryInfo recoveryInfo) { + boolean onTransportRecoveryStop(VSocketRecoveryInfo recoveryInfo) { try { synchronized (recoveryInfo) { while (recoveryInfo.isRecoveryAttemptInProgress()) { @@ -697,12 +750,13 @@ boolean onTransportBroken(VSocketRecoveryInfo recoveryInfo) { return recoveryFailed || (!removed && !recoveryInfo.isRecoverySucceeded()); } } catch (InterruptedException e) { + Thread.currentThread().interrupt(); return true; } } @Override - void onReconnected() { + void onConnected() { if (listener != null) listener.onReconnected(); } @@ -730,8 +784,6 @@ public void disposed (VSDispatcher d) { contextContainer.getQuickExecutor().shutdownInstance(); dispatcher = null; - - onDisconnected(); } } } @@ -748,4 +800,15 @@ public int getSSLPort() { public String toString () { return ("VSClient (" + host + ":" + port + ")"); } + + /** Returns default client options based on system properties, with "sslTermination" set to provided value */ + private static VSClientOptions withSslTermination(boolean sslTermination) { + if (sslTermination == DEFAULT_CLIENT_OPTIONS.isSslTermination()) { + return DEFAULT_CLIENT_OPTIONS; + } else { + VSClientOptions result = new VSClientOptions(); + result.setSslTermination(sslTermination); + return result; + } + } } diff --git a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSClientOptions.java b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSClientOptions.java new file mode 100644 index 00000000..f69e71f5 --- /dev/null +++ b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSClientOptions.java @@ -0,0 +1,82 @@ +package com.epam.deltix.util.vsocket; + +/* + * Copyright 2026 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +import org.jetbrains.annotations.ApiStatus; + +@ApiStatus.Experimental +public class VSClientOptions { + // Init fields with default values + + private int socketSendBufferSize = VSocketImpl.SOCKET_SEND_BUFFER_SIZE; + private int socketReceiveBufferSize = VSocketImpl.SOCKET_RECEIVE_BUFFER_SIZE; + private boolean sslTermination = VSClient.SSL_TERMINATION; + private int transportReconnectAttemptInterval = Integer.getInteger("TimeBase.network.VSClient.transportReconnectAttemptInterval", 1000); + + // Controls socket timeout ("soTimeout") during the initial handshake. After handshake is complete, soTimeout is always "0" (infinite). + private int handshakeSocketTimeout = Integer.getInteger("TimeBase.network.VSClient.soTimeout", 5000); + private int socketConnectTimeout = Integer.getInteger("TimeBase.network.VSClient.timeout", 5000); + + + public int getSocketReceiveBufferSize() { + return socketReceiveBufferSize; + } + + public void setSocketReceiveBufferSize(int socketReceiveBufferSize) { + this.socketReceiveBufferSize = socketReceiveBufferSize; + } + + public int getSocketSendBufferSize() { + return socketSendBufferSize; + } + + public void setSocketSendBufferSize(int socketSendBufferSize) { + this.socketSendBufferSize = socketSendBufferSize; + } + + public int getHandshakeSocketTimeout() { + return handshakeSocketTimeout; + } + + public void setHandshakeSocketTimeout(int handshakeSocketTimeout) { + this.handshakeSocketTimeout = handshakeSocketTimeout; + } + + public boolean isSslTermination() { + return sslTermination; + } + + public void setSslTermination(boolean sslTermination) { + this.sslTermination = sslTermination; + } + + public int getSocketConnectTimeout() { + return socketConnectTimeout; + } + + public void setSocketConnectTimeout(int socketConnectTimeout) { + this.socketConnectTimeout = socketConnectTimeout; + } + + public int getTransportReconnectAttemptInterval() { + return transportReconnectAttemptInterval; + } + + public void setTransportReconnectAttemptInterval(int transportReconnectAttemptInterval) { + this.transportReconnectAttemptInterval = transportReconnectAttemptInterval; + } +} diff --git a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSDispatcher.java b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSDispatcher.java index 32f599e4..c3082a66 100644 --- a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSDispatcher.java +++ b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSDispatcher.java @@ -27,14 +27,20 @@ import com.epam.deltix.util.time.TimerRunner; import com.epam.deltix.util.memory.DataExchangeUtils; import net.jcip.annotations.GuardedBy; +import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.VisibleForTesting; import java.io.EOFException; import java.io.IOException; import java.net.SocketException; import java.net.SocketTimeoutException; import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Phaser; import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; /** @@ -51,7 +57,14 @@ public final class VSDispatcher implements Disposable { @GuardedBy("transportChannels") private final ObjectHashSet transportChannels = - new ObjectHashSet<> (); + new ObjectHashSet<> (); + + /* + Limit for the number of open channels to prevent potential DDoS attacks or user errors. + By default, there is no limit (-1 value) + */ + + private int channelsLimit = -1; /** * Set to non-null value during transport channel recovery. @@ -62,33 +75,52 @@ public final class VSDispatcher implements Disposable { @GuardedBy("freeChannels") // TODO: Replace by Deque private final Stack freeChannels = - new Stack <> (); + new Stack <> (); private final ArrayList channels = new ArrayList <> (10); //private volatile boolean hasAvailableTransport = false; - + volatile VSConnectionListener connectionListener = null; volatile ConnectionStateListener stateListener; private int activeChannels; - private int reconnectInterval; + private int lingerInterval; // How long Dispatcher will wait for reconnection to happen private String address; private final String clientAddress; private String applicationID; - // State of Dispatcher on remote side + // State of Dispatcher on remote side private volatile boolean remoteConnected = true; private volatile long throughput = 0; private volatile long totalBytes = 0; // number of bytes sent private final EMA average = new EMA(1000 * 60); // 1 minute - private volatile VSDispatcherState state = VSDispatcherState.DISCONNECTED; + /** + * State transitions: + *

+ */ + private final AtomicReference state = new AtomicReference<>(VSDispatcherState.INITIAL); + + // Latch that gets counted down when dispatcher is fully closed + private final CountDownLatch closeLatch = new CountDownLatch(1); + + // Works both as a counter of recovering transport channels + // and as a barrier to wait until all recovering transports finish recovering. + private final Phaser recoveringTransports = new Phaser() { + @Override + protected boolean onAdvance(int phase, int registeredParties) { + return false; // never terminate automatically + } + }; - // Set to "true" once all operations related to closing the dispatcher are completed, - // just before calling notifyListeners() - private final AtomicBoolean disposed = new AtomicBoolean(false); private TimerTask flusher = new TimerRunner() { private VSChannelImpl[] list = new VSChannelImpl[10]; @@ -98,7 +130,7 @@ public final class VSDispatcher implements Disposable { @Override protected void runInternal() { - int size = 0; + int size; synchronized (channels) { if ((size = channels.size()) > 0) list = channels.toArray(list); @@ -120,7 +152,7 @@ protected void runInternal() { } catch (ConnectionAbortedException e) { VSProtocol.LOGGER.log (Level.WARNING, "Client unexpectedly drop connection. Remote address: " + channel.getRemoteAddress()); } catch (Exception e) { - VSProtocol.LOGGER.log (Level.WARNING, "Exception while flushing data. Remote address: " + channel.getRemoteAddress(), e); + VSProtocol.LOGGER.log (Level.WARNING, "Exception while flushing data. Remote address: " + channel.getRemoteAddress(), e); } } @@ -154,8 +186,7 @@ protected void runInternal() { private final boolean isClient; private volatile int index = 0; - private final HashSet listeners = - new HashSet (); + private final HashSet> listeners = new HashSet<> (); /** * Constructs a dispatcher instance for the specified client. @@ -179,6 +210,26 @@ public VSDispatcher(String clientId, boolean isClient, ContextContainer contextC .build(); } + /* + Gets limit for the active channels. Default is -1, meaning no limits. + */ + + public int getChannelsLimit() { + return channelsLimit; + } + + + /* + Sets limit for the active channels. -1 means no limits. + */ + + public void setChannelsLimit(int limit) { + if (limit == 0 || limit < -1) + throw new IllegalArgumentException("Wrong channels limit: " + limit); + + this.channelsLimit = limit; + } + /** * Return current peak throughput (bytes per second) * @return number of bytes per second @@ -196,7 +247,7 @@ public double getAverageThroughput() { } public int getReconnectInterval() { - return reconnectInterval; + return lingerInterval; } public String getApplicationID() { @@ -216,7 +267,7 @@ public void setApplicationID(String applicationID) { } public void setLingerInterval(int reconnectInterval) { - this.reconnectInterval = reconnectInterval; + this.lingerInterval = reconnectInterval; } public String getClientId () { @@ -239,25 +290,33 @@ public boolean hasTransportChannels() { } } - public boolean hasAvailableTransport() { - return state == VSDispatcherState.CONNECTED; + public boolean isConnectedOrReconnecting() { + VSDispatcherState value = state.get(); + return value == VSDispatcherState.CONNECTED || value == VSDispatcherState.RECONNECTING; + } + + public boolean isConnectedAndNotReconnecting() { + VSDispatcherState value = state.get(); + return value == VSDispatcherState.CONNECTED; } public void addTransportChannel (VSocket socket) - throws IOException + throws IOException { - boolean hasTransport = state == VSDispatcherState.CONNECTED; + VSDispatcherState currentState = state.get(); + if (currentState == VSDispatcherState.DISCONNECTING || currentState == VSDispatcherState.DISCONNECTED) { + VSProtocol.LOGGER.log (Level.WARNING, "Attempt to add transport channel while dispatcher is disconnecting. Remote address: " + socket.getRemoteAddress() + ". Dispatcher: " + this); + } VSTransportChannel tc = new VSTransportChannel(this, socket, transportChannelThreadFactory); tc.checkedOut = true; // Initially this channel is not in "freeChannels" so it is effectively "checked out" - // set that we have transport before starting transport channel thread - if (!hasTransport) - state = VSDispatcherState.CONNECTED; + // Is that fist transport channel? + boolean fistConnected = state.compareAndSet(VSDispatcherState.INITIAL, VSDispatcherState.CONNECTED); // start transport tc.start (); - + synchronized (transportChannels) { if (address == null) @@ -269,20 +328,45 @@ public void addTransportChannel (VSocket socket) checkIn(tc); - if (!hasTransport && stateListener != null) - stateListener.onReconnected(); + // This may be triggered only once per dispatcher lifetime + if (fistConnected && stateListener != null) { + stateListener.onConnected(); + } } public void setConnectionListener (VSConnectionListener connectionListener) { this.connectionListener = connectionListener; } - public void setStateListener(ConnectionStateListener stateListener) { + void setStateListener(ConnectionStateListener stateListener) { this.stateListener = stateListener; } + @VisibleForTesting + VSDispatcherState getInternalState() { + return state.get(); + } + + /** + * Executed in the context of transport channel thread (VSTransportChannel.run() method) when error occurs on transport. + * + *

Corresponding transport channel will be closed after this method returns. + * + *

This method is expected to block until logical transport gets recovered or declared unrecoverably broken. + * In case of recovery failure, expected to trigger dispatcher shutdown, as loss of single transport channel + * means loss of data and inconsistent state for client and server. + * + *

Multiple transport channels may be lost concurrently, so this method may be executed concurrently. + * In that case, threads may compete for changing dispatcher state. + */ void transportStopped (VSTransportChannel channel, Throwable ex) { - IOException iex = ex instanceof IOException ? (IOException)ex : null; + if (!(ex instanceof Exception)) { + // This means major failure, possibly OOM or other serious error. + VSProtocol.LOGGER.log(Level.SEVERE, "Critical error on transport channel. Remote address: " + channel.socket.getRemoteAddress() + ". Dispatcher: " + this, ex); + // Just close dispatcher right away + close(); + return; + } Level disconnectLogLevel = ex instanceof EOFException ? Level.FINE : Level.INFO; if (VSProtocol.LOGGER.isLoggable(disconnectLogLevel)) { @@ -290,166 +374,279 @@ void transportStopped (VSTransportChannel channel, Throwa } long startTime = System.currentTimeMillis(); - long endTime = startTime + reconnectInterval; + long endTime = startTime + lingerInterval; + boolean registered = false; // true if we have registered this transport in "recoveringTransports" phaser boolean wasCheckedIn; - synchronized (transportChannels) { - if (transportChannels.isEmpty()) // already closed - return; + try { + synchronized (transportChannels) { + VSDispatcherState currentState = state.get(); + if (currentState == VSDispatcherState.DISCONNECTING || currentState == VSDispatcherState.DISCONNECTED) { + // Dispatcher is already closing or closed, no need to recover transport + return; + } - if (!transportChannels.remove(channel)) // check that channel already removed - return; + if (!transportChannels.remove(channel)) // check if that channel is already removed + return; - synchronized (freeChannels) { - wasCheckedIn = freeChannels.remove(channel); - assert wasCheckedIn == !channel.checkedOut; - freeChannels.notifyAll(); - } + // From this point we consider that we are recovering this transport channel. + + // Counter incremented before state change, + // so that should be impossible to see CONNECTING with 0 recovering transports and still pending recovery attempt. + recoveringTransports.register(); + registered = true; + state.compareAndSet(VSDispatcherState.CONNECTED, VSDispatcherState.RECONNECTING); + + synchronized (freeChannels) { + wasCheckedIn = freeChannels.remove(channel); + assert wasCheckedIn == !channel.checkedOut; + freeChannels.notifyAll(); + } - if (!wasCheckedIn) { - // Try to wait for the channel to become checked in - wasCheckedIn = waitForTransportCheckIn(channel, startTime, endTime); if (!wasCheckedIn) { - if (VSProtocol.LOGGER.isLoggable(Level.INFO)) { - VSProtocol.LOGGER.log(Level.INFO, "Error waiting to reconnect (transport was not checked in)."); + // Try to wait for the channel to become checked in + wasCheckedIn = waitForTransportCheckIn(channel, startTime, endTime); + if (!wasCheckedIn) { + if (VSProtocol.LOGGER.isLoggable(Level.INFO)) { + VSProtocol.LOGGER.log(Level.INFO, "Error waiting to reconnect (transport was not checked in)."); + } } } } + ConnectionStateListener stateListener = this.stateListener; - state = VSDispatcherState.CONNECTING; - } - - boolean transportIsUnrecoverablyBroken = false; + boolean transportIsUnrecoverablyBroken = false; + try { + // trying to recover transport + VSocketRecoveryInfo recoveryInfo = new VSocketRecoveryInfo(channel.socket, endTime); - // trying to recover transport - VSocketRecoveryInfo recoveryInfo = new VSocketRecoveryInfo(channel.socket, startTime); + long now = System.currentTimeMillis(); + if (wasCheckedIn && (now < endTime) && !isShutdownState()) { - long now = System.currentTimeMillis(); - if (wasCheckedIn && (now < endTime)) { + if (stateListener != null) { + if (stateListener.onTransportRecoveryStart(recoveryInfo)) { + transportIsUnrecoverablyBroken = true; + } + } - if (stateListener != null) { - if (stateListener.onTransportStopped(recoveryInfo)) { + if (remoteConnected && !transportIsUnrecoverablyBroken) { + // System.out.println("WAITED: remoteConnected=" + remoteConnected + " transportIsUnrecoverablyBroken=" + transportIsUnrecoverablyBroken); + try { + // We loop here waiting for recovery to complete or timeout to expire or dispatcher to be closed. + synchronized (recoveryInfo) { + long timeToWait; + while ((timeToWait = endTime - now) > 0 && recoveryInfo.isWaitingForRecovery() && remoteConnected && !isShutdownState()) { + recoveryInfo.wait(timeToWait); + if (recoveryInfo.isWaitingForRecovery() && remoteConnected) { + now = System.currentTimeMillis(); + } + } + if (recoveryInfo.isRecoveryFailed()) { + transportIsUnrecoverablyBroken = true; + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + if (VSProtocol.LOGGER.isLoggable(Level.FINE)) + VSProtocol.LOGGER.log(Level.FINE, "Error waiting to reconnect.", e); + } + } else { + //System.out.println("NOT WAITED: remoteConnected=" + remoteConnected + " transportIsUnrecoverablyBroken=" + transportIsUnrecoverablyBroken); + } + } else { transportIsUnrecoverablyBroken = true; + if (VSProtocol.LOGGER.isLoggable(Level.FINE)) { + VSProtocol.LOGGER.log(Level.FINE, "Cancelled recovery of failed connection because of timeout on waiting for check-in from other thread. Remote address: " + getRemoteAddress()); + } } - } - if (remoteConnected && !transportIsUnrecoverablyBroken) { - // System.out.println("WAITED: remoteConnected=" + remoteConnected + " transportIsUnrecoverablyBroken=" + transportIsUnrecoverablyBroken); - try { - // Try to wait for connection restore - state = VSDispatcherState.CONNECTING; - - synchronized (recoveryInfo) { - long timeToWait; - while ((timeToWait = endTime - now) > 0 && recoveryInfo.isWaitingForRecovery() && remoteConnected) { - recoveryInfo.wait(timeToWait); - if (recoveryInfo.isWaitingForRecovery() && remoteConnected) { - now = System.currentTimeMillis(); - } - } - if (recoveryInfo.isRecoveryFailed()) { - transportIsUnrecoverablyBroken = true; + // In general, VSDispatcher don't have to shut down if transport recovery fails, + // because other transport channels may remain functional. + // However, in our current design, loss of single transport channel means loss of data + // and inconsistent state for client and server, so we have to shut down the dispatcher. + // The decision to shut down the dispatcher is delegated to the state listener. + if (stateListener != null) { + if (stateListener.onTransportRecoveryStop(recoveryInfo)) { + transportIsUnrecoverablyBroken = true; + } + } + } finally { + if (transportIsUnrecoverablyBroken || isShutdownState()) { + // We lost this transport channel and were unable to recover it (because of explicit error, timeout or triggered shutdown state). + // This means it is not possible to recover from this state, and we have to properly close the dispatcher. + // We need to close all remaining connections and explicitly notify user about that. + + // Record a copy of state listener reference before updating state because it may be changed concurrently. + // If save "stateListener" before state update, then we can be sure that + // if we had non-null listener before state update, then we will have non-null listener for thread that gets "triggerDisconnectedEvent". + var savedStateListener = this.stateListener; + + // Try to set state to DISCONNECTING, before decrementing recoveringTransports counter, + // so other thread will not switch into CONNECTED state if this was the last recovering transport. + + VSDispatcherState stateBeforeUpdate = state.getAndUpdate(prevState -> { + switch (prevState) { + case CONNECTED: + // Should not happen + return VSDispatcherState.DISCONNECTING; + case RECONNECTING: + return VSDispatcherState.DISCONNECTING; + case DISCONNECTING: + return prevState; // remain in DISCONNECTING + case DISCONNECTED: + return prevState; // remain in DISCONNECTED + default: + throw new IllegalStateException("Unexpected dispatcher state: " + prevState); } + }); + // Disconnected event should be triggered only if we changed the state. + // So that event should be triggered only once per dispatcher lifetime. + // Also, it disables trigger of onDisconnected event if dispatcher is closed normally via direct call to close(). + boolean triggerDisconnectedEvent = stateBeforeUpdate == VSDispatcherState.CONNECTED || stateBeforeUpdate == VSDispatcherState.RECONNECTING; + + registered = false; + recoveringTransports.arriveAndDeregister(); + + processChannelRecoveryFailure(ex, triggerDisconnectedEvent, savedStateListener); + } else { + // Successfully recovered this transport channel + registered = false; + recoveringTransports.arriveAndDeregister(); + int remaining = recoveringTransports.getUnarrivedParties(); + if (!recoveringTransports.isTerminated() && remaining == 0) { + // All lost transport channels are recovered, try to update state to CONNECTED + state.getAndUpdate(prev -> { + if (prev == VSDispatcherState.RECONNECTING) { + return VSDispatcherState.CONNECTED; + } else { + return prev; // Keep state unchanged + } + }); } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - if (VSProtocol.LOGGER.isLoggable(Level.FINE)) - VSProtocol.LOGGER.log(Level.FINE, "Error waiting to reconnect.", e); } - } else { - //System.out.println("NOT WAITED: remoteConnected=" + remoteConnected + " transportIsUnrecoverablyBroken=" + transportIsUnrecoverablyBroken); } - } else { - transportIsUnrecoverablyBroken = true; - if (VSProtocol.LOGGER.isLoggable(Level.FINE)) { - VSProtocol.LOGGER.log(Level.FINE, "Cancelled recovery of failed connection because of timeout on waiting for check-in from other thread. Remote address: " + getRemoteAddress()); - } - } - if (stateListener != null) { - if (stateListener.onTransportBroken(recoveryInfo)) { - transportIsUnrecoverablyBroken = true; + } finally { + if (registered) { + // In case of any unexpected error, ensure that we release the counter + recoveringTransports.arriveAndDeregister(); } } + } - if (transportIsUnrecoverablyBroken) { - // We lost this transport channel and were unable to recover it. - // This means that we lost at least some data and can't recover from this state. - // We need to close all remaining connections and explicitly notify use about that. - - boolean wasConnected = remoteConnected; - - // mark that we lost transport completely - state = VSDispatcherState.DISCONNECTED; + /** + * Triggered when transport channel recovery has failed and dispatcher must be disconnected. + * @param triggerClose if true, then current thread is the one that first detected unrecoverable transport failure and responsible for shutdown + */ + private void processChannelRecoveryFailure(Throwable ex, boolean triggerClose, @Nullable ConnectionStateListener stateListenerCopy) { + boolean wasConnected = remoteConnected; - // notify all waiting for transport that connection is lost - onRemoteClosed(); + // notify all waiting for transport that connection is lost + onRemoteClosed(); - if (ex instanceof SocketException || ex instanceof EOFException || ex instanceof SocketTimeoutException) { - if (VSProtocol.LOGGER.isLoggable(Level.FINE)) - VSProtocol.LOGGER.log(Level.FINE, "Exception on transport channel. Remote address: " + getRemoteAddress(), ex); - } else { - VSProtocol.LOGGER.log(Level.SEVERE, "Exception on transport channel. Remote address: " + getRemoteAddress(), ex); - } + if (ex instanceof SocketException || ex instanceof EOFException || ex instanceof SocketTimeoutException) { + if (VSProtocol.LOGGER.isLoggable(Level.FINE)) + VSProtocol.LOGGER.log(Level.FINE, "Exception on transport channel. Remote address: " + getRemoteAddress(), ex); + } else { + VSProtocol.LOGGER.log(Level.SEVERE, "Exception on transport channel. Remote address: " + getRemoteAddress(), ex); + } - if (wasConnected) { - VSProtocol.LOGGER.log(Level.WARNING, "Disconnecting due to unrecoverable transport channel loss. Remote address: " + getRemoteAddress(), ex); - } else { - VSProtocol.LOGGER.log(Level.FINER, "Disconnecting (re-triggered) due to unrecoverable transport channel loss. Remote address: " + getRemoteAddress(), ex); - } + if (wasConnected) { + VSProtocol.LOGGER.log(Level.WARNING, "Disconnecting due to unrecoverable transport channel loss. Remote address: " + getRemoteAddress(), ex); + } else { + VSProtocol.LOGGER.log(Level.FINER, "Disconnecting (re-triggered) due to unrecoverable transport channel loss. Remote address: " + getRemoteAddress(), ex); + } - // and then notify all channels that we lost transport - synchronized (channels) { - for (VSChannelImpl vsChannel : channels) - if (vsChannel != null) - vsChannel.onDisconnected(iex); + // and then notify all channels that we lost transport + IOException iex = ex instanceof IOException ? (IOException)ex : null; + synchronized (channels) { + for (VSChannelImpl vsChannel : channels) { + if (vsChannel != null) { + // TODO: Review. Calling onDisconnected while holding "channels" lock may lead to deadlocks + vsChannel.onDisconnected(iex); + } } + } + if (triggerClose) { // notify state listener that connections lost - if (stateListener != null) - stateListener.onDisconnected(); + if (stateListenerCopy != null) { + if (VSProtocol.LOGGER.isLoggable(Level.FINER)) { + VSProtocol.LOGGER.log(Level.FINER, "Notifying state listener about disconnection. Remote address: " + getRemoteAddress()); + } + stateListenerCopy.onDisconnected(); + } close(); } else { - state = VSDispatcherState.CONNECTED; + // If this thread is not responsible for closing dispatcher, + // just wait until dispatcher gets closed by other thread. + boolean success; + try { + success = closeLatch.await(lingerInterval + 1_000, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Failed waiting for dispatcher to close after transport recovery failure.", e); + } + if (!success) { + VSProtocol.LOGGER.log(Level.WARNING, "Timeout waiting for dispatcher to close after transport recovery failure. Remote address: " + getRemoteAddress()); + // No other thread closed the dispatcher in a timely manner, close it ourselves. Even if it may break order between .onDisconnected() and .disposed() events. + close(); + } } } + boolean isShutdownState() { + VSDispatcherState value = state.get(); + return value == VSDispatcherState.DISCONNECTING || value == VSDispatcherState.DISCONNECTED; + } + /** * Return true, if it has CONNECTED state. - * Return false, if it has DISCONNECTED state. - * Otherwise, waits at least {@link #reconnectInterval} until status gets CONNECTED or DISCONNECTED. + * Return false, if it has INITIAL, DISCONNECTED or DISCONNECTING state. + * Otherwise, waits at least {@link #lingerInterval} until status gets CONNECTED or DISCONNECTED. * * @return true if connected, false if disconnected */ public boolean tryGetConnectionStatus() { - - // set timeout > reconnectInterval - int timeout = reconnectInterval * 2; - - long timeLimit = TimeKeeper.currentTime + timeout; - if (timeLimit < 0) // overflow check - timeLimit = Long.MAX_VALUE; - - long period = Math.min(timeout, 1000); - try { - while (TimeKeeper.currentTime < timeLimit) { - if (state == VSDispatcherState.CONNECTED) - return true; - else if (state == VSDispatcherState.DISCONNECTED) + // Get current phase before checking state + int phase = recoveringTransports.getPhase(); + + // Read current status + switch (state.get()) { + case CONNECTED: + return true; + case INITIAL: + case DISCONNECTED: + case DISCONNECTING: + return false; + case RECONNECTING: + // Wait below + } + + // What for the phase to change + recoveringTransports.awaitAdvance(phase); + + // Check new status + switch (state.get()) { + case CONNECTED: + return true; + case INITIAL: + case DISCONNECTED: + case DISCONNECTING: + return false; + case RECONNECTING: + default: { + // Special case: we are still in reconnection state, even after phase advanced. + if (recoveringTransports.isTerminated()) { return false; - Thread.sleep(period); + } + // It may be possible that the waiting transport count was just decremented to zero + // but state was not updated yet. Check that. + return recoveringTransports.getUnarrivedParties() == 0; } - } catch (InterruptedException e) { } - - if (state == VSDispatcherState.CONNECTED) - return true; - else if (state == VSDispatcherState.DISCONNECTED) - return false; - - return false; } /** @@ -466,7 +663,7 @@ private boolean waitForTransportCheckIn(VSTransportChannel channel, long now, lo boolean checkedIn = false; try { - while (now < endTime && !checkedIn && !disposed.get()) { + while (now < endTime && !checkedIn && !isShutdownState()) { transportChannels.wait(endTime - now); now = System.currentTimeMillis(); synchronized (freeChannels) { @@ -493,7 +690,7 @@ public void closeTransport() throws IOException, Interru } } - public void checkIn (VSTransportChannel tc) { + void checkIn (VSTransportChannel tc) { synchronized (transportChannels) { if (transportChannels.contains(tc)) { synchronized (freeChannels) { @@ -502,6 +699,7 @@ public void checkIn (VSTransportChannel tc) { freeChannels.notify(); } } else { + // This was removed from dispatcher, possibly channel recovery in progress. synchronized (freeChannels) { tc.checkedOut = false; } @@ -517,8 +715,9 @@ VSTransportChannel checkOut () { synchronized (freeChannels) { for (;;) { - if (state != VSDispatcherState.CONNECTED && !remoteConnected) + if (isShutdownState() && !remoteConnected) { throw new ConnectionAbortedException("Connection aborted from remote side [" + getRemoteAddress() + "]"); + } if (!freeChannels.isEmpty ()) { VSTransportChannel channel = freeChannels.pop(); @@ -565,7 +764,7 @@ public void close(boolean wait) { void onRemoteClosed() { remoteConnected = false; - + // notify all waiting threads in checkOut() synchronized (freeChannels) { freeChannels.notifyAll(); @@ -573,9 +772,11 @@ void onRemoteClosed() { } private void sendClosing() { - if (!remoteConnected || state != VSDispatcherState.CONNECTED) + // TODO: In theory we can try to check if there are any free transport channels and try to use them to send + // the close message. But in practice, if we are not connected anymore, then we are not very likely to succeed. + if (!remoteConnected || state.get() != VSDispatcherState.CONNECTED) return; - + VSTransportChannel channel = null; try { byte[] buffer = new byte[2]; @@ -590,10 +791,22 @@ private void sendClosing() { } } + @Override public void close () { sendClosing(); - + + // Change state to DISCONNECTING if it was not DISCONNECTED already. + // This state change disables triggering of stateListener.onDisconnected() on transport channel error. + // So normal dispatcher.close() will not trigger onDisconnected() event. + state.getAndUpdate(prevState -> { + if (prevState == VSDispatcherState.DISCONNECTED) { + return VSDispatcherState.DISCONNECTED; + } else { + return VSDispatcherState.DISCONNECTING; + } + }); + synchronized (transportChannels) { for (VSTransportChannel tc : transportChannels) Util.close (tc); @@ -602,7 +815,6 @@ public void close () { transportChannels.notify(); } - state = VSDispatcherState.DISCONNECTED; remoteConnected = false; // disable free channels to prevent locking on code below @@ -631,8 +843,15 @@ public void close () { t.cancel(); // stop timer thread timer = null; // for GC - if (disposed.compareAndSet(false, true)) - notifyListeners(); + VSDispatcherState prevState = state.getAndUpdate(x -> VSDispatcherState.DISCONNECTED); + if (prevState != VSDispatcherState.DISCONNECTED) { + // This may be triggered only once per dispatcher lifetime + notifyDisposedEventListeners(); + } + + recoveringTransports.forceTermination(); + + closeLatch.countDown(); } VSChannelImpl newChannel (int inCapacity, int outCapacity, boolean compressed) { @@ -643,6 +862,9 @@ VSChannelImpl newChannel (int inCapacity, int outCapacity, boolean } synchronized (channels) { + if (activeChannels >= channelsLimit && channelsLimit > 0) + throw new IllegalStateException("Attempt to create new channel above channels limit = " + channelsLimit); + int localId = channels.indexOf (null); boolean extend = localId < 0; @@ -654,7 +876,7 @@ VSChannelImpl newChannel (int inCapacity, int outCapacity, boolean index += isClient ? -1 : 1; vsc = new VSChannelImpl (this, inCapacity, outCapacity, compressed, localId, index, contextContainer); - + if (extend) { channels.add (vsc); } else { @@ -667,7 +889,7 @@ VSChannelImpl newChannel (int inCapacity, int outCapacity, boolean } return (vsc); - } + } public VSChannel [] getVirtualChannels () { synchronized (channels) { @@ -703,6 +925,7 @@ long getLatency() { try { return (tc = checkOut()).getLatency(); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); return 0; } catch (ConnectionAbortedException e) { return 0; @@ -727,21 +950,21 @@ void channelClosed (VSChannelImpl vsc) { } } - public void addDisposableListener(DisposableListener listener) { + public void addDisposableListener(DisposableListener listener) { synchronized (listeners) { - if (!listeners.contains(listener)) - listeners.add(listener); + listeners.add(listener); } } - public void removeDisposableListener(DisposableListener listener) { + public void removeDisposableListener(DisposableListener listener) { synchronized (listeners) { listeners.remove(listener); } } - private DisposableListener[] getListeners() { - DisposableListener[] list; + @SuppressWarnings("unchecked") + private DisposableListener[] getListeners() { + DisposableListener[] list; synchronized (listeners) { //noinspection ToArrayCallWithZeroLengthArrayArgument @@ -751,15 +974,15 @@ private DisposableListener[] getListeners() { return list; } - @SuppressWarnings("unchecked") - private void notifyListeners() { - DisposableListener[] list = getListeners(); + private void notifyDisposedEventListeners() { + DisposableListener[] list = getListeners(); - for (DisposableListener aList : list) + for (var aList : list) { aList.disposed(this); + } } - public QuickExecutor getQuickExecutor() { + public QuickExecutor getQuickExecutor() { return contextContainer.getQuickExecutor(); } @@ -767,4 +990,4 @@ public QuickExecutor getQuickExecutor() { public String toString() { return getClass().getSimpleName() + "@" + Integer.toHexString(hashCode()) + " for clientId='" + clientId; } -} \ No newline at end of file +} diff --git a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSDispatcherState.java b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSDispatcherState.java index cccd66dd..53267eee 100644 --- a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSDispatcherState.java +++ b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSDispatcherState.java @@ -1,7 +1,9 @@ package com.epam.deltix.util.vsocket; -public enum VSDispatcherState { - CONNECTED, - CONNECTING, - DISCONNECTED -} +enum VSDispatcherState { + INITIAL, // No connection attempt made yet + CONNECTED, // At least one connection established, no transports in "recovery" state + RECONNECTING, // At least one transport in "recovery" state, trying to reconnect + DISCONNECTING, // Disconnect process initiated, waiting for all shutdown-related actions to complete + DISCONNECTED // Can be set only at the end of VSDispatcher.close() method +} \ No newline at end of file diff --git a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSServer.java b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSServer.java index be7675b6..2912a686 100644 --- a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSServer.java +++ b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSServer.java @@ -19,6 +19,7 @@ import com.epam.deltix.util.ContextContainer; import com.epam.deltix.util.concurrent.QuickExecutor; import com.epam.deltix.util.io.IOUtil; +import org.jetbrains.annotations.VisibleForTesting; import java.net.*; import java.io.*; @@ -92,6 +93,11 @@ public void setSoTimeout (int readTimeout) throws SocketExceptio serverSocket.setSoTimeout(readTimeout); } + @VisibleForTesting + public void setTransportsLimit(short transportsLimit) { + this.framework.setTransportsLimit(transportsLimit); + } + @Override public void run () { Socket s = null; @@ -126,6 +132,11 @@ public void run () { IOUtil.close (serverSocket); } + @VisibleForTesting + VSDispatcher[] getDispatchers() { + return framework.getDispatchers(); + } + public void close () { IOUtil.close (serverSocket); interrupt (); diff --git a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSServerFramework.java b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSServerFramework.java index 2548e88f..2f66dc58 100644 --- a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSServerFramework.java +++ b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSServerFramework.java @@ -42,24 +42,32 @@ public class VSServerFramework implements ConnectionHandshakeHandler, Disposable public final static int MAX_CONNECTIONS = 100; public final static short MAX_SOCKETS_PER_CONNECTION = 8; + public final static short MAX_CHANNELS_PER_CONNECTION = 1000; private final Map dispatchers = - new HashMap <> (); + new HashMap <> (); private final QuickExecutor executor; - private final ContextContainer contextContainer; + private final ContextContainer contextContainer; private volatile VSConnectionListener connectionListener; + + // max number of connections (VSDispatchers) private final int connectionsLimit; - private final short transportsLimit; + // max number of sockets per connection (VSTransportChannels) + private short transportsLimit; + // max number channels per connection (VSChannels) + private final int channelsLimit; private final long time; - private final int reconnectInterval; + private final int lingerInterval; private final VSCompression compression; private TLSContext tlsContext; private TransportType transportType = TransportType.SOCKET_TCP; + + private final DBConnectionAcceptor connectionAcceptor; public static final Comparator comparator = new Comparator () { @@ -70,16 +78,15 @@ public int compare (VSDispatcher o1, VSDispatcher o2) { } }; - public VSServerFramework(QuickExecutor executor, - int reconnectInterval, - VSCompression compression, - int connectionsLimit, - short socketsPerConnection, + public VSServerFramework(QuickExecutor executor, int lingerInterval, + VSCompression compression, int connectionsLimit, + short socketsPerConnection, int channelsPerConnection, ContextContainer contextContainer, DBConnectionAcceptor connectionAcceptor) { + this.channelsLimit = channelsPerConnection; this.connectionAcceptor = connectionAcceptor; this.executor = executor; - this.reconnectInterval = reconnectInterval; + this.lingerInterval = lingerInterval; this.time = System.currentTimeMillis(); this.compression = compression; this.connectionsLimit = connectionsLimit; @@ -88,8 +95,8 @@ public VSServerFramework(QuickExecutor executor, INSTANCE = this; } - public VSServerFramework(QuickExecutor executor, int reconnectInterval, VSCompression compression, ContextContainer contextContainer) { - this(executor, reconnectInterval, compression, MAX_CONNECTIONS, MAX_SOCKETS_PER_CONNECTION, contextContainer, DefaultConnectionAcceptor.INSTANCE); + public VSServerFramework(QuickExecutor executor, int lingerInterval, VSCompression compression, ContextContainer contextContainer) { + this(executor, lingerInterval, compression, MAX_CONNECTIONS, MAX_SOCKETS_PER_CONNECTION, -1, contextContainer, DefaultConnectionAcceptor.INSTANCE); } public QuickExecutor getExecutor () { @@ -163,7 +170,7 @@ public boolean handleHandshake (Socket s) throws IOException { BufferedInputStream bis = new BufferedInputStream(s.getInputStream(), VSocketImpl.INPUT_STREAM_BUFFER_SIZE); return handleHandshake( - SocketConnectionFactory.createConnection(s, bis, s.getOutputStream()) + SocketConnectionFactory.createConnection(s, bis, s.getOutputStream()) ); } @@ -175,7 +182,7 @@ public boolean handleHandshake(Socket s, BufferedInputStream is, OutputStream os s.setKeepAlive(true); return handleHandshake( - SocketConnectionFactory.createConnection(s, is, os) + SocketConnectionFactory.createConnection(s, is, os) ); } @@ -224,10 +231,10 @@ private boolean handleHandshakeInternal (Connection c) throws IOExc if (!isCompatible) { VSProtocol.LOGGER.severe ( - "Connection from " + clientId + " rejected due to incompatible protocol version #" + - clientVersion + " (accepted: " + - MIN_COMPATIBLE_CLIENT_VERSION + " .. " + - MAX_COMPATIBLE_CLIENT_VERSION + ")" + "Connection from " + clientId + " rejected due to incompatible protocol version #" + + clientVersion + " (accepted: " + + MIN_COMPATIBLE_CLIENT_VERSION + " .. " + + MAX_COMPATIBLE_CLIENT_VERSION + ")" ); dout.writeByte (VSProtocol.CONN_RESP_INCOMPATIBLE_CLIENT); @@ -300,7 +307,7 @@ private boolean handleHandshakeInternal (Connection c) throws IOExc dout.writeByte(VSProtocol.CONN_RESP_OK); dout.writeLong(time); - dout.writeInt(reconnectInterval); + dout.writeInt(lingerInterval); dout.writeUTF(compression.toString()); // writing -1 means socket wasn't found @@ -325,9 +332,10 @@ private boolean handleHandshakeInternal (Connection c) throws IOExc synchronized (brokenSocketRecoveryInfo) { brokenSocketRecoveryInfo.stopRecoveryAttempt(); if (success) { - brokenSocketRecoveryInfo.markRecoverySucceeded(); + if (brokenSocketRecoveryInfo.tryMarkRecoverySucceeded()) { + brokenSocketRecoveryInfo.notifyAll(); + } } - brokenSocketRecoveryInfo.notifyAll(); } } } @@ -348,8 +356,9 @@ private Connector process(String clientId) { if (connector == null) { VSDispatcher dispatcher = new VSDispatcher (clientId, false, contextContainer); + dispatcher.setChannelsLimit(channelsLimit); dispatcher.setConnectionListener(connectionListener); - dispatcher.setLingerInterval(reconnectInterval); + dispatcher.setLingerInterval(lingerInterval); dispatcher.addDisposableListener(this); dispatchers.put (clientId, (connector = new Connector(dispatcher, transportsLimit))); } @@ -428,6 +437,10 @@ public void close() { throw new RuntimeException("Legacy version of Aeron IPC is not supported"); } + void setTransportsLimit(short transportsLimit) { + this.transportsLimit = transportsLimit; + } + static class Connector extends ConnectionStateListener implements Closeable { // May contain null values. Null value indicates that transport is still considered active (not stopped). private final IntegerToObjectHashMap stopped = @@ -460,7 +473,7 @@ boolean addTransportChannel(VSocket socket) throws IOException { } @Override - boolean onTransportStopped(VSocketRecoveryInfo recoveryInfo) { + boolean onTransportRecoveryStart(VSocketRecoveryInfo recoveryInfo) { VSocket socket = recoveryInfo.getSocket(); int code = socket.getCode(); @@ -482,7 +495,7 @@ boolean onTransportStopped(VSocketRecoveryInfo recoveryInfo) { } @Override - boolean onTransportBroken(VSocketRecoveryInfo recoveryInfo) { + boolean onTransportRecoveryStop(VSocketRecoveryInfo recoveryInfo) { try { synchronized (recoveryInfo) { while (recoveryInfo.isRecoveryAttemptInProgress()) { @@ -563,7 +576,7 @@ void onDisconnected() { } @Override - void onReconnected() { + void onConnected() { } } @@ -579,7 +592,7 @@ private static class FakeRecoveryInfo extends VSocketRecoveryInfo { private final String label; FakeRecoveryInfo(String label) { - super(null, Long.MIN_VALUE); + super(null, Long.MAX_VALUE); this.label = label; } @@ -588,4 +601,4 @@ public String toString() { return "FakeVSocket{" + label + '}'; } } -} \ No newline at end of file +} diff --git a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSocketOutputStream.java b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSocketOutputStream.java index 0b3c20a9..76490793 100644 --- a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSocketOutputStream.java +++ b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSocketOutputStream.java @@ -17,6 +17,8 @@ package com.epam.deltix.util.vsocket; import com.epam.deltix.util.collections.ByteQueue; +import net.jcip.annotations.GuardedBy; +import org.jetbrains.annotations.ApiStatus; import java.io.IOException; import java.io.OutputStream; @@ -28,14 +30,16 @@ public class VSocketOutputStream extends OutputStream { private final String socketIdStr; - //@ApiStatus.Experimental + @ApiStatus.Experimental public static int CAPACITY = Integer.getInteger("TimeBase.network.socketOutputStream.bufferCapacity", 1024 * 512); public static int INCREMENT = CAPACITY / 4; /** Controls how often {@link VSProtocol#BYTES_RECIEVED} message will be sent from {@link VSTransportChannel} */ - //@ApiStatus.Experimental + @ApiStatus.Experimental public static int REPORT_THRESHOLD = Integer.getInteger("TimeBase.network.socketOutputStream.reportThreshold", CAPACITY / 4); + @GuardedBy("buffer") private final ByteQueue buffer; + @GuardedBy("out") private final OutputStream out; long confirmed; @@ -64,7 +68,7 @@ public void write(int b) { } } catch (IOException e) { broken = true; - //throw new com.epam.deltix.util.io.UncheckedIOException(e); + //throw new deltix.util.io.UncheckedIOException(e); } finally { dump(b); } @@ -78,7 +82,7 @@ public void write(byte[] b, int off, int len) { } } catch (IOException e) { broken = true; - //throw new com.epam.deltix.util.io.UncheckedIOException(e); + //throw new deltix.util.io.UncheckedIOException(e); } finally { dump(b, off, len); } @@ -96,7 +100,7 @@ public void writeTwoArrays(byte[] b1, int off1, int len1, byte[] b2, int off } } catch (IOException e) { broken = true; - //throw new com.epam.deltix.util.io.UncheckedIOException(e); + //throw new deltix.util.io.UncheckedIOException(e); } finally { dumpTwoArrays(b1, off1, len1, b2, off2, len2); } @@ -113,11 +117,24 @@ private void dump(byte[] b, int off, int len) { */ private void dumpTwoArrays(byte[] b1, int off1, int len1, byte[] b2, int off2, int len2) { synchronized (buffer) { - dumpInternal(b1, off1, len1); - dumpInternal(b2, off2, len2); + dumpInternal(b1, off1, len1, b2, off2, len2); } } + /** Same as {@link #dumpInternal(byte[], int, int)} but for two arrays. */ + @GuardedBy("buffer") + private void dumpInternal(byte[] b1, int off1, int len1, byte[] b2, int off2, int len2) { + int overflow = buffer.size() + len1 + len2 - buffer.capacity(); + if (overflow > 0) { + int incrementsToAdd = divideRoundUp(overflow, INCREMENT); + buffer.addCapacity(INCREMENT * incrementsToAdd); + } + + buffer.offer(b1, off1, len1); + buffer.offer(b2, off2, len2); + } + + @GuardedBy("buffer") private void dumpInternal(byte[] b, int off, int len) { // assert Thread.holdsLock(buffer); int overflow = buffer.size() + len - buffer.capacity(); diff --git a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSocketRecoveryInfo.java b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSocketRecoveryInfo.java index ac75e70c..540b55ff 100644 --- a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSocketRecoveryInfo.java +++ b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/VSocketRecoveryInfo.java @@ -16,6 +16,9 @@ */ package com.epam.deltix.util.vsocket; +import com.epam.deltix.util.annotations.TimestampMs; +import net.jcip.annotations.GuardedBy; + /** * @author Alexei Osipov */ @@ -24,16 +27,18 @@ class VSocketRecoveryInfo { private int reconnectAttempts = 0; private long lastReconnectAttemptTs = Long.MIN_VALUE; - private final long disconnectTs; + @TimestampMs + private final long recoveryDeadlineTs; private boolean recoveryFailed; private boolean recoverySucceeded; + // If true, it means that currently a thread actively attempts to recover corresponding channel private boolean recoveryAttemptInProgress; - VSocketRecoveryInfo(VSocket socket, long disconnectTimestamp) { + VSocketRecoveryInfo(VSocket socket, long recoveryDeadlineTs) { this.socket = socket; - this.disconnectTs = disconnectTimestamp; + this.recoveryDeadlineTs = recoveryDeadlineTs; } int addReconnectAttempt(long reconnectAttemptTimestamp) { @@ -50,8 +55,8 @@ long getLastReconnectAttemptTs() { return lastReconnectAttemptTs; } - long getDisconnectTs() { - return disconnectTs; + long getRecoveryDeadlineTs() { + return recoveryDeadlineTs; } VSocket getSocket() { @@ -62,8 +67,14 @@ void markRecoveryFailed() { recoveryFailed = true; } - void markRecoverySucceeded() { - recoverySucceeded = true; + @GuardedBy("this") + boolean tryMarkRecoverySucceeded() { + if (!isRecoveryEnded()) { + recoverySucceeded = true; + return true; + } else { + return false; + } } boolean isRecoveryFailed() { @@ -83,7 +94,6 @@ boolean isWaitingForRecovery() { } boolean startRecoveryAttempt() { - //noinspection RedundantIfStatement if (recoveryAttemptInProgress || isRecoveryEnded()) { // Only one attempt at a time return false; @@ -94,7 +104,6 @@ boolean startRecoveryAttempt() { } void stopRecoveryAttempt() { - //noinspection RedundantIfStatement if (recoveryAttemptInProgress) { // Only one attempt at a time recoveryAttemptInProgress = false; @@ -106,4 +115,4 @@ void stopRecoveryAttempt() { public boolean isRecoveryAttemptInProgress() { return recoveryAttemptInProgress; } -} \ No newline at end of file +} diff --git a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/util/TestVServerSocketFactory.java b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/util/TestVServerSocketFactory.java index d3a3bfc3..4660df56 100644 --- a/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/util/TestVServerSocketFactory.java +++ b/java/timebase/api/src/main/java/com/epam/deltix/util/vsocket/util/TestVServerSocketFactory.java @@ -17,6 +17,7 @@ package com.epam.deltix.util.vsocket.util; import com.epam.deltix.util.concurrent.QuickExecutor; +import com.epam.deltix.util.io.EOQException; import com.epam.deltix.util.memory.DataExchangeUtils; import com.epam.deltix.util.vsocket.ChannelClosedException; import com.epam.deltix.util.vsocket.VSChannel; @@ -53,6 +54,12 @@ public static VSServer createEmptyVServer(int port) throws IOException { return createVServerSocket(port, ((executor, serverChannel) -> new EmptyServer(executor, serverChannel).submit())); } + public static VSServer createBinaryEchoVServer(int port) throws IOException { + return createVServerSocket(port, ((executor, serverChannel) -> { + new BinaryEchoServer(executor, serverChannel).submit(); + })); + } + private static VSServer createVServerSocket(int port, VSConnectionListener listener) throws IOException { VSServer server = new VSServer(port); server.setConnectionListener(listener); @@ -88,7 +95,6 @@ public void run () { throw new IllegalStateException("mismatch: first(" + first + ") != index (" + index + ")"); index += 2; - } } catch (Throwable x) { x.printStackTrace (); @@ -122,6 +128,59 @@ public void run() throws InterruptedException { } } + /** + * Unlike EchoServer, this server reads and writes any binary data, not just text strings. + */ + static class BinaryEchoServer extends QuickExecutor.QuickTask { + private final VSChannel channel; + private final byte[] buffer = new byte[8 * 1024]; + private long total = 0; + volatile boolean closed = false; // Protects from extra-execution immediately after the channel closure + + public BinaryEchoServer(QuickExecutor executor, VSChannel channel) { + super(executor); + this.channel = channel; + channel.setAvailabilityListener(this::submit); + } + + @Override + public void run() { + // This task will be re-armed when more data will be available + if (closed) { + return; + } + String oldName = Thread.currentThread().getName(); + Thread.currentThread().setName("BinaryEchoServer"); + + DataInputStream is = channel.getDataInputStream(); + DataOutputStream out = channel.getDataOutputStream(); + + try { + int available; + while ((available = is.available()) > 0) { + // This read should not block because we read only up to "available" bytes + int read = is.read(buffer, 0, Math.min(available, buffer.length)); + out.write(buffer, 0, read); + total += read; + } + out.flush(); + } catch (EOQException e) { + finish(); + } catch (IOException e) { + throw new RuntimeException(e); + } finally { + Thread.currentThread().setName(oldName); + } + } + + private void finish() { + closed = true; + channel.setAvailabilityListener(null); + channel.close(); + System.out.println("BinaryEchoServer: total bytes echoed: " + total); + } + } + static class InputThroughputServer extends QuickExecutor.QuickTask { private VSChannel channel; private byte[] buffer; diff --git a/java/timebase/api/src/test/java/com/epam/deltix/util/vsocket/Test_ClientReconnect.java b/java/timebase/api/src/test/java/com/epam/deltix/util/vsocket/Test_ClientReconnect.java new file mode 100644 index 00000000..bf098c61 --- /dev/null +++ b/java/timebase/api/src/test/java/com/epam/deltix/util/vsocket/Test_ClientReconnect.java @@ -0,0 +1,427 @@ +/* + * Copyright 2026 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.epam.deltix.util.vsocket; + +import com.epam.deltix.gflog.api.Log; +import com.epam.deltix.gflog.api.LogFactory; +import com.epam.deltix.gflog.jul.JulBridge; +import com.epam.deltix.qsrv.hf.spi.conn.DisconnectEventListener; +import com.epam.deltix.util.lang.Util; +import com.epam.deltix.util.vsocket.util.TestVServerSocketFactory; +import org.jetbrains.annotations.NotNull; + +import org.netcrusher.NetFreezer; +import org.netcrusher.core.reactor.NioReactor; +import org.netcrusher.tcp.TcpCrusher; +import org.netcrusher.tcp.TcpCrusherBuilder; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BooleanSupplier; +import java.util.function.Consumer; + +import org.junit.jupiter.api.*; + +import static org.junit.jupiter.api.Assertions.*; + + +/** + * Contains tests that use an intermediate proxy to simulate network issues between {@link VSClient} and server. + */ +public class Test_ClientReconnect { + + + // Warning: having more than 120 transports may lead to Gradle test instability because + // of insufficient off-heap buffer capacity for all connections. + private static final int RECOVERY_TEST_TRANSPORTS = Integer.parseInt(System.getProperty("Test_ClientReconnect.transports", "100")); + + // Linger interval is 10 seconds + 5 seconds for notification delays (especially on CI) + private static final int DISCONNECT_WAIT_TIMEOUT = 15; + + static { + JulBridge.install(); + } + + private static final Log LOG = LogFactory.getLog(Test_ClientReconnect.class); + + public static final int PROXY_PORT = 34_781; + public static final int EMBEDDED_SERVER_PORT = 35_782; + + + private VSServer server; + private NioReactor nioReactor; + private TcpCrusher tcpCrusher; + private NetFreezer acceptorFreezer; + private final AtomicInteger connectCounter = new AtomicInteger(0); + + private volatile Consumer proxyConnectListener = null; + + @BeforeEach + public void start() throws Throwable { + this.server = TestVServerSocketFactory.createBinaryEchoVServer(EMBEDDED_SERVER_PORT); + this.server.setTransportsLimit((short) 1000); // Allow many transports for reconnect tests + this.server.start(); + + int serverPort = server.getLocalPort(); + String serverHost = "localhost"; + + this.nioReactor = new NioReactor(); + + this.tcpCrusher = TcpCrusherBuilder.builder() + .withReactor(nioReactor) + .withBindAddress("localhost", PROXY_PORT) + .withConnectAddress(serverHost, serverPort) + //.withBacklog(1) + .withCreationListener(clientAddress -> { + int connectionNumber = connectCounter.incrementAndGet(); + LOG.info("Proxy: Client %s connected: %s").with(connectionNumber).with(clientAddress); + if (proxyConnectListener != null) { + proxyConnectListener.accept(clientAddress); + } + }) + .buildAndOpen(); + this.acceptorFreezer = tcpCrusher.getAcceptorFreezer(); + } + + @AfterEach + public void stop() { + proxyConnectListener = null; +// if (acceptorFreezer != null && acceptorFreezer.isFrozen()) { +// // Prevent incorrect state on close +// try { +// acceptorFreezer.unfreeze(); +// } catch (RuntimeException e) { +// LOG.warn("Failed to unfreeze acceptor during cleanup: %s").with(e); +// } +// } + + if (tcpCrusher != null) { + tcpCrusher.close(); + } + if (nioReactor != null) { + nioReactor.close(); + } + if (server != null) { + server.close(); + } + } + + @NotNull + private static VSClient connectClient() throws IOException { + return new VSClient("localhost", PROXY_PORT); + } + + private int getConnectedClientCount() { + return tcpCrusher.getClientAddresses().size(); + } + + /** + * Client should not get blocked on connection loss. + */ + @RepeatedTest(1) + @Timeout(20) + public void testConnectionLoss() throws Exception { + VSClient client = connectClient(); + try { + assertEquals(0, getConnectedClientCount()); + client.connect(); + LOG.info("Client connected"); + assertTrue(client.isConnected()); + assertEquals(3, getConnectedClientCount()); + Test_VSocket_Correctness.assertEchoClientCorrectness(client, 1_000); + + CountDownLatch disconnectedLatch = new CountDownLatch(1); + AtomicInteger disconnectCount = new AtomicInteger(0); + long listenerInstallTime = System.currentTimeMillis(); + client.setDisconnectedListener(new DisconnectEventListener() { + @Override + public void onDisconnected() { + LOG.info("Client onDisconnected listener triggered after %s ms") + .with(System.currentTimeMillis() - listenerInstallTime); + disconnectCount.incrementAndGet(); + disconnectedLatch.countDown(); + } + + @Override + public void onReconnected() { + } + }); + + long disconnectStart = System.currentTimeMillis(); + + // Close all connections and disable proxy + tcpCrusher.close(); + + boolean success = disconnectedLatch.await(DISCONNECT_WAIT_TIMEOUT, TimeUnit.SECONDS); + long disconnectEnd = System.currentTimeMillis(); + LOG.info("Stopped to wait for disconnected after %s ms").with(disconnectEnd - disconnectStart); + LOG.info("Disconnect event count: %s").with(disconnectCount.get()); + + assertTrue(success, "Client did not receive disconnect event in time"); + // Client is still disconnected + assertFalse(client.isConnected()); + + // Wait for any redundant events + Thread.sleep(10); + assertEquals(1, disconnectCount.get(), "Disconnect event should be fired exactly once"); + } finally { + // TODO: Probably we may want to reconsider this in future and allow graceful client close + // For this test we do not care if client throws exception + Util.close(client); + } + } + + /** + * Client should be able to reconnect after single recoverable connection loss. + */ + @RepeatedTest(1) + @Timeout(200) + public void testReconnectAfterSingleDisconnected() throws Exception { + try (VSClient client = connectClient()) { + client.connect(); + var eventListener = installListener(client); + + // Multiple iterations to ensure that we return to stable state + for (int i = 0; i < 10; i++) { + assertTrue(client.isConnected()); + assertEquals(3, getConnectedClientCount()); + Test_VSocket_Correctness.assertEchoClientCorrectness(client, 1_000); + + // Simulates temporary connection loss for single connection + InetSocketAddress clientSocketAddress = tcpCrusher.getClientAddresses().iterator().next(); + Assertions.assertNotNull(clientSocketAddress); + + boolean closed = tcpCrusher.closeClient(clientSocketAddress); + assertTrue(closed, "Failed to close client connection"); + + Thread.sleep(1000); // Wait for the connection to be closed + + boolean gotDisconnectEvent = eventListener.disconnectedLatch.await(DISCONNECT_WAIT_TIMEOUT, TimeUnit.SECONDS); + assertFalse(gotDisconnectEvent, "Client is not supposed to generate disconnect if it was able to reconnect"); + assertEquals(0, eventListener.disconnectCount.get()); + //Assert.assertEquals(0, eventListener.reconnectCount.get()); + + LOG.info("State: %s").with(client.getDispatcher().getInternalState()); + assertTrue(client.tryGetConnectionStatus()); + assertEquals(3, getConnectedClientCount()); + } + } + } + + /** + * Client should be able to reconnect after connection loss if network is restored. + */ + @RepeatedTest(1) + @Timeout(200) + public void testReconnectAfterAllDisconnected() throws Exception { + try (VSClient client = connectClient()) { + client.connect(); + var eventListener = installListener(client); + + // Multiple iterations to ensure that we return to stable state + for (int i = 0; i < 10; i++) { + assertTrue(client.isConnected()); + assertEquals(3, getConnectedClientCount()); + Test_VSocket_Correctness.assertEchoClientCorrectness(client, 1_000); + + // Simulates temporary connection loss for all transports + tcpCrusher.close(); + tcpCrusher.open(); + + Thread.sleep(1000); // Wait for the connection to be closed + + + boolean gotDisconnectEvent = eventListener.disconnectedLatch.await(DISCONNECT_WAIT_TIMEOUT, TimeUnit.SECONDS); + assertFalse(gotDisconnectEvent, "Client is not supposed to generate disconnect if it was able to reconnect"); + assertEquals(0, eventListener.disconnectCount.get()); + // TODO: Uncomment - currently client fires redundant reconnect event + //Assert.assertEquals(0, eventListener.reconnectCount.get()); + + LOG.info("State: %s").with(client.getDispatcher().getInternalState()); + assertTrue(client.tryGetConnectionStatus()); + assertEquals(3, getConnectedClientCount()); + } + } + // Should be closed gracefully + } + + /** + * Ensure that if client is closed during disconnect event, it does not get stuck. + */ + @RepeatedTest(1) + @Timeout(20) + public void testCloseOnDisconnect() throws Exception { + try (VSClient client = connectClient()) { + client.connect(); + + assertTrue(client.isConnected()); + assertEquals(3, getConnectedClientCount()); + Test_VSocket_Correctness.assertEchoClientCorrectness(client, 1_000); + + CountDownLatch disconnectedLatch = new CountDownLatch(1); + AtomicInteger disconnectCount = new AtomicInteger(0); + client.setDisconnectedListener(new DisconnectEventListener() { + @Override + public void onDisconnected() { + LOG.info("Client disconnected"); + disconnectCount.incrementAndGet(); + client.close(); + disconnectedLatch.countDown(); + } + + @Override + public void onReconnected() { + } + }); + + // Simulates connection loss for all transports + tcpCrusher.close(); + + disconnectedLatch.await(); + + // Wait for any redundant events + Thread.sleep(10); + assertEquals(1, disconnectCount.get(), "Disconnect event should be fired exactly once"); + } + } + + /** + * Starts with 1000 connections, kills them, recovers only some of them. + * Expected to end up in DISCONNECTED state. + */ + @RepeatedTest(1) + //@Test + @Timeout(60) + public void testPartialRecovery() throws Exception { + int transports = RECOVERY_TEST_TRANSPORTS; + + int halfTransports = transports / 2; + if (halfTransports == 0) { + throw new IllegalStateException("Number of transports is too low for this test"); + } + + try (VSClient client = connectClient()) { + client.setNumTransportChannels(transports); + client.connect(); + + waitUntil(10_000, "Client is not connected in time", client::isConnected); + + assertEquals(transports, getConnectedClientCount()); + LOG.info("Client connected with %s transports").with(transports); + Test_VSocket_Correctness.assertEchoClientCorrectness(client, 1_000); + + List initialConnections = new ArrayList<>(tcpCrusher.getClientAddresses()); + + + // Allow only one reconnection + AtomicInteger reconnectedCount = new AtomicInteger(0); + proxyConnectListener = (inetSocketAddress) -> { + // Warning: this listener is asynchronous, so it's not guaranteed that exactly 500 connections will fail + + int newCount = reconnectedCount.incrementAndGet(); + + if (newCount == halfTransports) { + // Disable new connections + acceptorFreezer.freeze(); + } + }; + + LOG.info("Killing initial connections"); + for (InetSocketAddress address : initialConnections) { + tcpCrusher.closeClient(address); + } + + VSDispatcher[] dispatchers = server.getDispatchers(); + assertEquals(1, dispatchers.length); + VSDispatcher dispatcher = dispatchers[0]; + + // Wait until dispatcher detects broken transport + LOG.info("Waiting for dispatcher to detect disconnection"); + long now = System.currentTimeMillis(); + long start = now; + long deadline = now + 20_000; + while ((now = System.currentTimeMillis()) < deadline) { + VSDispatcherState state = dispatcher.getInternalState(); + if (state != VSDispatcherState.DISCONNECTED) { + Thread.sleep(100); + } else { + break; + } + } + LOG.info("Waited %s ms for dispatcher to detect disconnection").with(now - start); + + LOG.info("Dispatcher state: %s").with(dispatcher.getInternalState()); + + // Loss of any transport must cause dispatcher to go to DISCONNECTED state + assertEquals(VSDispatcherState.DISCONNECTED, dispatcher.getInternalState()); + } + } + + static void waitUntil(int timeoutMs, String errorMessage, BooleanSupplier condition) { + long startTime = System.currentTimeMillis(); + long endTime = startTime + timeoutMs; + while (System.currentTimeMillis() < endTime) { + if (condition.getAsBoolean()) { + // Success + return; + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted while waiting", e); + } + } + + fail(errorMessage); + } + + static TestEventListener installListener(VSClient client) { + TestEventListener eventListener = new TestEventListener(); + client.setDisconnectedListener(eventListener); + return eventListener; + } + + static class TestEventListener implements DisconnectEventListener { + CountDownLatch disconnectedLatch = new CountDownLatch(1); + CountDownLatch reconnectedLatch = new CountDownLatch(1); + AtomicInteger disconnectCount = new AtomicInteger(0); + AtomicInteger reconnectCount = new AtomicInteger(0); + + public TestEventListener() { + } + + @Override + public void onDisconnected() { + LOG.info("Client disconnected"); + disconnectCount.incrementAndGet(); + disconnectedLatch.countDown(); + } + + @Override + public void onReconnected() { + LOG.info("Client reconnected"); + reconnectCount.incrementAndGet(); + reconnectedLatch.countDown(); + } + } +} \ No newline at end of file diff --git a/java/timebase/api/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketChannelLeak.java b/java/timebase/api/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketChannelLeak.java new file mode 100644 index 00000000..50f5e6ad --- /dev/null +++ b/java/timebase/api/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketChannelLeak.java @@ -0,0 +1,155 @@ +/* + * Copyright 2024 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.concurrent.QuickExecutor; +import com.epam.deltix.util.lang.DisposableListener; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Tests if there is a memory leak in VSChannel when channel gets closed on the client side. + * + *

Run the test with -Xmx500m to see the problem. + */ +@SuppressWarnings("NewClassNamingConvention") +public class Test_VSocketChannelLeak { + private static final boolean enableCloseFix = true; + private static final int ITERATIONS = 1000; + private static final int PAYLOAD_SIZE = 1_024 * 1_024; // 1 MB + private static final boolean waitForFreeMem = false; // Needed for CI env - it's slow + +// public static void main (String [] args) throws Exception { +// testImpl(); +// //Thread.sleep(Long.MAX_VALUE); +// } + + @Ignore // Fails on CI + @Test(timeout = 60_000) + public void test() throws IOException, InterruptedException { + testImpl(); + } + + @SuppressWarnings("Convert2Lambda") + private static void testImpl() throws IOException, InterruptedException { + VSServer server = new VSServer(0); + + AtomicLong openChannels = new AtomicLong(); + AtomicLong closedChannels = new AtomicLong(); + + ExecutorService executorService = Executors.newCachedThreadPool(); + + server.setConnectionListener(new VSConnectionListener() { + @Override + public void connectionAccepted(QuickExecutor executor, VSChannel serverChannel) { + openChannels.incrementAndGet(); + + // This payload will be kept in memory until the channel is closed + byte[] payload = new byte[PAYLOAD_SIZE]; + payload[0] = 1; + + serverChannel.addDisposableListener(new DisposableListener<>() { + @Override + public void disposed(VSChannel resource) { + byte val = payload[11]; + if (val != 0) { + System.out.println("Should never happen"); + } + // Intentionally do not remove the listener from the channel to release the memory only if channel is released + } + }); + + if (enableCloseFix) { + executorService.submit(() -> { + DataInputStream dis = serverChannel.getDataInputStream(); + //noinspection TryFinallyCanBeTryWithResources + try { + while (true) { + try { + dis.readByte(); + } catch (EOFException e) { + // Graceful close + break; + } catch (IOException e) { + break; + } + } + } finally { + serverChannel.close(); // This will release the memory + closedChannels.incrementAndGet(); + } + }); + } + } + }); + server.setDaemon(true); + server.start(); + System.out.println("Server started on " + server.getLocalPort()); + + try { + createConnections("localhost", server.getLocalPort()); + } finally { + System.out.println("Open channels: " + openChannels.get()); + System.out.println("Closed channels: " + closedChannels.get()); + } + + server.close(); + executorService.shutdown(); + } + + public static void createConnections(String host, int port) + throws IOException, InterruptedException { + + VSClient client = new VSClient(host, port); + client.connect(); + + // This loop fails with OutOfMemoryError + for (int i = 0; i < ITERATIONS; i++) { + VSChannel s = client.openChannel(); + s.close(false); + + if (waitForFreeMem && (Runtime.getRuntime().freeMemory() < Runtime.getRuntime().totalMemory() / 5)) { + // Wait if we below 20% of free memory + System.gc(); + Thread.sleep(200); + } else { + Thread.yield(); + } + } + long usedMemory1 = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); + + System.gc(); + Thread.sleep(3000); + + long usedMemory2 = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); + System.out.println("Used memory before gc: " + usedMemory1); + System.out.println("Used memory after gc: " + usedMemory2); + + client.close(); + + if (usedMemory2 > ITERATIONS * PAYLOAD_SIZE) { + throw new RuntimeException("Memory leak detected"); + } + } +} diff --git a/java/timebase/api/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketEcho.java b/java/timebase/api/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketEcho.java index 60da33bb..60bf03cd 100644 --- a/java/timebase/api/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketEcho.java +++ b/java/timebase/api/src/test/java/com/epam/deltix/util/vsocket/Test_VSocketEcho.java @@ -23,7 +23,7 @@ import java.io.*; public class Test_VSocketEcho { - public static void main (String args []) throws Throwable { + public static void main (String[] args) throws Throwable { int port = SocketTestUtilities.parsePort(args); VSServer server = TestVServerSocketFactory.createEchoVServer(port); diff --git a/java/timebase/api/src/test/java/com/epam/deltix/util/vsocket/Test_VSocket_Correctness.java b/java/timebase/api/src/test/java/com/epam/deltix/util/vsocket/Test_VSocket_Correctness.java new file mode 100644 index 00000000..16952ec6 --- /dev/null +++ b/java/timebase/api/src/test/java/com/epam/deltix/util/vsocket/Test_VSocket_Correctness.java @@ -0,0 +1,153 @@ +/* + * Copyright 2024 EPAM Systems, Inc + * + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. Licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.epam.deltix.util.vsocket; + +import com.epam.deltix.util.io.BasicIOUtil; +import com.epam.deltix.util.vsocket.util.SocketTestUtilities; +import com.epam.deltix.util.vsocket.util.TestVServerSocketFactory; +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Arrays; +import java.util.Random; + +/** + * Sends random data to an echo server and checks that correct data is sent back. + */ +@SuppressWarnings("SameParameterValue") +public class Test_VSocket_Correctness { + + public static final int WRITE_COUNT = 1_000_000; + + @Test + public void testSocket() throws Throwable { + Test_VSocket_Correctness.main(new String[0]); + } + + public static void main(String[] args) throws Throwable { + int port = SocketTestUtilities.parsePort(args); + + VSServer server = TestVServerSocketFactory.createBinaryEchoVServer(port); + server.setDaemon(true); + server.start(); + System.out.println("Server started on " + server.getLocalPort()); + + try { + client("localhost", server.getLocalPort()); + } finally { + server.close(); + } + } + + private static void client(String host, int port) { + try (VSClient client = new VSClient(host, port)) { + client.connect(); + + assertEchoClientCorrectness(client, WRITE_COUNT); + } catch (IOException x) { + throw new UncheckedIOException(x); + } + } + + static void assertEchoClientCorrectness(VSClient client, int writeCount) { + Random rngSrc = new Random(0); + Random rngDst = new Random(0); + + // Fills data with values 1..127 + byte[] data = makeTestData(); + + try (VSChannel channel = client.openChannel()) { + DataOutputStream os = channel.getDataOutputStream(); + + // Data writer thread + new Thread(() -> { + Thread.currentThread().setName("PRODUCER"); + sendData(rngSrc, os, data, writeCount); + }).start(); + + // Reader + readData(channel, data, rngDst, writeCount); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static void sendData(Random rngSrc, DataOutputStream os, byte[] data, int writeCount) { + long totalSent = 0; + try { + for (int i = 0; i < writeCount; i++) { + int size = generateNextSize(rngSrc); + //rngSrc.nextBytes(data); + os.write(data, 0, size); + totalSent += size; + os.write(Byte.MIN_VALUE); + totalSent += 1; + } + os.flush(); + System.out.println("Producer finished to send data. Sent: " + totalSent + " bytes"); + } catch (IOException x) { + throw new UncheckedIOException(x); + } + } + + private static void readData(VSChannel channel, byte[] expected, Random rngDst, int readCount) throws IOException { + long totalRead = 0; + DataInputStream is = channel.getDataInputStream(); + byte[] actual = new byte[8 * 1024]; + for (int i = 0; i < readCount; i++) { + int size = generateNextSize(rngDst); + //rngDst.nextBytes(expected); + BasicIOUtil.readFully(is, actual, 0, size); + + int mismatch = Arrays.mismatch(expected, 0, size, actual, 0, size); + if (mismatch >= 0) { + throw new AssertionError("Data mismatch at position " + (totalRead + mismatch)); + } + totalRead += size; + + int single = is.read(); + if (single < 0) { + throw new AssertionError("Unexpected end of stream"); + } + if (single - 256 != Byte.MIN_VALUE) { + throw new AssertionError("Unexpected value " + single + " at position " + totalRead); + } + totalRead += 1; + } + System.out.println("Consumer finished to read data. Read: " + totalRead + " bytes"); + } + + private static int generateNextSize(Random rngSrc) { + return 10 + rngSrc.nextInt(1000); + } + + private static byte[] makeTestData() { + byte[] data = new byte[8 * 1024]; + byte val = 0; + for (int ii = 0; ii < data.length; ii++) { + val++; + data[ii] = val; + if (val == Byte.MAX_VALUE) { + val = 0; + } + } + return data; + } +} diff --git a/java/timebase/api/src/test/java/com/epam/deltix/util/vsocket/VSDispatcherTest.java b/java/timebase/api/src/test/java/com/epam/deltix/util/vsocket/VSDispatcherTest.java index 089c9ed8..62393536 100644 --- a/java/timebase/api/src/test/java/com/epam/deltix/util/vsocket/VSDispatcherTest.java +++ b/java/timebase/api/src/test/java/com/epam/deltix/util/vsocket/VSDispatcherTest.java @@ -33,7 +33,7 @@ */ public class VSDispatcherTest { - @Test (timeout = 5_000) // Note: test timeout must be greater than reconnectInterval + 2000ms + @Test (timeout = 10_000) // Note: test timeout must be greater than reconnectInterval + 2000ms public void testNoHangsOnConcurrentDisconnects() throws IOException, InterruptedException { int reconnectInterval = 2000; @@ -45,16 +45,16 @@ void onDisconnected() { } @Override - void onReconnected() { + void onConnected() { } @Override - boolean onTransportStopped(VSocketRecoveryInfo recoveryInfo) { + boolean onTransportRecoveryStart(VSocketRecoveryInfo recoveryInfo) { return false; } @Override - boolean onTransportBroken(VSocketRecoveryInfo recoveryInfo) { + boolean onTransportRecoveryStop(VSocketRecoveryInfo recoveryInfo) { return true; } }); @@ -115,14 +115,22 @@ boolean onTransportBroken(VSocketRecoveryInfo recoveryInfo) { out.write(buffer, 0, buffer.length); } - assertTrue(dispatcher.hasAvailableTransport()); + assertTrue(dispatcher.isConnectedOrReconnecting()); System.out.println("Emulating broken transports..."); startBarrier.countDown(); - Thread.sleep(100); // Let threads get into blocked state + + // Let threads get into waiting for recovery state, + // but maximum of 1sec, which is half of the waiting time for recovery + for (int i = 0; i < 10; i++) { + Thread.sleep(100); + if (errorThreads.stream().allMatch(thread -> thread.getState() == Thread.State.TIMED_WAITING)) { + break; + } + } // Now no transports should be available - assertFalse(dispatcher.hasAvailableTransport()); + assertFalse(dispatcher.isConnectedAndNotReconnecting()); // Emulate Flusher thread VSChannelImpl vsChannel = channels.get(0); @@ -140,5 +148,8 @@ boolean onTransportBroken(VSocketRecoveryInfo recoveryInfo) { for (Thread thread : errorThreads) { thread.join(); } + + Test_ClientReconnect.waitUntil(5000, "Dispatcher did not reach DISCONNECTED state in time", + () -> dispatcher.getInternalState() == VSDispatcherState.DISCONNECTED); } -} \ No newline at end of file +} diff --git a/java/timebase/client/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/client/SafeDisconnectableEventHandler.java b/java/timebase/client/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/client/SafeDisconnectableEventHandler.java new file mode 100644 index 00000000..40937753 --- /dev/null +++ b/java/timebase/client/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/client/SafeDisconnectableEventHandler.java @@ -0,0 +1,66 @@ +package com.epam.deltix.qsrv.hf.tickdb.comm.client; + +import com.epam.deltix.gflog.api.Log; +import com.epam.deltix.gflog.api.LogFactory; +import com.epam.deltix.qsrv.hf.spi.conn.DisconnectEventListener; +import com.epam.deltix.qsrv.hf.spi.conn.Disconnectable; + +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * Same as {@link com.epam.deltix.qsrv.hf.spi.conn.DisconnectableEventHandler} but catches and logs exceptions from listeners + * instead of propagating them. + *

+ * Helps implement the {@link Disconnectable} interface. + *

Doesn't maintain a connection status, so isConnected must be implemented by a client.

+ */ +class SafeDisconnectableEventHandler implements Disconnectable { + public static final Log LOGGER = LogFactory.getLog(SafeDisconnectableEventHandler.class); + + private final CopyOnWriteArrayList listeners = new CopyOnWriteArrayList<>(); + + @Override + public void addDisconnectEventListener(DisconnectEventListener listener) { + listeners.addIfAbsent(listener); + } + + @Override + public void removeDisconnectEventListener(DisconnectEventListener listener) { + listeners.remove(listener); + } + + @Override + public boolean isConnected() { + throw new UnsupportedOperationException(); + } + + public void onReconnected() { + for (DisconnectEventListener listener : listeners) { + try { + listener.onReconnected(); + } catch (Throwable t) { + LOGGER.error("Error processing reconnect event: %s") + .with(t) + .with(t); + if (!(t instanceof Exception)) { + throw t; + } + } + } + } + + public void onDisconnected() { + for (DisconnectEventListener listener : listeners) { + try { + listener.onDisconnected(); + } catch (Throwable t) { + LOGGER.error("Error processing reconnect event: %s") + .with(t) + .with(t); + if (!(t instanceof Exception)) { + throw t; + } + } + } + } +} diff --git a/java/timebase/client/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/client/TBConnectionParams.java b/java/timebase/client/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/client/TBConnectionParams.java new file mode 100644 index 00000000..cf2b3b25 --- /dev/null +++ b/java/timebase/client/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/client/TBConnectionParams.java @@ -0,0 +1,59 @@ +package com.epam.deltix.qsrv.hf.tickdb.comm.client; + +import com.epam.deltix.qsrv.hf.tickdb.pub.TickDBFactory; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Nullable; + +import java.util.Map; +import java.util.Objects; + +/** + * {@link com.epam.deltix.qsrv.hf.tickdb.comm.client.TickDBClient} connection parameters, + * as if in {@link TickDBFactory#connect(String, int, boolean, String, String, Map)}. + */ +@ApiStatus.Internal +public class TBConnectionParams { + private final String host; + private final int port; + private final boolean enableSSL; + private final String user; + private final String pass; + private final Map params; + + public TBConnectionParams( + String host, int port, boolean enableSSL, + @Nullable String user, @Nullable String pass, + @Nullable Map params + ) { + this.host = Objects.requireNonNull(host, "url cannot be null"); + this.port = port; + this.enableSSL = enableSSL; + this.user = user; + this.pass = pass; + this.params = params; + } + + public boolean isEnableSSL() { + return enableSSL; + } + + public String getHost() { + return host; + } + + public Map getParams() { + return params; + } + + public String getPass() { + return pass; + } + + public int getPort() { + return port; + } + + public String getUser() { + return user; + } +} diff --git a/java/timebase/client/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/client/TickDBClient.java b/java/timebase/client/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/client/TickDBClient.java index 89589fcc..fb442513 100644 --- a/java/timebase/client/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/client/TickDBClient.java +++ b/java/timebase/client/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/client/TickDBClient.java @@ -85,8 +85,10 @@ import com.epam.deltix.util.vsocket.VSClient; import com.epam.deltix.util.vsocket.VSProtocol; import io.aeron.Aeron; +import org.apache.commons.lang3.mutable.MutableBoolean; import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.VisibleForTesting; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -103,13 +105,15 @@ import java.util.*; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; /** * */ -public class TickDBClient implements DXRemoteDB, DBStateNotifier, RemoteTickDB, ReconnectableImpl.Reconnector, TopicDB { +public class TickDBClient implements DXRemoteDB, DBStateNotifier, RemoteTickDB, TickDBReconnectableImpl.Reconnector, TopicDB { // Verifying, that we are under JDK, not JRE. static { @@ -118,12 +122,18 @@ public class TickDBClient implements DXRemoteDB, DBStateNotifier, RemoteTickDB, public static final Log LOGGER = LogFactory.getLog("tickdb.client"); + @ApiStatus.Experimental private static final int MAX_REVERSE_BUFFER_SIZE = Integer.getInteger("TimeBase.transport.channel.maxReverseBufferSize", 64 * 1024); @ApiStatus.Experimental private static final int DEFAULT_LOCAL_CHANNEL_SIZE = Integer.getInteger("TimeBase.transport.channel.local.defaultSize", VSProtocol.CHANNEL_BUFFER_SIZE); @ApiStatus.Experimental private static final int DEFAULT_REMOTE_CHANNEL_SIZE = Integer.getInteger("TimeBase.transport.channel.remote.defaultSize", VSProtocol.CHANNEL_MAX_BUFFER_SIZE); + @ApiStatus.Experimental + private static final int DEFAULT_HANDLER_WAIT_TIMEOUT = Integer.getInteger("TimeBase.client.eventHandler.waitTimeout", 5_000); + + // Will contain "true" if current thread is inside connect/disconnect event handler + private static final ThreadLocal insideOfEventHandler = ThreadLocal.withInitial(() -> new MutableBoolean(false)); private static int getConnectionsNumber(boolean isRemote) { @@ -151,7 +161,7 @@ private static int getConnectionsNumber(boolean isRemo private long[] latency; private long availableBandwidth = 0; - private final ReconnectableImpl connMgr; + private final TickDBReconnectableImpl connMgr; private final Runnable updater = this::sendMetaDataUpdate; private VSClient connection; @@ -168,6 +178,8 @@ private static int getConnectionsNumber(boolean isRemo private String applicationId; private String address; + private Integer numTransportChannels = null; // Null means auto-configure + private boolean secured = false; private final CodecFactory intpCodecFactory = @@ -176,6 +188,7 @@ private static int getConnectionsNumber(boolean isRemo private final CodecFactory compCodecFactory = CodecFactory.newCompiledCachingFactory (); + private int defaultChannelCapacity; private boolean useCompression = false; private boolean isRemoteConnection = false; @@ -192,7 +205,10 @@ private static int getConnectionsNumber(boolean isRemo private ThreadFactory topicConsumerThreadFactory; private final TimeSource timeSource; + private int eventHandlerWaitTimeout = DEFAULT_HANDLER_WAIT_TIMEOUT; + private final CopyOnWriteArrayList stateListeners = new CopyOnWriteArrayList<>(); + private final ConnectionNotificationTask connectionNotifier; private final DisconnectEventListener listener = new DisconnectEventListener() { @Override @@ -221,8 +237,9 @@ protected TickDBClient (String host, int port, boolean enableSSL, UserPrincipal } this.timeout = isRemoteConnection ? 5000 : 1000; + this.defaultChannelCapacity = isRemoteConnection ? DEFAULT_REMOTE_CHANNEL_SIZE : DEFAULT_LOCAL_CHANNEL_SIZE; - connMgr = new ReconnectableImpl("TickDBClient", this); + connMgr = new TickDBReconnectableImpl("TickDBClient", this); //connMgr.setLazyLogger(LOGGER); // connMgr.setLogger(LOGGER); // connMgr.setLogLevel (Level.INFO); @@ -235,7 +252,8 @@ protected TickDBClient (String host, int port, boolean enableSSL, UserPrincipal this.topicConsumerThreadFactory = createTopicConsumerThreadFactory(); this.timeSource = null; - // DefaultTimeSourceProvider.getTimeSourceForApp("TickDBClient"); TODO: @MERGE + + this.connectionNotifier = new ConnectionNotificationTask(TickDBClient.this.getQuickExecutor()); } public TickDBClient (String host, int port, String user, String pass) { @@ -260,12 +278,49 @@ public SessionClient getSession() { return session; } + /** + * Assigns an {@link Oauth2Client} to the current TimeBase connection. + *

+ * The {@code TickDBClient} takes ownership of the provided {@code Oauth2Client} + * and will automatically close it when the TimeBase connection is closed. + *

+ * This method must be called before opening the connection. + * + * @param oauth2Client the OAuth2 client to associate with this connection + */ public void setOauth2Client(Oauth2Client oauth2Client) { - this.userPrincipalResolver.setOauth2Client(oauth2Client); + this.setOauth2Client(oauth2Client, false); } /** - * Sets user access token to login to the Timebase server when OAUTH type of authentication defined on server. + * Assigns an {@link Oauth2Client} to the current TimeBase connection. + *

+ * If {@code external} is {@code false}, the {@code TickDBClient} takes ownership + * of the provided client and will automatically close it when the TimeBase + * connection is closed. + *

+ * If {@code external} is {@code true}, the {@code TickDBClient} will use the + * provided client but will not manage its lifecycle. In this case, the + * caller is responsible for closing the {@code Oauth2Client}. + *

+ * This method must be called before opening the connection. + * + * @param oauth2Client the OAuth2 client to associate with this connection + * @param external {@code true} if the client is managed externally and should + * not be closed by {@code TickDBClient}; {@code false} if + * ownership should be transferred to {@code TickDBClient} + */ + public void setOauth2Client(Oauth2Client oauth2Client, boolean external) { + this.userPrincipalResolver.setOauth2Client(oauth2Client, external); + } + + @VisibleForTesting + UserPrincipalResolver getUserPrincipalResolver() { + return userPrincipalResolver; + } + + /** + * Sets user access token for login into Timebase server when OAUTH type of authentication defined on server. * @param token Access token */ public void setAccessToken(String token) { @@ -376,7 +431,12 @@ private VSClient getConnectedVSClient() throws IOException { if (address != null) connection.setClientAddress(address, idd); - connection.setNumTransportChannels(isRemoteConnection ? 1 : getConnectionsNumber(isRemoteConnection)); + // Use explicitly configured number of transport channels or determine it automatically + int transportChannels = numTransportChannels != null ? + numTransportChannels : + (isRemoteConnection ? 1 : getConnectionsNumber(true)); + + connection.setNumTransportChannels(transportChannels); connection.setTimeout(timeout); connection.setDisconnectedListener(listener); connection.setSslContext(SSLClientContextProvider.getSSLContext()); @@ -402,9 +462,7 @@ protected VSChannel createChannel(ChannelType type, boolean autoCommit, int inCapacity; int outCapacity; - int defaultCapacity = isRemoteConnection ? DEFAULT_REMOTE_CHANNEL_SIZE : DEFAULT_LOCAL_CHANNEL_SIZE; - - int configuredCapacity = channelBufferSize > 0 ? channelBufferSize : defaultCapacity; + int configuredCapacity = channelBufferSize > 0 ? channelBufferSize : defaultChannelCapacity; switch (type) { case Input: @@ -507,6 +565,24 @@ public synchronized long getServerStartTime() { public void open(boolean readOnly) { if (syncOpen(readOnly)) onReconnected(); + + boolean insideOfEventHandler = TickDBClient.insideOfEventHandler.get().isTrue(); + if (insideOfEventHandler) { + LOGGER.warn("open() is called from inside of event handler. This is not intended API usage. " + + "This can lead to unexpected deadlocks " + + "and revokes guaranties on handler event order execution. " + + "Please make sure that TickDBClient.open() is called outside of it's own event handlers. " + + "Event handler thread: %s").with(Thread.currentThread().getName()); + } else { + // Wait for existing events to be processed before we return from open() method. + // This is necessary to avoid situation when a listener that added after open() method call, + // receives events that were generated during the connection recovery process. + if (!this.connectionNotifier.waitForSubmittedEvents(eventHandlerWaitTimeout)) { + LOGGER.warn("Some of connection listener events were not processed within timeout after open() method call. " + + "Please make sure that event handlers are processing events in a timely manner and do not block. " + + "Thread for open(): %s").with(Thread.currentThread().getName()); + } + } } private synchronized boolean syncOpen (boolean readOnly) { @@ -1120,6 +1196,14 @@ public boolean isReadOnly () { } public void close () { + boolean insideOfEventHandler = TickDBClient.insideOfEventHandler.get().isTrue(); + if (insideOfEventHandler) { + LOGGER.warn("close() is called from inside of event handler. This is not intended API usage. " + + "This can lead to unexpected deadlocks " + + "and revokes guaranties on handler event order execution. " + + "Please make sure that TickDBClient.close() is called outside of it's own event handlers. " + + "Event handler thread: %s").with(Thread.currentThread().getName()); + } connMgr.cancelReconnect(); @@ -1154,17 +1238,63 @@ public void close () { isOpen = false; } + int eventId = -1; + synchronized (this) { + boolean wasConnected = connMgr.isConnected(); + if (wasConnected) { + connMgr.disconnected(); + // Warning: here we submit a task for QuickExecutor + eventId = connectionNotifier.addDisconnectEvent(); + } + } + + if (eventId >= 0) { + // Wait for disconnect event to be processed, by event handlers, + // so we can guarantee that any handler installed by client before close() + // will be executed before close() returns. + + // However if we are inside of event handler right now, + // then the client code called close() from the handler, + // so waiting for event to be processed is pointless (and will cause a deadlock) + if (!insideOfEventHandler) { + if (!connectionNotifier.waitForEventProcessing(eventId, eventHandlerWaitTimeout)) { + LOGGER.warn("Some of connection listener events were not processed within timeout after close() method call. " + + "Please make sure that event handlers are processing events in a timely manner and do not block. " + + "Thread for close(): %s").with(Thread.currentThread().getName()); + } + } + } + + userPrincipalResolver.close(); + + boolean asyncExecutorShutdown = false; + QuickExecutor quickExecutor = contextContainer.getQuickExecutor(); + // shutdown QuickExecutor only if 'open' if (shutdown) { - contextContainer.getQuickExecutor().shutdownInstance(); + // This have to be done after notifying listeners, as we use QE to execute callbacks + + if (insideOfEventHandler) { + // Warning: if we are inside of event handler right now, + // then this call will basically mean interruption of THIS thread. + // So to avoid the deadlock and waiting for executor to interrupt this thread, + // we delegate the shutdown call to another thread, + // so current thead can be able to be stopped properly by the executor. + asyncExecutorShutdown = true; + } else { + quickExecutor.shutdownInstance(); + } + // We can be already stopped due to a connection loss aeronContext.stopIfStarted(); } - if (connMgr.isConnected()) - connMgr.disconnected(); - userPrincipalResolver.close(); + + if (asyncExecutorShutdown) { + LOGGER.warn("Shutting down QuickExecutor from inside of event handler unsing separate thread"); + new Thread(quickExecutor::shutdownInstance, "TickDBClient-QE-Shutdown").start(); + } } public File[] getDbDirs() { @@ -1259,7 +1389,7 @@ public synchronized MetaData getMetaData () { // DisconnectableImpl.Reconnector impl. @Override - public boolean tryReconnect(int numAttempts, long timeSinceDisconnected, ReconnectableImpl helper) throws Exception { + public boolean tryReconnect(int numAttempts, long timeSinceDisconnected, TickDBReconnectableImpl helper) { if (isOpen) open(isReadOnly); @@ -1267,16 +1397,22 @@ public boolean tryReconnect(int numAttempts, long timeSinceDisconnected, Reconne } // Disconnectable impl. + /** + * Warning: code that installs this listener is also responsible to remove it eventually + * using {@link #removeDisconnectEventListener(DisconnectEventListener)}, otherwise it can cause memory leaks. + * Usually this is should be done just before closing the client or immediately after closing it. + */ @Override public void addDisconnectEventListener(DisconnectEventListener listener) { - connMgr.addDisconnectEventListener(listener); + connectionNotifier.disconnectListeners.addDisconnectEventListener(listener); } @Override public void removeDisconnectEventListener(DisconnectEventListener listener) { - connMgr.removeDisconnectEventListener(listener); + connectionNotifier.disconnectListeners.removeDisconnectEventListener(listener); } + @Override public boolean isSecured() { assertOpen(); @@ -1298,6 +1434,8 @@ public boolean isConnected() { } void onSessionDisconnected() { + // TODO: This isConnected() check should be done under lock. + // For now it's left as is to find out instability source of Test_Reconnect. if (connMgr.isConnected()) { synchronized (this) { session = new SessionClient(this, serverProtocolVersion); @@ -1310,27 +1448,36 @@ void onSessionDisconnected() { } private void onDisconnected() { - if (connMgr.isConnected()) { - connMgr.scheduleReconnect(); - // listeners can actually stop reconnecting using "close" - connMgr.disconnected(); + synchronized (this) { + boolean wasConnected = connMgr.isConnected(); - // closing session - synchronized (this) { - session = Util.close(session); + if (wasConnected) { + connMgr.scheduleReconnect(); + // listeners can actually stop reconnecting using "close" + connMgr.disconnected(); + + // closing session + if (session != null) { + LOGGER.info("Closing session due to disconnection"); + } + Util.close(session); + session = null; + connectionNotifier.addDisconnectEvent(); } } } private void onReconnected() { - if (!connMgr.isConnected()) { + synchronized (this) { + boolean wasConnected = connMgr.isConnected(); - synchronized (this) { - if (session == null) + if (!wasConnected) { + if (session == null) { session = new SessionClient(this, serverProtocolVersion); + } + connMgr.connected(); + connectionNotifier.addConnectEvent(); } - - connMgr.connected(); } } @@ -1666,6 +1813,28 @@ public TopicDB getTopicDB() { return this; } + /** + * Allows to explicitly set number of transport channels used by the client. + * Must be set before connecting to the server. + */ + public void setNumTransportChannels(Integer numTransportChannels) { + this.numTransportChannels = numTransportChannels; + } + + @VisibleForTesting + Integer getNumTransportChannels() { + return numTransportChannels; + } + + public void setReconnectIntervalAdjuster(ReconnectableImpl.ReconnectIntervalAdjuster adjuster) { + connMgr.setAdjuster(adjuster); + } + + @VisibleForTesting + ReconnectableImpl.ReconnectIntervalAdjuster getReconnectIntervalAdjuster() { + return connMgr.getAdjuster(); + } + @Override public boolean isTopicDBSupported() { return true; @@ -1684,4 +1853,110 @@ public Thread newThread(@NotNull Runnable r) { return thread; } } + + private enum ConnectionUpdateEvent { + CONNECT, + DISCONNECT + } + + /** + * Enforces sequential processing of connection status change events. + * So even if client event handlers are slow/block we still guarantee correct order of events. + *

+ * Expected behavior of TickDBClient: + *

    + *
  • Disconnect/connect events are processed sequentially in the order they were added
    • + *
  • Last event published to listeners must match actual connection status of the client
    • + *
  • User clode should not call .close() on the TB client from an event handler, + * but if it does, we should not deadlock and close used resources properly
    • + *
  • If event listener gets added before .open() then that event listener should receive event generated during that open()
    • + *
  • If event listener gets added after .open() then that event listener should not receive event generated during that open()
    • * + *
  • If event listener gets removed before .close() then that event listener should not receive event generated during that close()
    • + *
  • If event listener gets removed after .close() then that event listener should receive event generated during that close()
    • + *
+ */ + private class ConnectionNotificationTask extends QuickExecutor.QuickTask { + private final Queue taskQueue = new LinkedBlockingDeque<>(); + private final SafeDisconnectableEventHandler disconnectListeners = new com.epam.deltix.qsrv.hf.tickdb.comm.client.SafeDisconnectableEventHandler(); + private final AtomicInteger eventsAdded = new AtomicInteger(0); + private final AtomicInteger eventsProcessed = new AtomicInteger(0); + private final Object waitLock = new Object(); + + public ConnectionNotificationTask(QuickExecutor quickExecutor) { + super(quickExecutor); + } + + public int addConnectEvent() { + return addEvent(ConnectionUpdateEvent.CONNECT); + } + + public int addDisconnectEvent() { + return addEvent(ConnectionUpdateEvent.DISCONNECT); + } + + private int addEvent(ConnectionUpdateEvent connect) { + int eventId = eventsAdded.incrementAndGet(); + taskQueue.add(connect); + this.submit(); + return eventId; + } + + /** @return true if event with eventId is processed, false if timeout happened before that */ + public boolean waitForEventProcessing(int eventId, int eventHandlerWaitTimeout) { + long deadline = System.currentTimeMillis() + eventHandlerWaitTimeout; + + while (eventsProcessed.get() < eventId) { + long now = System.currentTimeMillis(); + if (now >= deadline) { + return false; + } + synchronized (waitLock) { + try { + // No need for precise wait here + waitLock.wait(deadline - now); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + } + return true; + } + + /** Waits till currently submitted events will be processed */ + public boolean waitForSubmittedEvents(int eventHandlerWaitTimeout) { + return waitForEventProcessing(eventsAdded.get(), eventHandlerWaitTimeout); + } + + @Override + public void run() { + ConnectionUpdateEvent poll; + while ((poll = taskQueue.poll()) != null) { + MutableBoolean handlerCallFlag = insideOfEventHandler.get(); + handlerCallFlag.setTrue(); + try { + switch (poll) { + case CONNECT: { + disconnectListeners.onReconnected(); + break; + } + case DISCONNECT: { + + disconnectListeners.onDisconnected(); + break; + } + default: { + LOGGER.error("Unknown connection event: " + poll); + } + } + } finally { + handlerCallFlag.setFalse(); + eventsProcessed.incrementAndGet(); + synchronized (waitLock) { + waitLock.notifyAll(); + } + } + } + } + } } \ No newline at end of file diff --git a/java/timebase/client/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/client/TickDBReconnectableImpl.java b/java/timebase/client/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/client/TickDBReconnectableImpl.java new file mode 100644 index 00000000..d295ab28 --- /dev/null +++ b/java/timebase/client/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/client/TickDBReconnectableImpl.java @@ -0,0 +1,220 @@ +package com.epam.deltix.qsrv.hf.tickdb.comm.client; + +import com.epam.deltix.gflog.api.Log; +import com.epam.deltix.gflog.api.LogFactory; +import com.epam.deltix.gflog.api.LogLevel; +import com.epam.deltix.qsrv.hf.spi.conn.ReconnectableImpl.ReconnectIntervalAdjuster; +import com.epam.deltix.util.time.GlobalTimer; +import com.epam.deltix.util.time.TimerRunner; +import net.jcip.annotations.GuardedBy; + +import java.util.Objects; +import java.util.TimerTask; + +/** + * Similar to {@link com.epam.deltix.qsrv.hf.spi.conn.ReconnectableImpl} + * but {@link #connected()} and {@link #disconnected()} do not execute callbacks directly. + */ +class TickDBReconnectableImpl { + protected static final Log LOG = LogFactory.getLog("tickdb.client"); + + private volatile long initialReconnectInterval = 5000; + private volatile ReconnectIntervalAdjuster adjuster = null; + private volatile String logprefix; + private static final LogLevel logLevel = LogLevel.DEBUG; + + private final Object lockObject; + + @GuardedBy("lockObject") + private TickDBClient reconnector = null; + + private volatile boolean isConnected = false; + + @GuardedBy("lockObject") + private long timeDisconnected; + + @GuardedBy("lockObject") + private int numReconnectAttempts; + + @GuardedBy("lockObject") + private long currentReconnectInterval; + + @GuardedBy("lockObject") + private TimerTask reconnectTask; + + @GuardedBy("lockObject") + private String lastExceptionAsString; + + /** + * @param lockObject an object to use as lock for synchronization instead of "this" + */ + public TickDBReconnectableImpl(String logprefix, Object lockObject) { + this.logprefix = logprefix; + // Use this as lock object if not provided. That's preserves old behavior. + this.lockObject = Objects.requireNonNull(lockObject, "lockObject may not be null"); + } + + public ReconnectIntervalAdjuster getAdjuster() { + return adjuster; + } + + public void setAdjuster(ReconnectIntervalAdjuster adjuster) { + this.adjuster = adjuster; + } + + public void setReconnector(TickDBClient reconnector) { + this.reconnector = reconnector; + } + + public long getInitialReconnectInterval() { + return initialReconnectInterval; + } + + public void setInitialReconnectInterval(long initialReconnectInterval) { + this.initialReconnectInterval = initialReconnectInterval; + } + + public void setLogPrefix(String logprefix) { + this.logprefix = logprefix; + } + + public void connected() { + synchronized (lockObject) { + if (reconnectTask != null) + reconnectTask.cancel(); + + isConnected = true; + lastExceptionAsString = null; + } + + LOG.log(logLevel, "[%s] Connected").with(logprefix); + + // onReconnected(); // deferred to fireOnReconnected() + } + + public void disconnected() { + synchronized (lockObject) { + isConnected = false; + timeDisconnected = System.currentTimeMillis(); + } + + LOG.log(logLevel, "[%s] Disconnected").with(logprefix); + + // onDisconnected(); // deferred to fireOnDisconnected() + } + + public boolean isConnected() { + synchronized (lockObject) { + return isConnected; + } + } + + private void tryReconnect() { + synchronized (lockObject) { + reconnectTask = null; + + boolean reschedule = false; + + if (!isConnected && reconnector != null) { + try { + reschedule = + reconnector.tryReconnect( + numReconnectAttempts, + System.currentTimeMillis() - timeDisconnected, + this + ); + } catch (Throwable x) { + String check = x.toString(); + if (check.equals(lastExceptionAsString)) { + LOG.log(logLevel, "[%s] Reconnect failed due to: %s").with(logprefix).with(lastExceptionAsString); + } else { + LOG.log(logLevel, "[%s] Reconnect failed: %s").with(logprefix).with(x); + lastExceptionAsString = check; + } + + reschedule = true; + } + + numReconnectAttempts++; + } + + if (!isConnected && reschedule) { + ReconnectIntervalAdjuster adj = adjuster; + + if (adj != null) + currentReconnectInterval = + adj.nextInterval( + numReconnectAttempts, + timeDisconnected, + currentReconnectInterval + ); + + scheduleTask(); + } + } + } + + @GuardedBy("lockObject") + private void scheduleTask() { + assert Thread.holdsLock(lockObject); + + reconnectTask = + new TimerRunner() { + @Override + public void runInternal() { + try { + tryReconnect(); + } catch (Throwable x) { + LOG.error("[%s] Unexpected: %s").with(logprefix).with(x); + } + } + }; + + GlobalTimer.INSTANCE.schedule(reconnectTask, currentReconnectInterval); + + LOG.log(logLevel, "[%s] Next reconnect in %s").with(logprefix).with(currentReconnectInterval); + } + + public void scheduleReconnect() { + synchronized (lockObject) { + if (reconnector == null) + throw new IllegalStateException("[" + logprefix + "] Call setReconnector() first."); + + if (reconnectTask != null) + reconnectTask.cancel(); + + numReconnectAttempts = 0; + currentReconnectInterval = initialReconnectInterval; + scheduleTask(); + } + } + + public void cancelReconnect() { + synchronized (lockObject) { + if (reconnectTask != null) { + reconnectTask.cancel(); + } + } + } + + interface Reconnector { + /** + * Try and reconnect. If successful, this method must call + * {@link TickDBReconnectableImpl#connected} on helper. After that, the return + * value is irrelevant. If unsucessful, this method can either throw + * an exception, or return true to reschedule the reconnect, + * or, in rare instances, return false to give up. + * + * @return Whether reconnection should be rescheduled. + * @throws Exception + * If thrown, reconnect will be rescheduled. Therefore, this + * method can freely throw exceptions due to reconnect failure. + * + */ + boolean tryReconnect( + int numAttempts, + long timeSinceDisconnected, + TickDBReconnectableImpl helper + ) throws Exception; + } +} diff --git a/java/timebase/client/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/client/UserPrincipalResolver.java b/java/timebase/client/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/client/UserPrincipalResolver.java index 553dde75..701501b8 100644 --- a/java/timebase/client/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/client/UserPrincipalResolver.java +++ b/java/timebase/client/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/client/UserPrincipalResolver.java @@ -27,26 +27,37 @@ public class UserPrincipalResolver { public static final Log LOGGER = LogFactory.getLog("tickdb.client"); private volatile Oauth2Client oauth2Client; + private volatile boolean isExternal; void setOauth2Client(Oauth2Client oauth2Client) { + this.setOauth2Client(oauth2Client, false); + } + + void setOauth2Client(Oauth2Client oauth2Client, boolean external) { + if (this.oauth2Client != null && !isExternal) { + throw new RuntimeException("OAuth2 client is not empty."); + } + this.oauth2Client = oauth2Client; + this.isExternal = external; } UserPrincipal resolve(UserPrincipal user) { if (SecretsStorage.isSecretsStorageValue(user.getPass())) { try { return new UserPrincipal( - user.getName(), - SecretsStorage.INSTANCE.getSecret(user.getPass()) + user.getName(), + SecretsStorage.INSTANCE.getSecret(user.getPass()) ); } catch (Throwable t) { LOGGER.warn().append("Failed to resolve SecretsStorage value for user ").append(user.getName()).append(t).commit(); } } - if (oauth2Client != null) { - String clientId = oauth2Client.clientId(); - String token = oauth2Client.token(); + Oauth2Client currentOauth2Client = oauth2Client; + if (currentOauth2Client != null) { + String clientId = currentOauth2Client.clientId(); + String token = currentOauth2Client.token(); return new UserPrincipal(clientId, token); } @@ -54,8 +65,9 @@ UserPrincipal resolve(UserPrincipal user) { } void close() { - if (oauth2Client != null) { - oauth2Client.close(); + Oauth2Client currentOauth2Client = oauth2Client; + if (currentOauth2Client != null && !isExternal) { + currentOauth2Client.close(); } } -} \ No newline at end of file +} diff --git a/java/timebase/commons/src/main/java/com/epam/deltix/util/Version.java b/java/timebase/commons/src/main/java/com/epam/deltix/util/Version.java index 23084720..4d17e816 100644 --- a/java/timebase/commons/src/main/java/com/epam/deltix/util/Version.java +++ b/java/timebase/commons/src/main/java/com/epam/deltix/util/Version.java @@ -17,9 +17,9 @@ public abstract class Version { public static final int MAJOR = 6; public static final int MINOR = 2; public static final String NAME = "18-SNAPSHOT"; - public static final String BUILD = "2c3be72f"; + public static final String BUILD = "f9c92576"; public static final Integer COMMITS_AFTER_TAG = null; - public static final String BUILD_DATE = "2026-06-25 21:23:51 +0300"; + public static final String BUILD_DATE = "2026-07-17 14:24:22 +0300"; public static final String VERSION_STRING; public static final String MAJOR_VERSION_STRING; diff --git a/java/timebase/commons/src/main/java/com/epam/deltix/util/collections/IntegerRingedList.java b/java/timebase/commons/src/main/java/com/epam/deltix/util/collections/IntegerRingedList.java new file mode 100644 index 00000000..0958f450 --- /dev/null +++ b/java/timebase/commons/src/main/java/com/epam/deltix/util/collections/IntegerRingedList.java @@ -0,0 +1,156 @@ +package com.epam.deltix.util.collections; + +import com.epam.deltix.util.collections.generated.IntegerList; + +import java.util.AbstractList; +import java.util.Arrays; + +public class IntegerRingedList extends AbstractList implements IntegerList { + private static final int MAX_ARRAY_LENGTH = 2147483639; + protected int[] array; + private int first; + private int size; + + public IntegerRingedList() { + this(16); + } + + public IntegerRingedList(int capacity) { + if (capacity < 0) { + throw new IllegalArgumentException("Initial capacity (" + capacity + ") is negative"); + } else { + this.array = new int[capacity]; + this.first = this.size = 0; + } + } + + public int size() { + return this.size; + } + + public Integer get(int index) { + return this.getInteger(index); + } + + public int getInteger(int index) { + if (index > this.size) { + throw new IndexOutOfBoundsException(); + } else { + return this.getIntegerNoRangeCheck(index); + } + } + + public int getIntegerNoRangeCheck(int index) { + index += this.first; + index = index >= this.array.length ? index - this.array.length : index; + return this.array[index]; + } + + public boolean add(int x) { + this.ensureFreeSpace(1); + int index = this.first + this.size++; + index = index >= this.array.length ? index - this.array.length : index; + this.array[index] = x; + return true; + } + + public void set(int index, int element) { + if (index > this.size) { + throw new IndexOutOfBoundsException(); + } else { + index += this.first; + index = index >= this.array.length ? index - this.array.length : index; + this.array[index] = element; + } + } + + public int first() { + return this.array[this.first]; + } + + public int last() { + return this.get(this.size - 1); + } + + public int pop() { + int res = this.array[this.first++]; + this.first = this.first >= this.array.length ? this.first - this.array.length : this.first; + --this.size; + return res; + } + + public boolean contains(int elem) { + return this.indexOf(elem) >= 0; + } + + public int indexOf(int elem) { + throw new UnsupportedOperationException(); + } + + public int lastIndexOf(int elem) { + throw new UnsupportedOperationException(); + } + + public int[] toIntArray() { + int[] result = new int[this.size]; + this.toArray(result, 0); + return result; + } + + public void toArray(int[] data, int offset) { + if (this.size > data.length - offset) { + throw new IllegalArgumentException(); + } else { + int i = 0; + + for(int j = this.first; i < this.size; ++i) { + data[i] = this.array[j++]; + j = j == this.array.length ? 0 : j; + } + + } + } + + private void ensureFreeSpace(int required) { + long requiredCapacity = (long)this.size + (long)required; + this.ensureCapacity(requiredCapacity); + } + + public void ensureCapacity(long minCapacity) { + if (minCapacity > (long)this.array.length) { + this.extend(minCapacity); + } + + } + + private void extend(long requiredCapacity) { + if (requiredCapacity > 2147483639L) { + throw new OutOfMemoryError("required capacity=" + requiredCapacity + " exceeds max"); + } else { + int newCapacity = (int)Math.min(2147483639L, requiredCapacity + (requiredCapacity >>> 1)); + int[] newArray = new int[newCapacity]; + System.arraycopy(this.array, this.first, newArray, 0, this.array.length - this.first); + if (this.first > 0) { + System.arraycopy(this.array, 0, newArray, this.array.length - this.first, this.first); + } + + this.first = 0; + this.array = newArray; + } + } + + public void clear() { + this.first = this.size = 0; + } + + public void setSize(int newSize) { + this.ensureCapacity((long)newSize); + this.size = newSize; + } + + public void sort() { + this.array = this.toIntArray(); + this.first = 0; + Arrays.sort(this.array, 0, this.size); + } +} \ No newline at end of file diff --git a/java/timebase/qql/src/main/java/com/epam/deltix/qsrv/hf/tickdb/lang/runtime/FilterIMSImpl.java b/java/timebase/qql/src/main/java/com/epam/deltix/qsrv/hf/tickdb/lang/runtime/FilterIMSImpl.java index 0e51a4d7..860638f3 100644 --- a/java/timebase/qql/src/main/java/com/epam/deltix/qsrv/hf/tickdb/lang/runtime/FilterIMSImpl.java +++ b/java/timebase/qql/src/main/java/com/epam/deltix/qsrv/hf/tickdb/lang/runtime/FilterIMSImpl.java @@ -32,6 +32,7 @@ import com.epam.deltix.qsrv.hf.tickdb.pub.DXTickDB; import com.epam.deltix.qsrv.hf.tickdb.pub.query.FixedMessageSource; import com.epam.deltix.qsrv.hf.tickdb.pub.query.InstrumentMessageSource; +import com.epam.deltix.qsrv.util.json.DateFormatter; import com.epam.deltix.util.collections.IndexedArrayList; import java.util.Arrays; @@ -86,6 +87,8 @@ public abstract class FilterIMSImpl private long firstMessageTimestamp = Long.MIN_VALUE; protected long aggregatedMessages; + + private final DateFormatter datetimeFormatter = new DateFormatter(); protected FilterIMSImpl ( InstrumentMessageSource source, @@ -553,4 +556,8 @@ private int getParamIndex(int n) { return n & ~(TimestampLimits.EXCLUSIVE_BIT | TimestampLimits.NANOS_BIT); } + public DateFormatter datetimeFormatter() { + return datetimeFormatter; + } + } diff --git a/java/timebase/qql/src/main/java/com/epam/deltix/qsrv/hf/tickdb/lang/runtime/FilterState.java b/java/timebase/qql/src/main/java/com/epam/deltix/qsrv/hf/tickdb/lang/runtime/FilterState.java index 02a360a7..88884f76 100644 --- a/java/timebase/qql/src/main/java/com/epam/deltix/qsrv/hf/tickdb/lang/runtime/FilterState.java +++ b/java/timebase/qql/src/main/java/com/epam/deltix/qsrv/hf/tickdb/lang/runtime/FilterState.java @@ -48,8 +48,6 @@ public abstract class FilterState { private final FilterIMSImpl filter; - private final DateFormatter datetimeFormatter = new DateFormatter(); - private final Introspector introspector = Introspector.createEmptyMessageIntrospector(); public FilterState(FilterIMSImpl filter) { @@ -126,7 +124,7 @@ public EnumClassDescriptor getEnumDescriptor(String name) { } public DateFormatter datetimeFormatter() { - return datetimeFormatter; + return filter.datetimeFormatter(); } public void setHavingAccepted(boolean havingAccepted) { diff --git a/java/timebase/qql/src/main/java/com/epam/deltix/qsrv/hf/tickdb/lang/runtime/GroupByFilterState.java b/java/timebase/qql/src/main/java/com/epam/deltix/qsrv/hf/tickdb/lang/runtime/GroupByFilterState.java index d7d30ddd..80e1d84d 100644 --- a/java/timebase/qql/src/main/java/com/epam/deltix/qsrv/hf/tickdb/lang/runtime/GroupByFilterState.java +++ b/java/timebase/qql/src/main/java/com/epam/deltix/qsrv/hf/tickdb/lang/runtime/GroupByFilterState.java @@ -28,7 +28,7 @@ */ public abstract class GroupByFilterState extends FilterState implements Serializer { - protected final MemoryDataOutput mdo = new MemoryDataOutput(); + protected final MemoryDataOutput mdo = new MemoryDataOutput(16); protected final MemoryDataInput mdi = new MemoryDataInput(); public GroupByFilterState(FilterIMSImpl filter) { diff --git a/java/timebase/server/src/main/java/com/epam/deltix/qsrv/config/TimebaseServiceExecutor.java b/java/timebase/server/src/main/java/com/epam/deltix/qsrv/config/TimebaseServiceExecutor.java index 6cd42b1e..40b5fdb5 100644 --- a/java/timebase/server/src/main/java/com/epam/deltix/qsrv/config/TimebaseServiceExecutor.java +++ b/java/timebase/server/src/main/java/com/epam/deltix/qsrv/config/TimebaseServiceExecutor.java @@ -183,15 +183,16 @@ public void run() { String compression = config.getString("compression", VSCompression.AUTO.toString()); int maxConnections = config.getInt("maxConnections", VSServerFramework.MAX_CONNECTIONS); - short maxSocketsPerConnection = (short) config.getInt("maxSocketsPerConnection", VSServerFramework.MAX_SOCKETS_PER_CONNECTION); + short maxChannels = (short) config.getInt("maxChannelsPerConnection", -1); + short maxSocketsPerConnection = (short) config.getInt("maxSocketsPerConnection", Runtime.getRuntime().availableProcessors() * 2); contextContainer.getQuickExecutor().reuseInstance(); VSServerFramework framework = new VSServerFramework(contextContainer.getQuickExecutor(), (int) interval.toMilliseconds(), Enum.valueOf(VSCompression.class, compression), - maxConnections, - maxSocketsPerConnection, contextContainer, DefaultConnectionAcceptor.INSTANCE); + maxConnections, maxSocketsPerConnection, maxChannels, + contextContainer, DefaultConnectionAcceptor.INSTANCE); if (framework.getCompression() == VSCompression.OFF) LOGGER.info("Timebase communication compression disabled."); diff --git a/java/timebase/server/src/main/java/com/epam/deltix/qsrv/dtb/store/dataacc/DataReaderImpl.java b/java/timebase/server/src/main/java/com/epam/deltix/qsrv/dtb/store/dataacc/DataReaderImpl.java index 446c804f..d4bde6ff 100644 --- a/java/timebase/server/src/main/java/com/epam/deltix/qsrv/dtb/store/dataacc/DataReaderImpl.java +++ b/java/timebase/server/src/main/java/com/epam/deltix/qsrv/dtb/store/dataacc/DataReaderImpl.java @@ -423,6 +423,11 @@ public void checkoutForRead(TimeSlice slice) { } + @Override + public void onClosed(TimeSlice slice) { + notifier.submit(); + } + // // DAPrivate IMPLEMENTATION // diff --git a/java/timebase/server/src/main/java/com/epam/deltix/qsrv/dtb/store/dataacc/LiveDataReaderImpl.java b/java/timebase/server/src/main/java/com/epam/deltix/qsrv/dtb/store/dataacc/LiveDataReaderImpl.java index cb19038e..e3c0eff0 100644 --- a/java/timebase/server/src/main/java/com/epam/deltix/qsrv/dtb/store/dataacc/LiveDataReaderImpl.java +++ b/java/timebase/server/src/main/java/com/epam/deltix/qsrv/dtb/store/dataacc/LiveDataReaderImpl.java @@ -17,9 +17,11 @@ package com.epam.deltix.qsrv.dtb.store.dataacc; import com.epam.deltix.qsrv.dtb.store.pub.*; +import com.epam.deltix.util.collections.IntegerRingedList; import com.epam.deltix.util.collections.generated.IntegerEnumeration; import com.epam.deltix.util.collections.generated.IntegerToObjectHashMap; import com.epam.deltix.util.concurrent.*; +import net.jcip.annotations.GuardedBy; /** * @@ -37,15 +39,18 @@ public final class LiveDataReaderImpl // incoming updated blocks private final IntegerToObjectHashMap waiting = new IntegerToObjectHashMap<>(); - private final IntegerEnumeration e = waiting.keys(); + + // Keys of "waiting" map. Must be updated together with "waiting" map. + @GuardedBy("waiting") + private final IntegerRingedList waitingList = new IntegerRingedList(); private volatile Runnable listener; private final QuickExecutor.QuickTask notifier; public LiveDataReaderImpl(QuickExecutor exe) { - notifier = new QuickExecutor.QuickTask (exe) { + notifier = new QuickExecutor.QuickTask (exe) { @Override - public void run () { + public void run() { Runnable consistent = listener; if (consistent != null) { @@ -84,7 +89,7 @@ protected void closeInternal () { currentTimestamp = Long.MAX_VALUE; synchronized (waiting) { - waiting.clear(); + clearWaiting(); } super.close (); @@ -385,19 +390,23 @@ public void asyncDataInserted(DataBlock db, int dataOffset, int private void offerWaiting(DataBlock db) { synchronized (waiting) { - waiting.put(db.getEntity(), db); + int key = db.getEntity(); + boolean added = waiting.put(key, db); + if (added) { + waitingList.add(key); + } } } private DataBlock pollWaiting() { synchronized (waiting) { - e.reset(); + if (waitingList.isEmpty()) { + return null; + } + int key = waitingList.pop(); - if (e.hasMoreElements()) - return waiting.remove(e.nextIntElement(), null); + return waiting.remove(key, null); } - - return null; } private void clearCurrent() { @@ -407,12 +416,33 @@ private void clearCurrent() { } synchronized (waiting) { - waiting.clear(); + clearWaiting(); } clearLinks(); } + /** + * Cleans "waiting" map. + */ + @GuardedBy("waiting") + private void clearWaiting() { + int count = waitingList.size(); + if (count * 16 < waiting.getCapacity()) { + // This means that map's hash table is sparse, + // and we can clean it faster by looking up individual keys instead of iterating over all entries. + while (!waitingList.isEmpty()) { + int key = waitingList.pop(); + waiting.remove(key, null); + } + assert waiting.isEmpty(); + } else { + // Use regular clear (iterates over all hash table cells, but this is faster when map is dense). + waiting.clear(); + } + waitingList.clear(); + } + @Override public synchronized void associate(TimeSlice slice) { super.associate(slice); @@ -430,6 +460,11 @@ public void checkoutForRead(TimeSlice slice) { } + @Override + public void onClosed(TimeSlice slice) { + notifier.submit(); + } + // // DAPrivate IMPLEMENTATION // diff --git a/java/timebase/server/src/main/java/com/epam/deltix/qsrv/dtb/store/dataacc/SliceListener.java b/java/timebase/server/src/main/java/com/epam/deltix/qsrv/dtb/store/dataacc/SliceListener.java index d03d7914..3c78a6ca 100644 --- a/java/timebase/server/src/main/java/com/epam/deltix/qsrv/dtb/store/dataacc/SliceListener.java +++ b/java/timebase/server/src/main/java/com/epam/deltix/qsrv/dtb/store/dataacc/SliceListener.java @@ -18,7 +18,21 @@ public interface SliceListener { + /** + * Invoked when new slice is checked out for inserting data + * @param slice slice + */ void checkoutForInsert(TimeSlice slice); + /** + * Invoked when new slice is checked out for reading data + * @param slice slice + */ void checkoutForRead(TimeSlice slice); + + /** + * Invoked by TSRoot when slice is removed. + * @param slice slice that was changed, null if whole TSRoot is removed + */ + void onClosed(TimeSlice slice); } \ No newline at end of file diff --git a/java/timebase/server/src/main/java/com/epam/deltix/qsrv/dtb/store/impl/TSRootFolder.java b/java/timebase/server/src/main/java/com/epam/deltix/qsrv/dtb/store/impl/TSRootFolder.java index 74fc314a..72f166c3 100644 --- a/java/timebase/server/src/main/java/com/epam/deltix/qsrv/dtb/store/impl/TSRootFolder.java +++ b/java/timebase/server/src/main/java/com/epam/deltix/qsrv/dtb/store/impl/TSRootFolder.java @@ -242,16 +242,23 @@ private void storeRegistry() { } @Override - public synchronized void forceClose() { + public void forceClose() { if (!isOpen) return; - if (isActive()) - LOGGER.warn().append("FORCE-Closing ").append(this).append(" while in active state").commit(); + synchronized (sliceListeners) { + for (int i = 0; i < sliceListeners.size(); i++) + sliceListeners.get(i).onClosed(null); + } - storeRegistry(); - symRegistry.close(); - isOpen = false; + synchronized (this) { + if (isActive()) + LOGGER.warn().append("FORCE-Closing ").append(this).append(" while in active state").commit(); + + storeRegistry(); + symRegistry.close(); + isOpen = false; + } } @Override @@ -877,7 +884,7 @@ TSFile split(long nstime, TSFile file, DAPrivate accessor) throws IOException { return file; } - public final ArrayList sliceListeners = new ArrayList(5); + public final ArrayList sliceListeners = new ArrayList<>(5); @Override public void addSliceListener(SliceListener listener) { diff --git a/java/timebase/server/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/server/DownloadHandlerFactory.java b/java/timebase/server/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/server/DownloadHandlerFactory.java index 465f56e1..7258b944 100644 --- a/java/timebase/server/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/server/DownloadHandlerFactory.java +++ b/java/timebase/server/src/main/java/com/epam/deltix/qsrv/hf/tickdb/comm/server/DownloadHandlerFactory.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 EPAM Systems, Inc + * Copyright 2026 EPAM Systems, Inc * * See the NOTICE file distributed with this work for additional information * regarding copyright ownership. Licensed under the Apache License, @@ -17,7 +17,6 @@ package com.epam.deltix.qsrv.hf.tickdb.comm.server; import com.epam.deltix.qsrv.hf.pub.ChannelPerformance; -import com.epam.deltix.timebase.messages.IdentityKey; import com.epam.deltix.qsrv.hf.pub.RawMessage; import com.epam.deltix.qsrv.hf.tickdb.comm.SelectionOptionsCodec; import com.epam.deltix.qsrv.hf.tickdb.comm.TDBProtocol; @@ -43,6 +42,8 @@ import java.io.DataOutputStream; import java.io.IOException; import java.security.Principal; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import static com.epam.deltix.qsrv.hf.tickdb.comm.TDBProtocol.TRANSPORT_TYPE_AERON; import static com.epam.deltix.qsrv.hf.tickdb.comm.TDBProtocol.TRANSPORT_TYPE_SOCKET; @@ -53,9 +54,44 @@ * @author Alexei Osipov */ public class DownloadHandlerFactory { + + public static final int CURSORS_PER_CONNECTION = Integer.getInteger("TimeBase.maxCursorsPerConnection", -1); + + private static final ConcurrentHashMap limits = new ConcurrentHashMap<>(); + + static boolean checkLimits(SelectionOptions options, VSChannel ds) { + + if (options.live || CURSORS_PER_CONNECTION < 0) + return false; + + // limits check + AtomicInteger actual = new AtomicInteger(0); + + AtomicInteger value = limits.putIfAbsent(ds.getClientId(), actual); + if (value == null) + value = actual; + + if (value.incrementAndGet() >= CURSORS_PER_CONNECTION) { + value.decrementAndGet(); + throw new RuntimeException("Unable to create cursor due to limits: " + CURSORS_PER_CONNECTION); + } + + ds.addDisposableListener(DownloadHandlerFactory::cursorClosed); + return true; + } + + static void cursorClosed(VSChannel c) { + AtomicInteger value = limits.get(c.getClientId()); + if (value != null) { + if (value.decrementAndGet() < 0) + value.incrementAndGet(); + } + } + public static void start(Principal user, VSChannel ds, DXTickDB db, QuickExecutor executor, TimebaseAccessController ac, int clientVersion, AeronThreadTracker aeronThreadTracker, DXServerAeronContext aeronContext) throws IOException { boolean aeronSupported = clientVersion >= TDBProtocol.AERON_SUPPORT_VERSION; int requestedTransportType = TRANSPORT_TYPE_SOCKET; + if (aeronSupported) { DataInputStream din = ds.getDataInputStream(); requestedTransportType = din.read(); @@ -106,6 +142,8 @@ public static void start(Principal user, VSChannel ds, DXTickDB db, QuickExecuto UserLogger.trace(user, ds.getRemoteAddress(), ds.getRemoteApplication(), UserLogger.CREATE_CURSOR_PATTERN, qql); try { + boolean limited = checkLimits(options, ds); + cursor = db.executeQuery (qql, options, streams, ids, initTime, endTimestamp, params); } catch (CompilationException x) { UserLogger.warn(user, ds.getRemoteAddress(), ds.getRemoteApplication(), "Query stream error: ", x); @@ -125,6 +163,8 @@ public static void start(Principal user, VSChannel ds, DXTickDB db, QuickExecuto } else { try { + boolean limited = checkLimits(options, ds); + if (streams == null) cursor = tcursor = db.select (initTime, options, messageTypes, ids); else diff --git a/java/timebase/server/src/main/java/com/epam/deltix/qsrv/hf/tickdb/impl/PDStreamSource.java b/java/timebase/server/src/main/java/com/epam/deltix/qsrv/hf/tickdb/impl/PDStreamSource.java index 239671d3..5419f28e 100644 --- a/java/timebase/server/src/main/java/com/epam/deltix/qsrv/hf/tickdb/impl/PDStreamSource.java +++ b/java/timebase/server/src/main/java/com/epam/deltix/qsrv/hf/tickdb/impl/PDStreamSource.java @@ -471,7 +471,21 @@ public boolean spaceCreated(String space) { if (!isSubscribed(space)) return false; - long timestamp = mx.getCurrentTime(); + long timestamp = options.reversed ? Long.MIN_VALUE : Long.MAX_VALUE; + + if (options.allowLateOutOfOrder) { + + // find timestamp from sources + for (SourceSubscription source : sources) + timestamp = options.reversed ? Math.max(timestamp, source.timestamp) : Math.min(timestamp, source.timestamp); + + if (TimeStamp.isUndefined(timestamp)) + timestamp = mx.getCurrentTime(); + + } else { + timestamp = mx.getCurrentTime(); + } + long nstime = TimeStamp.getNanoTime(timestamp); SourceSubscription sub = isSubscribedToAllEntities ? diff --git a/java/timebase/test/build.gradle b/java/timebase/test/build.gradle index 9803950a..7e94a853 100644 --- a/java/timebase/test/build.gradle +++ b/java/timebase/test/build.gradle @@ -45,4 +45,6 @@ dependencies { //testImplementation 'org.f1x:f1x' testImplementation 'org.jetbrains:annotations' + + testImplementation 'com.github.netcrusherorg:netcrusher-core:0.10' } \ No newline at end of file diff --git a/java/timebase/test/src/test/java/com/epam/deltix/test/qsrv/hf/tickdb/LiveCursorTestBase.java b/java/timebase/test/src/test/java/com/epam/deltix/test/qsrv/hf/tickdb/LiveCursorTestBase.java index 37138001..67706558 100644 --- a/java/timebase/test/src/test/java/com/epam/deltix/test/qsrv/hf/tickdb/LiveCursorTestBase.java +++ b/java/timebase/test/src/test/java/com/epam/deltix/test/qsrv/hf/tickdb/LiveCursorTestBase.java @@ -16,6 +16,7 @@ */ package com.epam.deltix.test.qsrv.hf.tickdb; +import com.epam.deltix.qsrv.QSHome; import com.epam.deltix.qsrv.hf.tickdb.StreamConfigurationHelper; import com.epam.deltix.qsrv.hf.tickdb.TDBRunner; import com.epam.deltix.qsrv.hf.tickdb.pub.*; @@ -28,6 +29,7 @@ import com.epam.deltix.util.collections.*; import com.epam.deltix.util.lang.Util; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.Random; @@ -58,8 +60,15 @@ public abstract class LiveCursorTestBase SYMBOLS [ii] = "DLX" + ii; // Deltix' subsidiaries :) } - protected final DXTickDB localDB = TickDBFactory.create (TDBRunner.getTemporaryLocation()); - private TradeMessage generatorMessage = new TradeMessage(); + protected final DXTickDB localDB; + protected final String location; + + { + location = TDBRunner.getTemporaryLocation(); + localDB = TickDBFactory.create (location); + } + + private TradeMessage generatorMessage = new TradeMessage(); private int generatorCount; public static long countToTime (int count) { @@ -68,7 +77,7 @@ public static long countToTime (int count) { public static class Reader extends TestThread { private final TickStream stream; - private IdentityKey[] entities; + private IdentityKey[] entities; //private final FeedFilter filter; private final int cursorOpenDelay; private final int initialReadDelay; @@ -200,6 +209,8 @@ protected void addRandomReaders ( @Before public void setup () { + QSHome.set(new File(location).getParent()); + localDB.format (); DXTickStream ds = diff --git a/java/timebase/test/src/test/java/com/epam/deltix/test/qsrv/hf/tickdb/Test_MultipleLoaders.java b/java/timebase/test/src/test/java/com/epam/deltix/test/qsrv/hf/tickdb/Test_MultipleLoaders.java index 5f3d53d9..8c04b53b 100644 --- a/java/timebase/test/src/test/java/com/epam/deltix/test/qsrv/hf/tickdb/Test_MultipleLoaders.java +++ b/java/timebase/test/src/test/java/com/epam/deltix/test/qsrv/hf/tickdb/Test_MultipleLoaders.java @@ -62,8 +62,9 @@ public void test2Remote() throws InterruptedException { public void runTest(DXTickDB db) { - DXTickStream stream = db.createStream("test", - StreamOptions.fixedType(StreamScope.DURABLE, "test", "test", 0, + String name = "test"; + DXTickStream stream = db.createStream(name, + StreamOptions.fixedType(StreamScope.DURABLE, name, name, 0, StreamConfigurationHelper.mkBarMessageDescriptor(null, null, null, "DECIMAL(4)", "DECIMAL(0)"))); @@ -123,8 +124,10 @@ public void onError(LoadingError e) { public void runTest2(DXTickDB db) throws InterruptedException { final AtomicInteger expected = new AtomicInteger (199); - DXTickStream stream = db.createStream("test", - StreamOptions.fixedType(StreamScope.DURABLE, "test", "test", 0, + String name = "test2"; + + DXTickStream stream = db.createStream(name, + StreamOptions.fixedType(StreamScope.DURABLE, name, name, 0, StreamConfigurationHelper.mkBarMessageDescriptor(null, null, null, "DECIMAL(4)", "DECIMAL(0)"))); diff --git a/java/timebase/test/src/test/java/com/epam/deltix/test/qsrv/hf/tickdb/Test_SlowLoader.java b/java/timebase/test/src/test/java/com/epam/deltix/test/qsrv/hf/tickdb/Test_SlowLoader.java index 581b7713..ae7f8aa6 100644 --- a/java/timebase/test/src/test/java/com/epam/deltix/test/qsrv/hf/tickdb/Test_SlowLoader.java +++ b/java/timebase/test/src/test/java/com/epam/deltix/test/qsrv/hf/tickdb/Test_SlowLoader.java @@ -17,6 +17,7 @@ package com.epam.deltix.test.qsrv.hf.tickdb; +import com.epam.deltix.qsrv.QSHome; import com.epam.deltix.qsrv.hf.tickdb.StreamConfigurationHelper; import com.epam.deltix.qsrv.hf.tickdb.TDBRunner; import com.epam.deltix.qsrv.hf.tickdb.comm.client.TickDBClient; @@ -46,6 +47,8 @@ public class Test_SlowLoader { @Test public void go () throws InterruptedException { + QSHome.set(dbFile.getParent()); + DXTickDB db = TickDBFactory.create (dbFile); db.format (); diff --git a/java/timebase/test/src/test/java/com/epam/deltix/test/qsrv/hf/tickdb/Test_TickDBClient_Reconnect.java b/java/timebase/test/src/test/java/com/epam/deltix/test/qsrv/hf/tickdb/Test_TickDBClient_Reconnect.java new file mode 100644 index 00000000..c60961e3 --- /dev/null +++ b/java/timebase/test/src/test/java/com/epam/deltix/test/qsrv/hf/tickdb/Test_TickDBClient_Reconnect.java @@ -0,0 +1,269 @@ +package com.epam.deltix.test.qsrv.hf.tickdb; + +import com.epam.deltix.gflog.api.Log; +import com.epam.deltix.gflog.api.LogFactory; +import com.epam.deltix.gflog.jul.JulBridge; +import com.epam.deltix.qsrv.hf.pub.md.FloatDataType; +import com.epam.deltix.qsrv.hf.pub.md.RecordClassDescriptor; +import com.epam.deltix.qsrv.hf.spi.conn.DisconnectEventListener; +import com.epam.deltix.qsrv.hf.tickdb.StreamConfigurationHelper; +import com.epam.deltix.qsrv.hf.tickdb.TDBRunner; +import com.epam.deltix.qsrv.hf.tickdb.comm.client.TickDBClient; +import com.epam.deltix.qsrv.hf.tickdb.comm.server.TomcatServer; +import com.epam.deltix.qsrv.hf.tickdb.pub.*; +import com.epam.deltix.qsrv.hf.tickdb.pub.lock.DBLock; +import com.epam.deltix.qsrv.hf.tickdb.pub.lock.LockType; +import com.epam.deltix.util.JUnitCategories; +import org.jetbrains.annotations.NotNull; +import org.junit.*; +import org.junit.experimental.categories.Category; +import org.netcrusher.core.reactor.NioReactor; +import org.netcrusher.tcp.TcpCrusher; +import org.netcrusher.tcp.TcpCrusherBuilder; + +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; + +import static org.junit.Assert.*; + +/** + * Contains tests that use an intermediate proxy to simulate network issues between {@link TickDBClient} and server. + */ +@Category(JUnitCategories.TickDBFast.class) +public class Test_TickDBClient_Reconnect { + static { + JulBridge.install(); + } + private static final Log LOG = LogFactory.getLog(Test_TickDBClient_Reconnect.class); + + private static final String REMOTE_SERVER_HOST = System.getProperty("ClientReconnectTest.host"); + private static final int REMOTE_SERVER_PORT = Integer.getInteger("ClientReconnectTest.port", 8011); + + public static final int PROXY_PORT = 34781; + + private static final boolean USE_EMBEDDED = REMOTE_SERVER_HOST == null; + + + private static TDBRunner runner; + private NioReactor nioReactor; + private TcpCrusher tcpCrusher; + + private volatile Consumer proxyConnectListener = null; + + + /** + * It should be possible to acquire a lock on a stream after client lost connection, + * partially reconnected but failed to restore all transports. + */ + @Test(timeout = 400_000) // 10 iterations x 40 seconds each + public void testLockReleasedOnPartialReconnect() throws Exception { + for (int i = 0; i < 10; i++) { + LOG.info("=== Test iteration %s ===").with(i + 1); + long startTime = System.currentTimeMillis(); + testLockReleasedOnPartialReconnectIteration(); + LOG.info("=== Iteration %s completed in %s ms ===").with(i + 1).with(System.currentTimeMillis() - startTime); + } + } + + private void testLockReleasedOnPartialReconnectIteration() throws Exception { + String streamKey = "stream1"; + int transports = 16; // Current TB transport limit + + int halfTransports = transports / 2; + if (halfTransports == 0) { + throw new IllegalStateException("Number of transports is too low for this test"); + } + + try (TickDBClient client = (TickDBClient) connectClient()) { + client.setNumTransportChannels(transports); + client.setReconnectIntervalAdjuster((numAttempts, timeSinceDisconnected, lastInterval) -> { + // Effectively disable further reconnections + return java.util.concurrent.TimeUnit.DAYS.toMillis(1); + }); + client.open(false); + assertTrue(client.isConnected()); + assertEquals(transports, getConnectedClientCount()); + + LOG.info("Client connected with %s transports").with(transports); + + StreamOptions options = getTestStreamOptions(); + DXTickStream stream = client.createStream(streamKey, options); + + DBLock lock = stream.lock(LockType.WRITE); + assertNotNull(lock); + LOG.info("Acquired write lock on stream"); + + List initialConnections = new ArrayList<>(tcpCrusher.getClientAddresses()); + + + // Allow only one reconnection + AtomicInteger reconnectedCount = new AtomicInteger(0); + proxyConnectListener = (inetSocketAddress) -> { + // Warning: this listener is asynchronous, so it's not guaranteed that exactly 500 connections will fail + + int newCount = reconnectedCount.incrementAndGet(); + + if (newCount == halfTransports) { + // Disable new connections + tcpCrusher.getAcceptorFreezer().freeze(); + } + }; + + LOG.info("Killing initial connections"); + for (InetSocketAddress address : initialConnections) { + tcpCrusher.closeClient(address); + } + + while (reconnectedCount.get() < halfTransports) { + LOG.info("Waiting for %s transports to reconnect, current: %s") + .with(halfTransports).with(reconnectedCount.get()); + Thread.sleep(1000); + } + + // Wait until dispatcher detects broken transport + LOG.info("Waiting for dispatcher to detect disconnection"); + long now = System.currentTimeMillis(); + long start = now; + long deadline = now + 20_000; + while ((now = System.currentTimeMillis()) < deadline) { + if (client.isConnected()) { + Thread.sleep(100); + } else { + break; + } + } + assertFalse(client.isConnected()); + LOG.info("Waited %s ms for dispatcher to detect disconnection").with(now - start); + + + + LOG.info("Client connected: %s").with(client.isConnected()); + + tcpCrusher.close(); + + LOG.info("Attempting to close client"); + client.close(); + } + + tcpCrusher.open(); + + try (TickDBClient client2 = (TickDBClient) connectClient()) { + LOG.info("Opening client again"); + client2.open(false); + assertTrue(client2.isConnected()); + LOG.info("Client 2 connected"); + + DXTickStream stream = client2.getStream(streamKey); + DBLock lock2 = stream.lock(LockType.WRITE); + assertNotNull(lock2); + LOG.info("Acquired write lock on stream with client 2"); + + stream.delete(); + } + + tcpCrusher.reopen(); + } + + + @NotNull + private static StreamOptions getTestStreamOptions() { + RecordClassDescriptor mcd = StreamConfigurationHelper.mkMarketMessageDescriptor(null, false); + RecordClassDescriptor rcd = StreamConfigurationHelper.mkTradeMessageDescriptor( + mcd, null, null, FloatDataType.ENCODING_SCALE_AUTO, FloatDataType.ENCODING_SCALE_AUTO); + + return StreamOptions.fixedType(StreamScope.DURABLE, "message", "message", 0, rcd); + } + + @NotNull + private static RemoteTickDB connectClient() { + return TickDBFactory.connect("localhost", PROXY_PORT, false); + } + + private int getConnectedClientCount() { + return tcpCrusher.getClientAddresses().size(); + } + + @BeforeClass + public static void startClass() throws Throwable { + if (USE_EMBEDDED) { + runner = new TDBRunner(true, true, TDBRunner.getTemporaryLocation(), new TomcatServer()); + runner.startup(); + } + } + + @Before + public void start() throws Throwable { + int serverPort = USE_EMBEDDED ? runner.getPort() : REMOTE_SERVER_PORT; + String serverHost = USE_EMBEDDED ? "localhost" : REMOTE_SERVER_HOST; + + this.nioReactor = new NioReactor(); + + this.tcpCrusher = TcpCrusherBuilder.builder() + .withReactor(nioReactor) + .withBindAddress("localhost", PROXY_PORT) + .withConnectAddress(serverHost, serverPort) + .withCreationListener(clientAddress -> { + LOG.info("Proxy: Client connected: %s").with(clientAddress); + if (proxyConnectListener != null) { + proxyConnectListener.accept(clientAddress); + } + }) + .buildAndOpen(); + + } + + @After + public void stop() throws Throwable { + proxyConnectListener = null; + if (tcpCrusher != null) { + tcpCrusher.close(); + } + if (nioReactor != null) { + nioReactor.close(); + } + } + + + + @AfterClass + public static void stopClass() throws Throwable { + if (USE_EMBEDDED && runner != null) { + runner.shutdown(); + runner = null; + } + } + + private TestEventListener installListener(RemoteTickDB client) { + TestEventListener eventListener = new TestEventListener(); + client.addDisconnectEventListener(eventListener); + return eventListener; + } + + private static class TestEventListener implements DisconnectEventListener { + CountDownLatch disconnectedLatch = new CountDownLatch(1); + CountDownLatch reconnectedLatch = new CountDownLatch(1); + AtomicInteger disconnectCount = new AtomicInteger(0); + AtomicInteger reconnectCount = new AtomicInteger(0); + + public TestEventListener() { + } + + @Override + public void onDisconnected() { + LOG.info("Client disconnected"); + disconnectCount.incrementAndGet(); + disconnectedLatch.countDown(); + } + + @Override + public void onReconnected() { + LOG.info("Client reconnected"); + reconnectCount.incrementAndGet(); + reconnectedLatch.countDown(); + } + } +}