From 5691897661e836317bd8d73798ad948e0ed4c72e Mon Sep 17 00:00:00 2001 From: Ujjwal Date: Mon, 2 Jun 2025 13:36:35 +0530 Subject: [PATCH 01/83] ssl client principal propagation Signed-off-by: Ujjwal --- .../proxy/filter/FilterContext.java | 3 + .../proxy/internal/FilterHandler.java | 10 ++- .../internal/KafkaProxyFrontendHandler.java | 31 ++++++++ .../proxy/internal/FilterHarness.java | 2 +- .../KafkaProxyFrontendHandlerTest.java | 74 +++++++++++++++++-- 5 files changed, 111 insertions(+), 9 deletions(-) diff --git a/kroxylicious-api/src/main/java/io/kroxylicious/proxy/filter/FilterContext.java b/kroxylicious-api/src/main/java/io/kroxylicious/proxy/filter/FilterContext.java index cbaba8d63a..f8bfda2ca3 100644 --- a/kroxylicious-api/src/main/java/io/kroxylicious/proxy/filter/FilterContext.java +++ b/kroxylicious-api/src/main/java/io/kroxylicious/proxy/filter/FilterContext.java @@ -42,6 +42,9 @@ public interface FilterContext { @Nullable String sniHostname(); + @Nullable + String downStreamCertificatePrincipal(); + /** * Creates a builder for a request filter result objects. This object encapsulates * the request to forward and optionally orders for actions such as closing diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/FilterHandler.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/FilterHandler.java index a407709a02..fdca2a5bfe 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/FilterHandler.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/FilterHandler.java @@ -59,6 +59,7 @@ public class FilterHandler extends ChannelDuplexHandler { private static final Logger LOGGER = LoggerFactory.getLogger(FilterHandler.class); private final long timeoutMs; private final String sniHostname; + private final String downStreamCertificatePrincipal; private final VirtualClusterModel virtualClusterModel; private final Channel inboundChannel; private final FilterAndInvoker filterAndInvoker; @@ -67,10 +68,11 @@ public class FilterHandler extends ChannelDuplexHandler { private ChannelHandlerContext ctx; private PromiseFactory promiseFactory; - public FilterHandler(FilterAndInvoker filterAndInvoker, long timeoutMs, String sniHostname, VirtualClusterModel virtualClusterModel, Channel inboundChannel) { + public FilterHandler(FilterAndInvoker filterAndInvoker, long timeoutMs, String sniHostname, String downStreamCertificatePrincipal, VirtualClusterModel virtualClusterModel, Channel inboundChannel) { this.filterAndInvoker = Objects.requireNonNull(filterAndInvoker); this.timeoutMs = Assertions.requireStrictlyPositive(timeoutMs, "timeout"); this.sniHostname = sniHostname; + this.downStreamCertificatePrincipal = downStreamCertificatePrincipal; this.virtualClusterModel = virtualClusterModel; this.inboundChannel = inboundChannel; } @@ -475,6 +477,12 @@ public String sniHostname() { return sniHostname; } + @Nullable + @Override + public String downStreamCertificatePrincipal() { + return downStreamCertificatePrincipal; + } + public String getVirtualClusterName() { return virtualClusterModel.getClusterName(); } diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java index e3838c1e3a..bebc763c38 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java @@ -13,6 +13,8 @@ import java.util.List; import java.util.Objects; +import javax.net.ssl.SSLPeerUnverifiedException; + import org.apache.kafka.common.message.ApiVersionsRequestData; import org.apache.kafka.common.message.ApiVersionsResponseData; import org.apache.kafka.common.message.ApiVersionsResponseDataJsonConverter; @@ -34,6 +36,7 @@ import io.netty.handler.logging.LoggingHandler; import io.netty.handler.ssl.SniCompletionEvent; import io.netty.handler.ssl.SslHandler; +import io.netty.handler.ssl.SslHandshakeCompletionEvent; import io.kroxylicious.proxy.filter.FilterAndInvoker; import io.kroxylicious.proxy.filter.NetFilter; @@ -85,6 +88,22 @@ public class KafkaProxyFrontendHandler private @Nullable AuthenticationEvent authentication; private @Nullable String sniHostname; + @Nullable + public String getDownStreamCertificatePrincipal() { + return downStreamCertificatePrincipal; + } + + private @Nullable String downStreamCertificatePrincipal; + + @VisibleForTesting + SslHandler getSslHandler(ChannelHandlerContext ctx) { + if (ctx == null) { + throw new IllegalStateException("No context available"); + } + SslHandler sslHandler = ctx.pipeline().get(SslHandler.class); + return sslHandler; + } + // Flag if we receive a channelReadComplete() prior to outbound connection activation // so we can perform the channelReadComplete()/outbound flush & auto_read // once the outbound channel is active @@ -168,6 +187,17 @@ public void userEventTriggered( else if (event instanceof AuthenticationEvent authenticationEvent) { this.authentication = authenticationEvent; } + else if (event instanceof SslHandshakeCompletionEvent sslHandshakeCompletionEvent) { + if (sslHandshakeCompletionEvent.isSuccess()) { + SslHandler sslHandler = getSslHandler(ctx); + try { + downStreamCertificatePrincipal = sslHandler.engine().getSession().getPeerPrincipal().toString(); + } catch (SSLPeerUnverifiedException e) { + LOGGER.debug("No client principal received, setting principal as ANONYMOUS"); + downStreamCertificatePrincipal = "ANONYMOUS"; + } + } + } super.userEventTriggered(ctx, event); } @@ -625,6 +655,7 @@ private void addFiltersToPipeline( protocolFilter, 20000, sniHostname, + downStreamCertificatePrincipal, virtualClusterModel, inboundChannel)); } diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/FilterHarness.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/FilterHarness.java index ef5c189ee1..85e5cde488 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/FilterHarness.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/FilterHarness.java @@ -91,7 +91,7 @@ protected void buildChannel(Filter... filters) { return d2; })) // reverses order .stream() - .map(f -> new FilterHandler(getOnlyElement(FilterAndInvoker.build(f.getClass().getSimpleName(), f)), timeoutMs, null, testVirtualCluster, inboundChannel)) + .map(f -> new FilterHandler(getOnlyElement(FilterAndInvoker.build(f.getClass().getSimpleName(), f)), timeoutMs, null, null, testVirtualCluster, inboundChannel)) .map(ChannelHandler.class::cast); var handlers = Stream.concat(channelProcessors, filterHandlers); diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandlerTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandlerTest.java index 2def5d2473..e238c450b5 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandlerTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandlerTest.java @@ -5,6 +5,7 @@ */ package io.kroxylicious.proxy.internal; +import java.security.Principal; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -12,6 +13,10 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; +import javax.net.ssl.SSLEngine; +import javax.net.ssl.SSLPeerUnverifiedException; +import javax.net.ssl.SSLSession; + import org.apache.kafka.common.message.ApiVersionsRequestData; import org.apache.kafka.common.message.MetadataRequestData; import org.apache.kafka.common.message.RequestHeaderData; @@ -43,6 +48,8 @@ import io.netty.handler.codec.haproxy.HAProxyProxiedProtocol; import io.netty.handler.ssl.SniCompletionEvent; import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.SslHandler; +import io.netty.handler.ssl.SslHandshakeCompletionEvent; import io.kroxylicious.proxy.filter.NetFilter; import io.kroxylicious.proxy.frame.DecodedFrame; @@ -75,9 +82,12 @@ class KafkaProxyFrontendHandlerTest { public static final String SNI_HOSTNAME = "external.example.com"; public static final String CLUSTER_HOST = "internal.example.org"; + public static final String CLIENT_PRINCIPAL = "CN=client,O=example"; public static final int CLUSTER_PORT = 9092; EmbeddedChannel inboundChannel; EmbeddedChannel outboundChannel; + SSLSession sslSession = mock(SSLSession.class); + Principal mockPrincipal = mock(Principal.class); int corrId = 0; private final AtomicReference connectContext = new AtomicReference<>(); @@ -122,12 +132,14 @@ public void closeChannel() { public static List provideArgsForExpectedFlow() { var result = new ArrayList(); boolean[] tf = { true, false }; - for (boolean sslConfigured : tf) { - for (boolean haProxyConfigured : tf) { - for (boolean saslOffloadConfigured : tf) { - for (boolean sendApiVersions : tf) { - for (boolean sendSasl : tf) { - result.add(Arguments.of(sslConfigured, haProxyConfigured, saslOffloadConfigured, sendApiVersions, sendSasl)); + for (boolean clientAuthConfigured : tf) { + for (boolean sslConfigured : tf) { + for (boolean haProxyConfigured : tf) { + for (boolean saslOffloadConfigured : tf) { + for (boolean sendApiVersions : tf) { + for (boolean sendSasl : tf) { + result.add(Arguments.of(clientAuthConfigured, sslConfigured, haProxyConfigured, saslOffloadConfigured, sendApiVersions, sendSasl)); + } } } } @@ -289,12 +301,29 @@ ChannelFuture initConnection(String remoteHost, int remotePort, Bootstrap bootst outboundChannel.pipeline().fireChannelRegistered(); return outboundChannel.newPromise(); } + + @Override + SslHandler getSslHandler(ChannelHandlerContext ctx) { + // We don't need to test SSL here, so just return a mock + SSLEngine sslEngine = mock(SSLEngine.class); + when(sslEngine.getSession()).thenReturn(sslSession); + when(mockPrincipal.toString()).thenReturn(CLIENT_PRINCIPAL); + return new SslHandler(sslEngine) { + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) { + if (evt instanceof SniCompletionEvent) { + ctx.fireUserEventTriggered(evt); + } + } + }; + } }; } /** * Test the normal flow, in a number of configurations. * + * @param clientAuthConfigured * @param sslConfigured Whether SSL is configured * @param haProxyConfigured * @param saslOffloadConfigured @@ -303,7 +332,8 @@ ChannelFuture initConnection(String remoteHost, int remotePort, Bootstrap bootst */ @ParameterizedTest @MethodSource("provideArgsForExpectedFlow") - void expectedFlow(boolean sslConfigured, + void expectedFlow(boolean clientAuthConfigured, + boolean sslConfigured, boolean haProxyConfigured, boolean saslOffloadConfigured, boolean sendApiVersions, @@ -355,11 +385,41 @@ void expectedFlow(boolean sslConfigured, var handler = handler(filter, dp, virtualClusterListenerModel); initialiseInboundChannel(handler); + if (clientAuthConfigured){ + try { + when(sslSession.getPeerPrincipal()).thenReturn(mockPrincipal); + } + catch (SSLPeerUnverifiedException e) { + throw new RuntimeException(e); + } + } + else { + try { + when(sslSession.getPeerPrincipal()).thenThrow(new SSLPeerUnverifiedException("No peer certificate")); + } + catch (SSLPeerUnverifiedException e) { + throw new RuntimeException(e); + } + } + + if (sslConfigured) { // Simulate the SSL handler inboundChannel.pipeline().fireUserEventTriggered(new SniCompletionEvent(SNI_HOSTNAME)); + inboundChannel.pipeline().fireUserEventTriggered(SslHandshakeCompletionEvent.SUCCESS); + if (clientAuthConfigured) { + inboundChannel.pipeline().fireUserEventTriggered(SslHandshakeCompletionEvent.SUCCESS); + } } + if (sslConfigured) { + if (clientAuthConfigured) { + assertEquals(CLIENT_PRINCIPAL, handler.getDownStreamCertificatePrincipal()); + } + else { + assertEquals("ANONYMOUS", handler.getDownStreamCertificatePrincipal()); + } + } assertThat(proxyChannelStateMachine.state()).isExactlyInstanceOf(ProxyChannelState.ClientActive.class); if (haProxyConfigured) { From ba4bd9453322e25b85bb91a862fec86005b8fcbe Mon Sep 17 00:00:00 2001 From: Ujjwal Date: Wed, 4 Jun 2025 23:36:02 +0530 Subject: [PATCH 02/83] additional changes Signed-off-by: Ujjwal --- .../proxy/filter/FilterContext.java | 13 ++++++++++++- .../proxy/internal/FilterHandler.java | 5 +++-- .../internal/KafkaProxyFrontendHandler.java | 17 +++++++++-------- .../internal/KafkaProxyFrontendHandlerTest.java | 12 ++++-------- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/kroxylicious-api/src/main/java/io/kroxylicious/proxy/filter/FilterContext.java b/kroxylicious-api/src/main/java/io/kroxylicious/proxy/filter/FilterContext.java index f8bfda2ca3..64864f0b72 100644 --- a/kroxylicious-api/src/main/java/io/kroxylicious/proxy/filter/FilterContext.java +++ b/kroxylicious-api/src/main/java/io/kroxylicious/proxy/filter/FilterContext.java @@ -42,8 +42,19 @@ public interface FilterContext { @Nullable String sniHostname(); + /** + * Returns the principal name from the client's TLS certificate. + *

+ * For X.509 certificates, this is typically the Distinguished Name (DN) in format + * such as "CN=clientName, O=organizationName, C=US". This information can be used + * by filters for client authentication, authorization decisions, or audit logging. + * + * @return the client certificate principal as a string + */ @Nullable - String downStreamCertificatePrincipal(); + default String downstreamCertificatePrincipal() { + return null; + } /** * Creates a builder for a request filter result objects. This object encapsulates diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/FilterHandler.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/FilterHandler.java index fdca2a5bfe..074b43f1fa 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/FilterHandler.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/FilterHandler.java @@ -68,7 +68,8 @@ public class FilterHandler extends ChannelDuplexHandler { private ChannelHandlerContext ctx; private PromiseFactory promiseFactory; - public FilterHandler(FilterAndInvoker filterAndInvoker, long timeoutMs, String sniHostname, String downStreamCertificatePrincipal, VirtualClusterModel virtualClusterModel, Channel inboundChannel) { + public FilterHandler(FilterAndInvoker filterAndInvoker, long timeoutMs, String sniHostname, String downStreamCertificatePrincipal, + VirtualClusterModel virtualClusterModel, Channel inboundChannel) { this.filterAndInvoker = Objects.requireNonNull(filterAndInvoker); this.timeoutMs = Assertions.requireStrictlyPositive(timeoutMs, "timeout"); this.sniHostname = sniHostname; @@ -479,7 +480,7 @@ public String sniHostname() { @Nullable @Override - public String downStreamCertificatePrincipal() { + public String downstreamCertificatePrincipal() { return downStreamCertificatePrincipal; } diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java index bebc763c38..ce2fdbb4b1 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java @@ -68,6 +68,7 @@ public class KafkaProxyFrontendHandler private static final String NET_FILTER_INVOKED_IN_WRONG_STATE = "NetFilterContext invoked in wrong session state"; private static final Logger LOGGER = LoggerFactory.getLogger(KafkaProxyFrontendHandler.class); + private static final String ANONYMOUS = "ANONYMOUS"; /** Cache ApiVersions response which we use when returning ApiVersions ourselves */ private static final ApiVersionsResponseData API_VERSIONS_RESPONSE; @@ -87,14 +88,13 @@ public class KafkaProxyFrontendHandler private boolean pendingClientFlushes; private @Nullable AuthenticationEvent authentication; private @Nullable String sniHostname; + private @Nullable String downstreamCertificatePrincipal; @Nullable - public String getDownStreamCertificatePrincipal() { - return downStreamCertificatePrincipal; + public String getDownstreamCertificatePrincipal() { + return downstreamCertificatePrincipal; } - private @Nullable String downStreamCertificatePrincipal; - @VisibleForTesting SslHandler getSslHandler(ChannelHandlerContext ctx) { if (ctx == null) { @@ -191,10 +191,11 @@ else if (event instanceof SslHandshakeCompletionEvent sslHandshakeCompletionEven if (sslHandshakeCompletionEvent.isSuccess()) { SslHandler sslHandler = getSslHandler(ctx); try { - downStreamCertificatePrincipal = sslHandler.engine().getSession().getPeerPrincipal().toString(); - } catch (SSLPeerUnverifiedException e) { + downstreamCertificatePrincipal = sslHandler.engine().getSession().getPeerPrincipal().toString(); + } + catch (SSLPeerUnverifiedException e) { LOGGER.debug("No client principal received, setting principal as ANONYMOUS"); - downStreamCertificatePrincipal = "ANONYMOUS"; + downstreamCertificatePrincipal = ANONYMOUS; } } } @@ -655,7 +656,7 @@ private void addFiltersToPipeline( protocolFilter, 20000, sniHostname, - downStreamCertificatePrincipal, + downstreamCertificatePrincipal, virtualClusterModel, inboundChannel)); } diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandlerTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandlerTest.java index e238c450b5..4a453db85b 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandlerTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandlerTest.java @@ -323,7 +323,7 @@ public void userEventTriggered(ChannelHandlerContext ctx, Object evt) { /** * Test the normal flow, in a number of configurations. * - * @param clientAuthConfigured + * @param clientAuthConfigured whether mTLS is configured * @param sslConfigured Whether SSL is configured * @param haProxyConfigured * @param saslOffloadConfigured @@ -385,7 +385,7 @@ void expectedFlow(boolean clientAuthConfigured, var handler = handler(filter, dp, virtualClusterListenerModel); initialiseInboundChannel(handler); - if (clientAuthConfigured){ + if (clientAuthConfigured) { try { when(sslSession.getPeerPrincipal()).thenReturn(mockPrincipal); } @@ -402,22 +402,18 @@ void expectedFlow(boolean clientAuthConfigured, } } - if (sslConfigured) { // Simulate the SSL handler inboundChannel.pipeline().fireUserEventTriggered(new SniCompletionEvent(SNI_HOSTNAME)); inboundChannel.pipeline().fireUserEventTriggered(SslHandshakeCompletionEvent.SUCCESS); - if (clientAuthConfigured) { - inboundChannel.pipeline().fireUserEventTriggered(SslHandshakeCompletionEvent.SUCCESS); - } } if (sslConfigured) { if (clientAuthConfigured) { - assertEquals(CLIENT_PRINCIPAL, handler.getDownStreamCertificatePrincipal()); + assertEquals(CLIENT_PRINCIPAL, handler.getDownstreamCertificatePrincipal()); } else { - assertEquals("ANONYMOUS", handler.getDownStreamCertificatePrincipal()); + assertEquals("ANONYMOUS", handler.getDownstreamCertificatePrincipal()); } } assertThat(proxyChannelStateMachine.state()).isExactlyInstanceOf(ProxyChannelState.ClientActive.class); From 18100788ed57ee40e4a464a5584739e0f7afbbd7 Mon Sep 17 00:00:00 2001 From: Ujjwal Date: Wed, 4 Jun 2025 23:40:22 +0530 Subject: [PATCH 03/83] additional changes Signed-off-by: Ujjwal --- .../io/kroxylicious/proxy/internal/FilterHandler.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/FilterHandler.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/FilterHandler.java index 074b43f1fa..a44a0823c1 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/FilterHandler.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/FilterHandler.java @@ -59,7 +59,7 @@ public class FilterHandler extends ChannelDuplexHandler { private static final Logger LOGGER = LoggerFactory.getLogger(FilterHandler.class); private final long timeoutMs; private final String sniHostname; - private final String downStreamCertificatePrincipal; + private final String downstreamCertificatePrincipal; private final VirtualClusterModel virtualClusterModel; private final Channel inboundChannel; private final FilterAndInvoker filterAndInvoker; @@ -68,12 +68,12 @@ public class FilterHandler extends ChannelDuplexHandler { private ChannelHandlerContext ctx; private PromiseFactory promiseFactory; - public FilterHandler(FilterAndInvoker filterAndInvoker, long timeoutMs, String sniHostname, String downStreamCertificatePrincipal, + public FilterHandler(FilterAndInvoker filterAndInvoker, long timeoutMs, String sniHostname, String downstreamCertificatePrincipal, VirtualClusterModel virtualClusterModel, Channel inboundChannel) { this.filterAndInvoker = Objects.requireNonNull(filterAndInvoker); this.timeoutMs = Assertions.requireStrictlyPositive(timeoutMs, "timeout"); this.sniHostname = sniHostname; - this.downStreamCertificatePrincipal = downStreamCertificatePrincipal; + this.downstreamCertificatePrincipal = downstreamCertificatePrincipal; this.virtualClusterModel = virtualClusterModel; this.inboundChannel = inboundChannel; } @@ -481,7 +481,7 @@ public String sniHostname() { @Nullable @Override public String downstreamCertificatePrincipal() { - return downStreamCertificatePrincipal; + return downstreamCertificatePrincipal; } public String getVirtualClusterName() { From d2f3c92d7b3eb455cebde5eb4713d0d59262013c Mon Sep 17 00:00:00 2001 From: Kroxylicious Robot <153509303+kroxylicious-robot@users.noreply.github.com> Date: Mon, 2 Jun 2025 08:43:42 +0100 Subject: [PATCH 04/83] Update quay.io/apicurio/apicurio-registry-mem Docker tag to v2.6.9.Final (#2248) Signed-off-by: k-wall <18440250+k-wall@users.noreply.github.com> Co-authored-by: k-wall <18440250+k-wall@users.noreply.github.com> Signed-off-by: Ujjwal --- .../proxy/filter/validation/JsonSchemaRecordValidationIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/filter/validation/JsonSchemaRecordValidationIT.java b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/filter/validation/JsonSchemaRecordValidationIT.java index 55bab6de98..2b0e2ad104 100644 --- a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/filter/validation/JsonSchemaRecordValidationIT.java +++ b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/filter/validation/JsonSchemaRecordValidationIT.java @@ -108,7 +108,7 @@ class JsonSchemaRecordValidationIT extends RecordValidationBaseIT { @BeforeAll public static void init() throws IOException { // An Apicurio Registry instance is required for this test to work, so we start one using a Generic Container - DockerImageName dockerImageName = DockerImageName.parse("quay.io/apicurio/apicurio-registry-mem:2.6.8.Final"); + DockerImageName dockerImageName = DockerImageName.parse("quay.io/apicurio/apicurio-registry-mem:2.6.9.Final"); Consumer cmd = e -> e.withPortBindings( new PortBinding(Ports.Binding.bindPort(APICURIO_REGISTRY_PORT), new ExposedPort(APICURIO_REGISTRY_PORT))); From 7f0694b73637f5991ba9579714c1449f53d8f3c6 Mon Sep 17 00:00:00 2001 From: Keith Wall Date: Mon, 2 Jun 2025 08:48:54 +0100 Subject: [PATCH 05/83] =?UTF-8?q?Revert=20"Update=20quay.io/apicurio/apicu?= =?UTF-8?q?rio-registry-mem=20Docker=20tag=20to=20v2.6.9.Fi=E2=80=A6"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 602113274bea605a130cce21f26b7e8fd4ce4481. Signed-off-by: Keith Wall Signed-off-by: Ujjwal --- .../proxy/filter/validation/JsonSchemaRecordValidationIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/filter/validation/JsonSchemaRecordValidationIT.java b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/filter/validation/JsonSchemaRecordValidationIT.java index 2b0e2ad104..55bab6de98 100644 --- a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/filter/validation/JsonSchemaRecordValidationIT.java +++ b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/filter/validation/JsonSchemaRecordValidationIT.java @@ -108,7 +108,7 @@ class JsonSchemaRecordValidationIT extends RecordValidationBaseIT { @BeforeAll public static void init() throws IOException { // An Apicurio Registry instance is required for this test to work, so we start one using a Generic Container - DockerImageName dockerImageName = DockerImageName.parse("quay.io/apicurio/apicurio-registry-mem:2.6.9.Final"); + DockerImageName dockerImageName = DockerImageName.parse("quay.io/apicurio/apicurio-registry-mem:2.6.8.Final"); Consumer cmd = e -> e.withPortBindings( new PortBinding(Ports.Binding.bindPort(APICURIO_REGISTRY_PORT), new ExposedPort(APICURIO_REGISTRY_PORT))); From 17746fba4c9b708f20a3e6720014432a83cc1abc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jun 2025 04:10:33 +0000 Subject: [PATCH 06/83] Bump org.junit:junit-bom from 5.12.2 to 5.13.0 Bumps [org.junit:junit-bom](https://github.com/junit-team/junit5) from 5.12.2 to 5.13.0. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.12.2...r5.13.0) --- updated-dependencies: - dependency-name: org.junit:junit-bom dependency-version: 5.13.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: Ujjwal --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 02b606138f..035fb3b2cf 100644 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,7 @@ 33.4.8-jre 3.0 2.18.3 - 5.12.2 + 5.13.0 4.0.0 0.11.0 2.24.3 From 952c4be83a0e766e152e5345bca5167255c640a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jun 2025 15:48:35 +0000 Subject: [PATCH 07/83] Bump org.apache.maven.plugins:maven-clean-plugin from 3.4.1 to 3.5.0 Bumps [org.apache.maven.plugins:maven-clean-plugin](https://github.com/apache/maven-clean-plugin) from 3.4.1 to 3.5.0. - [Release notes](https://github.com/apache/maven-clean-plugin/releases) - [Commits](https://github.com/apache/maven-clean-plugin/compare/maven-clean-plugin-3.4.1...maven-clean-plugin-3.5.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-clean-plugin dependency-version: 3.5.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: Ujjwal --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 035fb3b2cf..910c8bfd5f 100644 --- a/pom.xml +++ b/pom.xml @@ -609,7 +609,7 @@ org.apache.maven.plugins maven-clean-plugin - 3.4.1 + 3.5.0 org.apache.maven.plugins From 285e181568979c0c83b6df56d07b652f3609aed1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jun 2025 03:14:04 +0000 Subject: [PATCH 08/83] Bump dawidd6/action-download-artifact from 8 to 10 Bumps [dawidd6/action-download-artifact](https://github.com/dawidd6/action-download-artifact) from 8 to 10. - [Release notes](https://github.com/dawidd6/action-download-artifact/releases) - [Commits](https://github.com/dawidd6/action-download-artifact/compare/v8...v10) --- updated-dependencies: - dependency-name: dawidd6/action-download-artifact dependency-version: '10' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Signed-off-by: Ujjwal --- .github/workflows/sonar.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sonar.yaml b/.github/workflows/sonar.yaml index 34ba8b2f90..742b6b1359 100644 --- a/.github/workflows/sonar.yaml +++ b/.github/workflows/sonar.yaml @@ -17,7 +17,7 @@ jobs: - name: echo event run: cat $GITHUB_EVENT_PATH - name: Download PR number artifact - uses: dawidd6/action-download-artifact@v8 + uses: dawidd6/action-download-artifact@v10 with: workflow: Build run_id: ${{ github.event.workflow_run.id }} From f129802028ce1d8ad859ca8753de05599c7c755a Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Fri, 30 May 2025 20:23:54 +1200 Subject: [PATCH 09/83] Enable remote debugging of the Operator Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../KroxyliciousOperatorBundleInstaller.java | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/operator/KroxyliciousOperatorBundleInstaller.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/operator/KroxyliciousOperatorBundleInstaller.java index a6b963f723..702d7b3983 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/operator/KroxyliciousOperatorBundleInstaller.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/operator/KroxyliciousOperatorBundleInstaller.java @@ -23,7 +23,13 @@ import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.platform.commons.PreconditionViolationException; +import io.fabric8.kubernetes.api.model.ContainerPortBuilder; +import io.fabric8.kubernetes.api.model.EnvVarBuilder; +import io.fabric8.kubernetes.api.model.IntOrString; import io.fabric8.kubernetes.api.model.LocalObjectReferenceBuilder; +import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.ServiceBuilder; +import io.fabric8.kubernetes.api.model.ServicePortBuilder; import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinition; import io.fabric8.kubernetes.api.model.apps.Deployment; import io.fabric8.kubernetes.api.model.apps.DeploymentBuilder; @@ -52,6 +58,7 @@ public class KroxyliciousOperatorBundleInstaller implements InstallationMethod { private static final String SEPARATOR = String.join("", Collections.nCopies(76, "=")); private static final KubeClusterResource cluster = KubeClusterResource.getInstance(); + protected static final int DEBUG_PORT_NUMBER = 5005; private static List operatorFiles; @@ -122,6 +129,8 @@ private void applyDeploymentFile() { .get(0) .getImage(); + String debugPortName = "remote-debug"; + //@formatter:off operatorDeployment = new DeploymentBuilder(operatorDeployment) .editOrNewMetadata() .withName(kroxyliciousOperatorName) @@ -142,6 +151,9 @@ private void applyDeploymentFile() { .withImage(ImageUtils.changeRegistryOrgImageAndTag(deploymentImage, Environment.KROXYLICIOUS_OPERATOR_REGISTRY, Environment.KROXYLICIOUS_OPERATOR_ORG, Environment.KROXYLICIOUS_OPERATOR_IMAGE, Environment.KROXYLICIOUS_OPERATOR_VERSION)) .withImagePullPolicy(Constants.PULL_IMAGE_IF_NOT_PRESENT) + .addToEnv(new EnvVarBuilder().withName("JAVA_OPTIONS").withValue("-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:" + DEBUG_PORT_NUMBER) + .build()) + .addToPorts(new ContainerPortBuilder().withName(debugPortName).withContainerPort(DEBUG_PORT_NUMBER).build()) .endContainer() .withImagePullSecrets(new LocalObjectReferenceBuilder() .withName("regcred") @@ -150,8 +162,25 @@ private void applyDeploymentFile() { .endTemplate() .endSpec() .build(); - - KubeResourceManager.get().createOrUpdateResourceWithWait(operatorDeployment); + Service debugService = new ServiceBuilder() + .editOrNewMetadata() + .withName("debug-" + kroxyliciousOperatorName) + .withNamespace(namespaceInstallTo) + .endMetadata() + .withNewSpec() + .withSelector(Map.of("app", "kroxylicious")) + .withType("LoadBalancer") + .withAllocateLoadBalancerNodePorts() + .addToPorts(new ServicePortBuilder() + .withName(debugPortName) + .withPort(DEBUG_PORT_NUMBER) + .withTargetPort(new IntOrString(debugPortName)) + .build()) + .withExternalIPs() + .endSpec() + .build(); + //@formatter:on + KubeResourceManager.get().createOrUpdateResourceWithWait(operatorDeployment, debugService); } /** From deec229ef502701ac72be2e9480f4fdc32b29859 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Tue, 3 Jun 2025 10:39:15 +1200 Subject: [PATCH 10/83] Mention debug access in operator DEV_GUIDE.md Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- kroxylicious-operator/DEV_GUIDE.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/kroxylicious-operator/DEV_GUIDE.md b/kroxylicious-operator/DEV_GUIDE.md index da8bd7776b..d3f4e5b669 100644 --- a/kroxylicious-operator/DEV_GUIDE.md +++ b/kroxylicious-operator/DEV_GUIDE.md @@ -90,4 +90,19 @@ Note that on its own the proxy won't try to connect to the Kafka cluster. # Testing -To test things properly you'll need to point your virtual clusters at a running Kafka and also run a Kafka client so the proxy is handling some load. \ No newline at end of file +To test things properly you'll need to point your virtual clusters at a running Kafka and also run a Kafka client so the proxy is handling some load. + +## System Tests + +THe Kroxylicious system test suite has been updated to use the operator to deploy resources, so tests can be written in +java and executed locally in one's IDE. They do however require access to a Kubernetes clusters (usually minkkube) and +helm, with the appropriate RBAC permissions to install operators and provision resources. + +System tests are slow running things and are often difficult to diagnose issues just from external observation. To +support developers working with the operator while the tests execute the system test framework will enable remote debug +connections to the Kroxylicious Operator and create a LoadBalancer service (`debug-kroxylicious-operator`) to expose it. +Running `minikube tunnel` will make that available to the IDE, thus allowing developers to connect to the operator and +add breakpoints and step through execution. Note if we find ourselves doing this regularly we should look at improving +our unit test coverage and logging to make the diagnosis and avoidance of such issues much easier in less accessible +environments. + From 18cb11cd6efb5cf0f5567e707955316814f6a406 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Tue, 3 Jun 2025 20:23:23 +1200 Subject: [PATCH 11/83] PR Review: typo fix Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- kroxylicious-operator/DEV_GUIDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kroxylicious-operator/DEV_GUIDE.md b/kroxylicious-operator/DEV_GUIDE.md index d3f4e5b669..3b497c8def 100644 --- a/kroxylicious-operator/DEV_GUIDE.md +++ b/kroxylicious-operator/DEV_GUIDE.md @@ -95,7 +95,7 @@ To test things properly you'll need to point your virtual clusters at a running ## System Tests THe Kroxylicious system test suite has been updated to use the operator to deploy resources, so tests can be written in -java and executed locally in one's IDE. They do however require access to a Kubernetes clusters (usually minkkube) and +java and executed locally in one's IDE. They do however require access to a Kubernetes clusters (usually minikube) and helm, with the appropriate RBAC permissions to install operators and provision resources. System tests are slow running things and are often difficult to diagnose issues just from external observation. To From 67c59f1e0ed31d731386565ded63b16a144f3a43 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Tue, 3 Jun 2025 21:06:47 +1200 Subject: [PATCH 12/83] Typo and grammer fixes Co-authored-by: Keith Wall Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- kroxylicious-operator/DEV_GUIDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kroxylicious-operator/DEV_GUIDE.md b/kroxylicious-operator/DEV_GUIDE.md index 3b497c8def..aba6292f77 100644 --- a/kroxylicious-operator/DEV_GUIDE.md +++ b/kroxylicious-operator/DEV_GUIDE.md @@ -94,7 +94,7 @@ To test things properly you'll need to point your virtual clusters at a running ## System Tests -THe Kroxylicious system test suite has been updated to use the operator to deploy resources, so tests can be written in +The Kroxylicious system test suite uses the operator to deploy resources, so tests can be written in java and executed locally in one's IDE. They do however require access to a Kubernetes clusters (usually minikube) and helm, with the appropriate RBAC permissions to install operators and provision resources. From 85bbde8eef911329c8ef448f4ced3bae7cc5f5f5 Mon Sep 17 00:00:00 2001 From: Keith Wall Date: Thu, 29 May 2025 17:13:53 +0100 Subject: [PATCH 13/83] Fix #2239: Downstream / upstream kafka message counting metrics * Frame now carries api key id/version * pushed responsibility for emitting metrics to dedicated handlers. Signed-off-by: Keith Wall Signed-off-by: Ujjwal --- kroxylicious-integration-test-support/pom.xml | 5 + .../test/tester/SimpleMetricAssert.java | 89 +++++++ .../test/tester/SimpleMetricAssertTest.java | 80 ++++++ .../java/io/kroxylicious/proxy/MetricsIT.java | 240 +++++++++++++++--- .../kroxylicious/proxy/filter/DecodeAll.java | 46 ++++ ...io.kroxylicious.proxy.filter.FilterFactory | 1 + .../io/kroxylicious/proxy/KafkaProxy.java | 17 +- .../proxy/frame/BareSaslRequest.java | 15 ++ .../proxy/frame/BareSaslResponse.java | 15 ++ .../proxy/frame/DecodedFrame.java | 25 +- .../proxy/frame/DecodedRequestFrame.java | 5 +- .../proxy/frame/DecodedResponseFrame.java | 3 +- .../io/kroxylicious/proxy/frame/Frame.java | 8 + .../kroxylicious/proxy/frame/OpaqueFrame.java | 29 ++- .../proxy/frame/OpaqueRequestFrame.java | 28 +- .../proxy/frame/OpaqueResponseFrame.java | 6 +- .../proxy/internal/FilterHandler.java | 2 +- .../internal/KafkaProxyFrontendHandler.java | 35 ++- .../proxy/internal/KafkaProxyInitializer.java | 24 +- .../internal/codec/KafkaRequestDecoder.java | 31 +-- .../internal/codec/KafkaResponseDecoder.java | 8 +- .../DeprecatedDownstreamMessageMetrics.java | 51 ++++ .../DeprecatedUpstreamMessageMetrics.java | 41 +++ .../internal/metrics/MessageMetrics.java | 69 +++++ .../proxy/internal/util/Metrics.java | 101 +++++++- .../proxy/internal/FilterHarness.java | 4 +- .../proxy/internal/KafkaAuthnHandlerTest.java | 15 ++ ...yFrontendHandlerMockCollaboratorsTest.java | 7 +- .../KafkaProxyFrontendHandlerTest.java | 70 +++-- .../internal/KafkaProxyInitializerTest.java | 3 + .../ProxyChannelStateMachineEndToEndTest.java | 26 +- .../ProxyChannelStateMachineTest.java | 2 +- .../proxy/internal/ResponseOrdererTest.java | 30 +++ .../codec/KafkaRequestDecoderTest.java | 6 +- .../internal/codec/RequestDecoderTest.java | 11 +- .../internal/codec/RequestEncoderTest.java | 22 +- .../internal/codec/ResponseEncoderTest.java | 3 +- 37 files changed, 1002 insertions(+), 171 deletions(-) create mode 100644 kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/tester/SimpleMetricAssert.java create mode 100644 kroxylicious-integration-test-support/src/test/java/io/kroxylicious/test/tester/SimpleMetricAssertTest.java create mode 100644 kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/filter/DecodeAll.java create mode 100644 kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedDownstreamMessageMetrics.java create mode 100644 kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedUpstreamMessageMetrics.java create mode 100644 kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/MessageMetrics.java diff --git a/kroxylicious-integration-test-support/pom.xml b/kroxylicious-integration-test-support/pom.xml index 72d8a8a025..adaaa60588 100644 --- a/kroxylicious-integration-test-support/pom.xml +++ b/kroxylicious-integration-test-support/pom.xml @@ -185,6 +185,11 @@ mockito-junit-jupiter test + + org.assertj + assertj-core + compile + diff --git a/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/tester/SimpleMetricAssert.java b/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/tester/SimpleMetricAssert.java new file mode 100644 index 0000000000..b000dcebe5 --- /dev/null +++ b/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/tester/SimpleMetricAssert.java @@ -0,0 +1,89 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.test.tester; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import org.assertj.core.api.AbstractAssert; +import org.assertj.core.api.AbstractDoubleAssert; +import org.assertj.core.api.AbstractStringAssert; +import org.assertj.core.api.Assertions; +import org.assertj.core.api.FactoryBasedNavigableListAssert; +import org.assertj.core.api.MapAssert; + +import static org.assertj.core.util.Lists.newArrayList; + +public class SimpleMetricAssert extends AbstractAssert { + + public static SimpleMetricAssert assertThat(SimpleMetric actual) { + return new SimpleMetricAssert(actual); + } + + public static SimpleMetricListAssert assertThat(List actual) { + return new SimpleMetricListAssert(actual); + } + + public static SimpleMetricAssert assertThat(Optional actual) { + return new SimpleMetricAssert(actual.orElse(null)); + } + + private SimpleMetricAssert(SimpleMetric simpleMetric) { + super(simpleMetric, SimpleMetricAssert.class); + describedAs(simpleMetric == null ? "null metric" : "metric"); + } + + public AbstractDoubleAssert value() { + isNotNull(); + return Assertions.assertThat(actual.value()) + .describedAs("metric value"); + } + + public AbstractStringAssert name() { + isNotNull(); + return Assertions.assertThat(actual.name()) + .describedAs("metric name"); + } + + public MapAssert labels() { + isNotNull(); + return Assertions.assertThat(actual.labels()) + .describedAs("metric labels"); + } + + public static class SimpleMetricListAssert extends FactoryBasedNavigableListAssert, SimpleMetric, SimpleMetricAssert> { + SimpleMetricListAssert(List simpleMetrics) { + super(simpleMetrics, SimpleMetricListAssert.class, SimpleMetricAssert::new); + describedAs(simpleMetrics == null ? "empty metric list" : "metrics"); + + } + + public SimpleMetricListAssert filterByName(String name) { + isNotNull(); + return filteredOn(metric -> metric.name().equals(name)); + } + + public SimpleMetricListAssert filterByTag(String key) { + isNotNull(); + return filteredOn(metric -> metric.labels() != null + && metric.labels().containsKey(key)); + } + + public SimpleMetricListAssert filterByTag(String key, String value) { + isNotNull(); + return filterByTag(key) + .filteredOn(metric -> Objects.equals(metric.labels().get(key), value)); + } + + @Override + protected SimpleMetricListAssert newAbstractIterableAssert(Iterable iterable) { + return new SimpleMetricListAssert(newArrayList(iterable)); + } + + } +} diff --git a/kroxylicious-integration-test-support/src/test/java/io/kroxylicious/test/tester/SimpleMetricAssertTest.java b/kroxylicious-integration-test-support/src/test/java/io/kroxylicious/test/tester/SimpleMetricAssertTest.java new file mode 100644 index 0000000000..f4bd500785 --- /dev/null +++ b/kroxylicious-integration-test-support/src/test/java/io/kroxylicious/test/tester/SimpleMetricAssertTest.java @@ -0,0 +1,80 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.test.tester; + +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import static io.kroxylicious.test.tester.SimpleMetricAssert.assertThat; + +class SimpleMetricAssertTest { + + @Test + void canAssertMetricName() { + var metric = new SimpleMetric("foo", Map.of(), 5); + var simpleMetricAssert = assertThat(metric); + simpleMetricAssert.name().isEqualTo("foo"); + } + + @Test + void canAssertMetricValue() { + var metric = new SimpleMetric("foo", Map.of(), 5); + var simpleMetricAssert = assertThat(metric); + simpleMetricAssert.value().isEqualTo(5.0); + } + + @Test + void canAssertMetricLabel() { + var metric = new SimpleMetric("foo", Map.of("fruit", "banana", "animal", "capybara"), 5); + var simpleMetricAssert = assertThat(metric); + simpleMetricAssert.labels() + .containsKey("animal") + .containsEntry("animal", "capybara"); + } + + @Test + void canFilterByMetricName() { + var metric1 = new SimpleMetric("foo", Map.of(), 1); + var metric2 = new SimpleMetric("bar", Map.of(), 1); + assertThat(List.of(metric1, metric2)) + .hasSize(2) + .filterByName("foo") + .hasSize(1) + .singleElement() + .name() + .isEqualTo("foo"); + } + + @Test + void canFilterByTag() { + var metric1 = new SimpleMetric("foo", Map.of("animal", "capybara"), 1); + var metric2 = new SimpleMetric("bar", Map.of("animal", "squirrel"), 1); + var metric3 = new SimpleMetric("baz", Map.of("planet", "venus"), 1); + assertThat(List.of(metric1, metric2, metric3)) + .hasSize(3) + .filterByTag("animal") + .hasSize(2) + .satisfiesExactly(e1 -> assertThat(e1).name().isEqualTo("foo"), + e2 -> assertThat(e2).name().isEqualTo("bar")); + } + + @Test + void canFilterByTagValue() { + var metric1 = new SimpleMetric("foo", Map.of("animal", "capybara"), 1); + var metric2 = new SimpleMetric("bar", Map.of("animal", "squirrel"), 1); + assertThat(List.of(metric1, metric2)) + .hasSize(2) + .filterByTag("animal", "squirrel") + .hasSize(1) + .singleElement() + .name() + .isEqualTo("bar"); + } + +} \ No newline at end of file diff --git a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java index 24f17e6885..103470ce4e 100644 --- a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java +++ b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java @@ -5,23 +5,35 @@ */ package io.kroxylicious.proxy; +import java.time.Duration; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Random; import java.util.UUID; import java.util.concurrent.ExecutionException; +import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.function.UnaryOperator; +import java.util.stream.Stream; +import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.protocol.ApiKeys; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import io.github.nettyplus.leakdetector.junit.NettyLeakDetectorExtension; import io.micrometer.core.instrument.Metrics; import io.netty.handler.codec.http.HttpResponseStatus; +import io.kroxylicious.proxy.config.ConfigurationBuilder; import io.kroxylicious.proxy.config.MicrometerDefinitionBuilder; import io.kroxylicious.proxy.config.NamedFilterDefinition; import io.kroxylicious.proxy.config.NamedFilterDefinitionBuilder; @@ -34,22 +46,26 @@ import edu.umd.cs.findbugs.annotations.NonNull; +import static io.kroxylicious.proxy.internal.util.Metrics.API_KEY_LABEL; +import static io.kroxylicious.proxy.internal.util.Metrics.DECODED_LABEL; +import static io.kroxylicious.proxy.internal.util.Metrics.NODE_ID_LABEL; import static io.kroxylicious.test.tester.KroxyliciousConfigUtils.proxy; import static io.kroxylicious.test.tester.KroxyliciousTesters.kroxyliciousTester; +import static io.kroxylicious.test.tester.SimpleMetricAssert.assertThat; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.tuple; +import static org.junit.jupiter.params.provider.Arguments.argumentSet; @ExtendWith(KafkaClusterExtension.class) @ExtendWith(NettyLeakDetectorExtension.class) class MetricsIT { @BeforeEach - public void beforeEach() { + void beforeEach() { assertThat(Metrics.globalRegistry.getMeters()).isEmpty(); } @AfterEach - public void afterEach() { + void afterEach() { assertThat(Metrics.globalRegistry.getMeters()).isEmpty(); } @@ -107,9 +123,10 @@ void knownPrometheusMetricPresent(KafkaCluster cluster) { Metrics.counter(counterName).increment(); var metrics = ahc.scrapeMetrics(); assertThat(metrics) - .hasSizeGreaterThan(0) - .extracting(SimpleMetric::name, SimpleMetric::value) - .contains(tuple(counterName, 1.0)); + .filterByName(counterName) + .singleElement() + .value() + .isEqualTo(1.0); } } @@ -128,9 +145,9 @@ void prometheusMetricFromNamedBinder(KafkaCluster cluster) { try (var tester = kroxyliciousTester(config); var ahc = tester.getManagementClient()) { assertThat(ahc.scrapeMetrics()) - .hasSizeGreaterThan(0) - .extracting(SimpleMetric::name) - .contains("jvm_gc_memory_allocated_bytes_total"); + .filterByName("jvm_gc_memory_allocated_bytes_total") + .singleElement() + .isNotNull(); } } @@ -152,14 +169,174 @@ void prometheusMetricsWithCommonTags(KafkaCluster cluster) { var metrics = ahc.scrapeMetrics(); assertThat(metrics) - .filteredOn("name", counterName) + .filterByName(counterName) .singleElement() - .extracting(SimpleMetric::labels) - .isEqualTo(Map.of("a", "b")); + .labels() + .containsEntry("a", "b"); + } + } + + static Stream transitScenarios() { + var decodeAll = new NamedFilterDefinitionBuilder("decodeAll", "DecodeAll").build(); + return Stream.of( + argumentSet("counts opaque requests from client", + (UnaryOperator) builder -> builder, + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_client_to_proxy_request_total") + .filterByTag(API_KEY_LABEL, ApiKeys.CREATE_TOPICS.name()) + .filterByTag(NODE_ID_LABEL, "0") + .filterByTag(DECODED_LABEL, "false") + .isEmpty(), + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_client_to_proxy_request_total") + .filterByTag(API_KEY_LABEL, ApiKeys.CREATE_TOPICS.name()) + .filterByTag(NODE_ID_LABEL, "0") + .filterByTag(DECODED_LABEL, "false") + .singleElement() + .value() + .isEqualTo(1.0)), + argumentSet("counts decoded requests from client", + (UnaryOperator) builder -> builder + .addToFilterDefinitions(decodeAll) + .addToDefaultFilters(decodeAll.name()), + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_client_to_proxy_request_total") + .filterByTag(API_KEY_LABEL, ApiKeys.CREATE_TOPICS.name()) + .filterByTag(NODE_ID_LABEL, "0") + .isEmpty(), + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_client_to_proxy_request_total") + .filterByTag(API_KEY_LABEL, ApiKeys.CREATE_TOPICS.name()) + .filterByTag(NODE_ID_LABEL, "0") + .filterByTag(DECODED_LABEL, "true") + .singleElement() + .value() + .isEqualTo(1.0)), + argumentSet("counts opaque requests to server", + (UnaryOperator) builder -> builder, + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_proxy_to_server_request_total") + .filterByTag(API_KEY_LABEL, ApiKeys.CREATE_TOPICS.name()) + .filterByTag(NODE_ID_LABEL, "0") + .filterByTag(DECODED_LABEL, "false") + .isEmpty(), + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_proxy_to_server_request_total") + .filterByTag(API_KEY_LABEL, ApiKeys.CREATE_TOPICS.name()) + .filterByTag(NODE_ID_LABEL, "0") + .filterByTag(DECODED_LABEL, "false") + .singleElement() + .value() + .isEqualTo(1.0)), + argumentSet("counts decoded requests to server", + (UnaryOperator) builder -> builder + .addToFilterDefinitions(decodeAll) + .addToDefaultFilters(decodeAll.name()), + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_proxy_to_server_request_total") + .filterByTag(API_KEY_LABEL, ApiKeys.CREATE_TOPICS.name()) + .filterByTag(NODE_ID_LABEL, "0") + .filterByTag(DECODED_LABEL, "true") + .isEmpty(), + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_proxy_to_server_request_total") + .filterByTag(API_KEY_LABEL, ApiKeys.CREATE_TOPICS.name()) + .filterByTag(NODE_ID_LABEL, "0") + .filterByTag(DECODED_LABEL, "true") + .singleElement() + .value() + .isEqualTo(1.0)), + argumentSet("counts bootstrap requests", + (UnaryOperator) builder -> builder, + (Consumer>) metricList -> { + }, + (Consumer>) metricList -> { + assertThat(metricList) + .filterByName("kroxylicious_client_to_proxy_request_total") + .filterByTag(API_KEY_LABEL, ApiKeys.METADATA.name()) + .filterByTag(NODE_ID_LABEL, "bootstrap") + .singleElement() + .value() + .isGreaterThanOrEqualTo(1.0); + assertThat(metricList) + .filterByName("kroxylicious_proxy_to_server_request_total") + .filterByTag(API_KEY_LABEL, ApiKeys.METADATA.name()) + .filterByTag(NODE_ID_LABEL, "bootstrap") + .singleElement() + .value() + .isGreaterThanOrEqualTo(1.0); + }), + argumentSet("counts opaque responses from server", + (UnaryOperator) builder -> builder, + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_server_to_proxy_response_total") + .filterByTag(API_KEY_LABEL, ApiKeys.CREATE_TOPICS.name()) + .filterByTag(NODE_ID_LABEL, "0") + .filterByTag(DECODED_LABEL, "false") + .isEmpty(), + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_server_to_proxy_response_total") + .filterByTag(API_KEY_LABEL, ApiKeys.CREATE_TOPICS.name()) + .filterByTag(NODE_ID_LABEL, "0") + .filterByTag(DECODED_LABEL, "false") + .singleElement() + .value() + .isEqualTo(1.0)), + argumentSet("counts decoded responses to client", + (UnaryOperator) builder -> builder + .addToFilterDefinitions(decodeAll) + .addToDefaultFilters(decodeAll.name()), + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_proxy_to_client_response_total") + .filterByTag(API_KEY_LABEL, ApiKeys.CREATE_TOPICS.name()) + .filterByTag(NODE_ID_LABEL, "0") + .filterByTag(DECODED_LABEL, "true") + .isEmpty(), + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_proxy_to_client_response_total") + .filterByTag(API_KEY_LABEL, ApiKeys.CREATE_TOPICS.name()) + .filterByTag(NODE_ID_LABEL, "0") + .filterByTag(DECODED_LABEL, "true") + .singleElement() + .value() + .isEqualTo(1.0))); + } + + @ParameterizedTest + @MethodSource("transitScenarios") + void shouldIncrementCountOnMessageTransit(UnaryOperator builder, + Consumer> beforeAssertion, + Consumer> afterAssertion, + KafkaCluster cluster) { + var config = proxy(cluster) + .withNewManagement() + .withNewEndpoints() + .withNewPrometheus() + .endPrometheus() + .endEndpoints() + .endManagement(); + + // Given + try (var tester = kroxyliciousTester(builder.apply(config)); + var managementClient = tester.getManagementClient(); + var admin = tester.admin()) { + assertThat(admin.describeCluster().clusterId()).succeedsWithin(Duration.ofSeconds(5)); + + var metricList = managementClient.scrapeMetrics(); + beforeAssertion.accept(metricList); + + // When + var future = admin.createTopics(List.of(new NewTopic(UUID.randomUUID().toString(), Optional.empty(), Optional.empty()))).all(); + + // Then + assertThat(future).succeedsWithin(Duration.ofSeconds(5)); + metricList = managementClient.scrapeMetrics(); + afterAssertion.accept(metricList); } } @Test + @Deprecated(since = "0.13.0", forRemoval = true) void shouldIncrementDownstreamMessagesOnProduceRequestWithoutFilter(KafkaCluster cluster, Topic topic) throws ExecutionException, InterruptedException { var config = proxy(cluster) .withNewManagement() @@ -174,8 +351,8 @@ void shouldIncrementDownstreamMessagesOnProduceRequestWithoutFilter(KafkaCluster var managementClient = tester.getManagementClient(); var producer = tester.producer()) { var metricList = managementClient.scrapeMetrics(); - var inboundDownstreamMessagesMetricsValue = getMetricsValue(metricList, "kroxylicious_inbound_downstream_messages_total", null); - var inboundDownstreamDecodedMessagesMetricsValue = getMetricsValue(metricList, "kroxylicious_inbound_downstream_decoded_messages_total", null); + var inboundDownstreamMessagesMetricsValue = getMetricsValue(metricList, "kroxylicious_inbound_downstream_messages_total"); + var inboundDownstreamDecodedMessagesMetricsValue = getMetricsValue(metricList, "kroxylicious_inbound_downstream_decoded_messages_total"); // When producer.send(new ProducerRecord<>(topic.name(), "my-key", "hello-world")).get(); @@ -183,14 +360,15 @@ void shouldIncrementDownstreamMessagesOnProduceRequestWithoutFilter(KafkaCluster // Then // updated metrics after some message were produced var updatedMetricsList = managementClient.scrapeMetrics(); - var updatedInboundDownstreamMessagesMetricsValue = getMetricsValue(updatedMetricsList, "kroxylicious_inbound_downstream_messages_total", null); - var updatedInboundDownstreamDecodedMessagesMetricsValue = getMetricsValue(updatedMetricsList, "kroxylicious_inbound_downstream_decoded_messages_total", null); + var updatedInboundDownstreamMessagesMetricsValue = getMetricsValue(updatedMetricsList, "kroxylicious_inbound_downstream_messages_total"); + var updatedInboundDownstreamDecodedMessagesMetricsValue = getMetricsValue(updatedMetricsList, "kroxylicious_inbound_downstream_decoded_messages_total"); assertThat(updatedInboundDownstreamMessagesMetricsValue).isGreaterThan(inboundDownstreamMessagesMetricsValue); assertThat(updatedInboundDownstreamDecodedMessagesMetricsValue).isGreaterThan(inboundDownstreamDecodedMessagesMetricsValue); } } @Test + @Deprecated(since = "0.13.0", forRemoval = true) void shouldIncrementDownstreamMessagesOnProduceRequestWithFilter(KafkaCluster cluster, Topic topic) throws ExecutionException, InterruptedException { // the downstream messages and decoded messages is not yet differentiated by ApiKey @@ -214,8 +392,8 @@ void shouldIncrementDownstreamMessagesOnProduceRequestWithFilter(KafkaCluster cl var managementClient = tester.getManagementClient(); var producer = tester.producer()) { var metricList = managementClient.scrapeMetrics(); - var inboundDownstreamMessagesMetricsValue = getMetricsValue(metricList, "kroxylicious_inbound_downstream_messages_total", null); - var inboundDownstreamDecodedMessagesMetricsValue = getMetricsValue(metricList, "kroxylicious_inbound_downstream_decoded_messages_total", null); + var inboundDownstreamMessagesMetricsValue = getMetricsValue(metricList, "kroxylicious_inbound_downstream_messages_total"); + var inboundDownstreamDecodedMessagesMetricsValue = getMetricsValue(metricList, "kroxylicious_inbound_downstream_decoded_messages_total"); // When producer.send(new ProducerRecord<>(topic.name(), "my-key", "hello-world")).get(); @@ -223,8 +401,8 @@ void shouldIncrementDownstreamMessagesOnProduceRequestWithFilter(KafkaCluster cl // Then // updated metrics after some message were produced var updatedMetricsList = managementClient.scrapeMetrics(); - var updatedInboundDownstreamMessagesMetricsValue = getMetricsValue(updatedMetricsList, "kroxylicious_inbound_downstream_messages_total", null); - var updatedInboundDownstreamDecodedMessagesMetricsValue = getMetricsValue(updatedMetricsList, "kroxylicious_inbound_downstream_decoded_messages_total", null); + var updatedInboundDownstreamMessagesMetricsValue = getMetricsValue(updatedMetricsList, "kroxylicious_inbound_downstream_messages_total"); + var updatedInboundDownstreamDecodedMessagesMetricsValue = getMetricsValue(updatedMetricsList, "kroxylicious_inbound_downstream_decoded_messages_total"); assertThat(updatedInboundDownstreamMessagesMetricsValue).isGreaterThan(inboundDownstreamMessagesMetricsValue); assertThat(updatedInboundDownstreamDecodedMessagesMetricsValue).isGreaterThan(inboundDownstreamDecodedMessagesMetricsValue); } @@ -294,7 +472,7 @@ void shouldIncrementPayloadSizeBytesMetricsOnProduceRequest(KafkaCluster cluster } void assertMetricsDoesNotExist(List metricList, String metricsName, ApiKeys apiKey) { - assertThat(metricList) + Assertions.assertThat(metricList) .hasSizeGreaterThan(0) .noneSatisfy(simpleMetric -> { assertThat(simpleMetric.name()).isEqualTo(metricsName); @@ -305,7 +483,7 @@ void assertMetricsDoesNotExist(List metricList, String metricsName } void assertMetricsWithValue(List metricList, String metricsName, ApiKeys apiKey) { - assertThat(metricList) + Assertions.assertThat(metricList) .hasSizeGreaterThan(0) .anySatisfy(simpleMetric -> { assertThat(simpleMetric.name()).isEqualTo(metricsName); @@ -316,14 +494,16 @@ void assertMetricsWithValue(List metricList, String metricsName, A }); } - double getMetricsValue(List metricList, String metricsName, ApiKeys apiKey) { - if (apiKey != null) { - return metricList.stream().filter(simpleMetric -> simpleMetric.name().equals(metricsName) - && simpleMetric.labels().containsValue(apiKey.toString())).findFirst().get().value(); - } - else { - return metricList.stream().filter(simpleMetric -> simpleMetric.name().equals(metricsName)).findFirst().get().value(); - } + double getMetricsValue(List metricList, String metricsName) { + return getMetricsValue(metricList, metricsName, labels -> true); + } + + double getMetricsValue(List metricList, String metricsName, Predicate> labelPredicate) { + return metricList.stream() + .filter(simpleMetric -> simpleMetric.name().equals(metricsName)) + .filter(simpleMetric -> labelPredicate.test(simpleMetric.labels())) + .findFirst() + .orElseThrow().value(); } @NonNull diff --git a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/filter/DecodeAll.java b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/filter/DecodeAll.java new file mode 100644 index 0000000000..e9ddf02515 --- /dev/null +++ b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/filter/DecodeAll.java @@ -0,0 +1,46 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.proxy.filter; + +import java.util.concurrent.CompletionStage; + +import org.apache.kafka.common.message.RequestHeaderData; +import org.apache.kafka.common.message.ResponseHeaderData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ApiMessage; + +import io.kroxylicious.proxy.plugin.Plugin; +import io.kroxylicious.proxy.plugin.PluginConfigurationException; + +import edu.umd.cs.findbugs.annotations.NonNull; + +@Plugin(configType = Void.class) +public class DecodeAll implements FilterFactory { + + @Override + public Void initialize(FilterFactoryContext context, Void config) throws PluginConfigurationException { + return config; + } + + @NonNull + @Override + public Filter createFilter(FilterFactoryContext context, Void initializationData) { + return new DecodeAllFilter(); + } + + private static class DecodeAllFilter implements RequestFilter, ResponseFilter { + @Override + public CompletionStage onRequest(ApiKeys apiKey, RequestHeaderData header, ApiMessage request, FilterContext context) { + return context.forwardRequest(header, request); + } + + @Override + public CompletionStage onResponse(ApiKeys apiKey, ResponseHeaderData header, ApiMessage response, FilterContext context) { + return context.forwardResponse(header, response); + } + } +} diff --git a/kroxylicious-integration-tests/src/test/resources/META-INF/services/io.kroxylicious.proxy.filter.FilterFactory b/kroxylicious-integration-tests/src/test/resources/META-INF/services/io.kroxylicious.proxy.filter.FilterFactory index d37c2c2b3b..aa34ee6cdc 100644 --- a/kroxylicious-integration-tests/src/test/resources/META-INF/services/io.kroxylicious.proxy.filter.FilterFactory +++ b/kroxylicious-integration-tests/src/test/resources/META-INF/services/io.kroxylicious.proxy.filter.FilterFactory @@ -2,5 +2,6 @@ io.kroxylicious.proxy.filter.FixedClientIdFilterFactory io.kroxylicious.proxy.filter.RequestResponseMarkingFilterFactory io.kroxylicious.proxy.filter.OutOfBandSendFilterFactory io.kroxylicious.proxy.filter.RejectingCreateTopicFilterFactory +io.kroxylicious.proxy.filter.DecodeAll io.kroxylicious.proxy.InvocationCountingFilterFactory io.kroxylicious.proxy.CreateTopicRequest \ No newline at end of file diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/KafkaProxy.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/KafkaProxy.java index 45cfb432e2..9b9d90170a 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/KafkaProxy.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/KafkaProxy.java @@ -153,12 +153,7 @@ public KafkaProxy startup() { .toArray(CompletableFuture[]::new)) .join(); - // Pre-register counters/summaries to avoid creating them on first request and thus skewing the request latency - virtualClusterModels.forEach(virtualClusterModel -> { - List tags = Metrics.tags(Metrics.FLOWING_TAG, Metrics.DOWNSTREAM, Metrics.VIRTUAL_CLUSTER_TAG, virtualClusterModel.getClusterName()); - Metrics.taggedCounter(Metrics.KROXYLICIOUS_INBOUND_DOWNSTREAM_MESSAGES, tags); - Metrics.taggedCounter(Metrics.KROXYLICIOUS_INBOUND_DOWNSTREAM_DECODED_MESSAGES, tags); - }); + initDeprecatedMessageMetrics(); STARTUP_SHUTDOWN_LOGGER.info("Kroxylicious is started"); return this; } @@ -168,6 +163,16 @@ public KafkaProxy startup() { } } + @SuppressWarnings("removal") + private void initDeprecatedMessageMetrics() { + // Pre-register counters/summaries to avoid creating them on first request and thus skewing the request latency + virtualClusterModels.forEach(virtualClusterModel -> { + List tags = Metrics.tags(Metrics.FLOWING_TAG, Metrics.DOWNSTREAM, Metrics.VIRTUAL_CLUSTER_TAG, virtualClusterModel.getClusterName()); + Metrics.taggedCounter(Metrics.KROXYLICIOUS_INBOUND_DOWNSTREAM_MESSAGES, tags); + Metrics.taggedCounter(Metrics.KROXYLICIOUS_INBOUND_DOWNSTREAM_DECODED_MESSAGES, tags); + }); + } + private Map getApiKeyMaxVersionOverride(Configuration config) { Map apiKeyIdMaxVersion = config.development() .map(m -> m.get("apiKeyIdMaxVersionOverride")) diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/BareSaslRequest.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/BareSaslRequest.java index 15ed5654c4..6c93ba321d 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/BareSaslRequest.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/BareSaslRequest.java @@ -37,6 +37,21 @@ public int correlationId() { return 0; } + @Override + public short apiKeyId() { + return -1; + } + + @Override + public short apiVersion() { + return 0; + } + + @Override + public boolean isDecoded() { + return true; + } + @Override public boolean decodeResponse() { return decodeResponse; diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/BareSaslResponse.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/BareSaslResponse.java index dcc3f94d17..e0b24deddf 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/BareSaslResponse.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/BareSaslResponse.java @@ -33,6 +33,21 @@ public int correlationId() { return 0; } + @Override + public short apiKeyId() { + return -1; + } + + @Override + public short apiVersion() { + return 0; + } + + @Override + public boolean isDecoded() { + return true; + } + public byte[] bytes() { return bytes; } diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/DecodedFrame.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/DecodedFrame.java index 6b6406d43d..63aeb0e60e 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/DecodedFrame.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/DecodedFrame.java @@ -60,6 +60,22 @@ public int correlationId() { return correlationId; } + public short apiKeyId() { + return body.apiKey(); + } + + public ApiKeys apiKey() { + return ApiKeys.forId(apiKeyId()); + } + + public short apiVersion() { + return apiVersion; + } + + public boolean isDecoded() { + return true; + } + protected abstract short headerVersion(); public H header() { @@ -70,14 +86,6 @@ public B body() { return body; } - public ApiKeys apiKey() { - return ApiKeys.forId(body.apiKey()); - } - - public short apiVersion() { - return apiVersion; - } - @Override public final int estimateEncodedSize() { if (headerAndBodyEncodedLength != -1) { @@ -140,4 +148,5 @@ public void transferBuffersTo(DecodedFrame frame) { frame.buffers.addAll(this.buffers); this.buffers.clear(); } + } diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/DecodedRequestFrame.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/DecodedRequestFrame.java index 539485f379..bbca35a235 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/DecodedRequestFrame.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/DecodedRequestFrame.java @@ -7,6 +7,7 @@ import org.apache.kafka.common.message.ProduceRequestData; import org.apache.kafka.common.message.RequestHeaderData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ApiMessage; import static org.apache.kafka.common.protocol.ApiKeys.PRODUCE; @@ -31,7 +32,7 @@ public DecodedRequestFrame(short apiVersion, @Override public short headerVersion() { - return apiKey().messageType.requestHeaderVersion(apiVersion); + return ApiKeys.forId(apiKeyId()).messageType.requestHeaderVersion(apiVersion); } @Override @@ -45,7 +46,7 @@ public boolean hasResponse() { } private boolean isZeroAcksProduceRequest() { - return apiKey() == PRODUCE && ((ProduceRequestData) body).acks() == 0; + return apiKeyId() == PRODUCE.id && ((ProduceRequestData) body).acks() == 0; } } diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/DecodedResponseFrame.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/DecodedResponseFrame.java index 4674abd1c6..733d48e7d9 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/DecodedResponseFrame.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/DecodedResponseFrame.java @@ -6,6 +6,7 @@ package io.kroxylicious.proxy.frame; import org.apache.kafka.common.message.ResponseHeaderData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ApiMessage; /** @@ -20,6 +21,6 @@ public DecodedResponseFrame(short apiVersion, int correlationId, ResponseHeaderD } public short headerVersion() { - return apiKey().messageType.responseHeaderVersion(apiVersion); + return ApiKeys.forId(apiKeyId()).messageType.responseHeaderVersion(apiVersion); } } diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/Frame.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/Frame.java index 0fd0a8949c..7c2709f11d 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/Frame.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/Frame.java @@ -28,4 +28,12 @@ public interface Frame { */ int correlationId(); + /** Api key id of this frame. */ + short apiKeyId(); + + /** Api version of this frame. */ + short apiVersion(); + + /** true if this frame is decoded, false otherwise */ + boolean isDecoded(); } diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/OpaqueFrame.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/OpaqueFrame.java index e7b2c9338a..c5aaf8da42 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/OpaqueFrame.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/OpaqueFrame.java @@ -22,6 +22,12 @@ public abstract class OpaqueFrame implements Frame { private static final Logger LOGGER = LoggerFactory.getLogger(OpaqueFrame.class); + /** Api key id */ + private final short apiKeyId; + + /* Api Key version */ + private final short apiVersion; + /** * Number of bytes required for storing the frame length. */ @@ -33,11 +39,15 @@ public abstract class OpaqueFrame implements Frame { protected final ByteBuf buf; /** + * @param apiKeyId api key id + * @param apiVersion api key version * @param buf The message buffer (excluding the frame size) * @param correlationId The correlation id * @param length The length of the frame within {@code buf}. */ - OpaqueFrame(ByteBuf buf, int correlationId, int length) { + OpaqueFrame(short apiKeyId, short apiVersion, ByteBuf buf, int correlationId, int length) { + this.apiKeyId = apiKeyId; + this.apiVersion = apiVersion; Objects.requireNonNull(buf); this.length = length; this.correlationId = correlationId; @@ -47,11 +57,26 @@ public abstract class OpaqueFrame implements Frame { } } + @Override + public short apiKeyId() { + return apiKeyId; + } + + @Override + public short apiVersion() { + return apiVersion; + } + @Override public int correlationId() { return correlationId; } + @Override + public boolean isDecoded() { + return false; + } + @Override public int estimateEncodedSize() { return FRAME_SIZE_LENGTH + length; @@ -88,6 +113,8 @@ public ByteBuf buf() { public String toString() { return getClass().getSimpleName() + "(" + "length=" + length + + ", apiKeyId=" + apiKeyId + + ", apiVersion=" + apiVersion + ", buf=" + buf + ')'; } diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/OpaqueRequestFrame.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/OpaqueRequestFrame.java index 3f4593aa8e..f23294cc11 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/OpaqueRequestFrame.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/OpaqueRequestFrame.java @@ -5,8 +5,6 @@ */ package io.kroxylicious.proxy.frame; -import org.apache.kafka.common.protocol.ApiKeys; - import io.netty.buffer.ByteBuf; /** @@ -21,17 +19,21 @@ public class OpaqueRequestFrame extends OpaqueFrame implements RequestFrame { * Creates an opaque request. * * @param buf The message buffer (excluding the frame size) + * @param apiKeyId api key + * @param apiVersion api version * @param correlationId correlation id * @param decodeResponse whether the response will be decoded * @param length length of the request * @param hasResponse true if the request expects a response */ public OpaqueRequestFrame(ByteBuf buf, + short apiKeyId, + short apiVersion, int correlationId, boolean decodeResponse, int length, boolean hasResponse) { - super(buf, correlationId, length); + super(apiKeyId, apiVersion, buf, correlationId, length); this.decodeResponse = decodeResponse; this.hasResponse = hasResponse; } @@ -48,21 +50,9 @@ public boolean hasResponse() { @Override public String toString() { - int index = buf.readerIndex(); - try { - var apiId = buf.readShort(); - // TODO handle unknown api key - ApiKeys apiKey = ApiKeys.forId(apiId); - short apiVersion = buf.readShort(); - return getClass().getSimpleName() + "(" + - "length=" + length + - ", apiKey=" + apiKey + - ", apiVersion=" + apiVersion + - ", buf=" + buf + - ')'; - } - finally { - buf.readerIndex(index); - } + return getClass().getSimpleName() + "(" + + "length=" + length + + ", buf=" + buf + + ')'; } } diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/OpaqueResponseFrame.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/OpaqueResponseFrame.java index f77cbe3aa7..f676a24735 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/OpaqueResponseFrame.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/OpaqueResponseFrame.java @@ -14,12 +14,14 @@ public class OpaqueResponseFrame extends OpaqueFrame implements ResponseFrame { /** * Creates an opaque response. * + * @param apiKeyId api key id + * @param apiVersion api key version * @param buf The message buffer (excluding the frame size) * @param correlationId correlation id * @param length length of the response */ - public OpaqueResponseFrame(ByteBuf buf, int correlationId, int length) { - super(buf, correlationId, length); + public OpaqueResponseFrame(short apiKeyId, short apiVersion, ByteBuf buf, int correlationId, int length) { + super(apiKeyId, apiVersion, buf, correlationId, length); } @Override diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/FilterHandler.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/FilterHandler.java index a44a0823c1..5415cbce0b 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/FilterHandler.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/FilterHandler.java @@ -379,7 +379,7 @@ private void forwardResponse(DecodedFrame decodedFrame, ResponseHeaderData throw new AssertionError("Filter '" + filterDescriptor() + "': Attempt to use forwardResponse with a non-response: " + name); } if (decodedFrame instanceof RequestFrame) { - if (message.apiKey() != decodedFrame.apiKey().id) { + if (message.apiKey() != decodedFrame.apiKeyId()) { throw new AssertionError( "Filter '" + filterDescriptor() + "': Attempt to respond with ApiMessage of type " + ApiKeys.forId(message.apiKey()) + " but request is of type " + decodedFrame.apiKey()); diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java index ce2fdbb4b1..f879372277 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java @@ -50,7 +50,11 @@ import io.kroxylicious.proxy.internal.codec.DecodePredicate; import io.kroxylicious.proxy.internal.codec.KafkaRequestEncoder; import io.kroxylicious.proxy.internal.codec.KafkaResponseDecoder; -import io.kroxylicious.proxy.internal.net.EndpointGateway; +import io.kroxylicious.proxy.internal.metrics.DeprecatedUpstreamMessageMetrics; +import io.kroxylicious.proxy.internal.metrics.MessageMetrics; +import io.kroxylicious.proxy.internal.net.BrokerEndpointBinding; +import io.kroxylicious.proxy.internal.net.EndpointBinding; +import io.kroxylicious.proxy.internal.util.Metrics; import io.kroxylicious.proxy.model.VirtualClusterModel; import io.kroxylicious.proxy.service.HostPort; import io.kroxylicious.proxy.tag.VisibleForTesting; @@ -76,7 +80,7 @@ public class KafkaProxyFrontendHandler private final boolean logNetwork; private final boolean logFrames; private final VirtualClusterModel virtualClusterModel; - private final EndpointGateway endpointGateway; + private final EndpointBinding endpointBinding; private final NetFilter netFilter; private final SaslDecodePredicate dp; private final ProxyChannelStateMachine proxyChannelStateMachine; @@ -124,21 +128,21 @@ SslHandler getSslHandler(ChannelHandlerContext ctx) { KafkaProxyFrontendHandler( @NonNull NetFilter netFilter, @NonNull SaslDecodePredicate dp, - EndpointGateway endpointGateway, + @NonNull EndpointBinding endpointBinding, @NonNull String clusterName) { - this(netFilter, dp, endpointGateway, new ProxyChannelStateMachine(clusterName)); + this(netFilter, dp, endpointBinding, new ProxyChannelStateMachine(clusterName)); } @VisibleForTesting KafkaProxyFrontendHandler( @NonNull NetFilter netFilter, @NonNull SaslDecodePredicate dp, - @NonNull EndpointGateway endpointGateway, + @NonNull EndpointBinding endpointBinding, @NonNull ProxyChannelStateMachine proxyChannelStateMachine) { this.netFilter = netFilter; this.dp = dp; - this.endpointGateway = endpointGateway; - this.virtualClusterModel = endpointGateway.virtualCluster(); + this.endpointBinding = endpointBinding; + this.virtualClusterModel = endpointBinding.endpointGateway().virtualCluster(); this.proxyChannelStateMachine = proxyChannelStateMachine; this.logNetwork = virtualClusterModel.isLogNetwork(); this.logFrames = virtualClusterModel.isLogFrames(); @@ -341,7 +345,7 @@ public void channelReadComplete(ChannelHandlerContext clientCtx) { */ @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - proxyChannelStateMachine.onClientException(cause, endpointGateway.getDownstreamSslContext().isPresent()); + proxyChannelStateMachine.onClientException(cause, endpointBinding.endpointGateway().getDownstreamSslContext().isPresent()); } /** @@ -509,6 +513,13 @@ void inConnecting( pipeline.addFirst("frameLogger", new LoggingHandler("io.kroxylicious.proxy.internal.UpstreamFrameLogger")); } addFiltersToPipeline(filters, pipeline, inboundChannel); + Integer nodeId = endpointBinding instanceof BrokerEndpointBinding brokerEndpointBinding ? brokerEndpointBinding.nodeId() : null; + var proxyToServerCounterProvider = Metrics.KROXYLICIOUS_PROXY_TO_SERVER_REQUEST_TOTAL_METER_PROVIDER + .create(this.virtualClusterModel.getClusterName(), nodeId); + var serverToProxyCounterProvider = Metrics.KROXYLICIOUS_SERVER_TO_PROXY_RESPONSE_TOTAL_METER_PROVIDER.create(virtualClusterModel.getClusterName(), nodeId); + + pipeline.addFirst("upstreamMetrics", new MessageMetrics(serverToProxyCounterProvider, proxyToServerCounterProvider)); + pipeline.addFirst("deprecatedUpstreamMetrics", getDeprecatedUpstreamMessageMetrics(this.virtualClusterModel.getClusterName())); pipeline.addFirst("responseDecoder", new KafkaResponseDecoder(correlationManager, virtualClusterModel.socketFrameMaxSizeBytes(), virtualClusterModel.getClusterName())); pipeline.addFirst("requestEncoder", new KafkaRequestEncoder(correlationManager)); @@ -538,6 +549,12 @@ void inConnecting( }); } + @NonNull + @SuppressWarnings("removal") + private DeprecatedUpstreamMessageMetrics getDeprecatedUpstreamMessageMetrics(String clusterName) { + return new DeprecatedUpstreamMessageMetrics(clusterName); + } + /** Ugly hack used for testing */ @NonNull @VisibleForTesting @@ -699,4 +716,4 @@ private ResponseFrame errorResponse( } return errorResponse; } -} \ No newline at end of file +} diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java index a75335b0f4..33ed7c8836 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java @@ -38,11 +38,16 @@ import io.kroxylicious.proxy.internal.filter.BrokerAddressFilter; import io.kroxylicious.proxy.internal.filter.EagerMetadataLearner; import io.kroxylicious.proxy.internal.filter.NettyFilterContext; +import io.kroxylicious.proxy.internal.metrics.DeprecatedDownstreamMessageMetrics; +import io.kroxylicious.proxy.internal.metrics.MessageMetrics; +import io.kroxylicious.proxy.internal.net.BrokerEndpointBinding; import io.kroxylicious.proxy.internal.net.Endpoint; import io.kroxylicious.proxy.internal.net.EndpointBinding; import io.kroxylicious.proxy.internal.net.EndpointBindingResolver; import io.kroxylicious.proxy.internal.net.EndpointGateway; import io.kroxylicious.proxy.internal.net.EndpointReconciler; +import io.kroxylicious.proxy.internal.util.Metrics; +import io.kroxylicious.proxy.model.VirtualClusterModel; import io.kroxylicious.proxy.tag.VisibleForTesting; public class KafkaProxyInitializer extends ChannelInitializer { @@ -192,9 +197,17 @@ void addHandlers(SocketChannel ch, EndpointBinding binding) { var dp = new SaslDecodePredicate(!authnHandlers.isEmpty()); // The decoder, this only cares about the filters // because it needs to know whether to decode requests - KafkaRequestDecoder decoder = new KafkaRequestDecoder(dp, virtualCluster.socketFrameMaxSizeBytes(), apiVersionsService, virtualCluster.getClusterName()); + Integer nodeId = binding instanceof BrokerEndpointBinding brokerEndpointBinding ? brokerEndpointBinding.nodeId() : null; + KafkaRequestDecoder decoder = new KafkaRequestDecoder(dp, virtualCluster.socketFrameMaxSizeBytes(), apiVersionsService); pipeline.addLast("requestDecoder", decoder); + pipeline.addLast("responseEncoder", new KafkaResponseEncoder()); + + var clientToProxyCounterProvider = Metrics.KROXYLICIOUS_CLIENT_TO_PROXY_REQUEST_TOTAL_METER_PROVIDER.create(virtualCluster.getClusterName(), nodeId); + var proxyToClientCounterProvider = Metrics.KROXYLICIOUS_PROXY_TO_CLIENT_RESPONSE_TOTAL_METER_PROVIDER.create(virtualCluster.getClusterName(), nodeId); + + pipeline.addLast("downstreamMetrics", new MessageMetrics(clientToProxyCounterProvider, proxyToClientCounterProvider)); + pipeline.addLast("deprecatedDownstreamMetrics", deprecatedMessageMetricHandler(virtualCluster)); pipeline.addLast("responseOrderer", new ResponseOrderer()); if (virtualCluster.isLogFrames()) { pipeline.addLast("frameLogger", new LoggingHandler("io.kroxylicious.proxy.internal.DownstreamFrameLogger", LogLevel.INFO)); @@ -214,7 +227,7 @@ void addHandlers(SocketChannel ch, EndpointBinding binding) { endpointReconciler, new ApiVersionsIntersectFilter(apiVersionsService), new ApiVersionsDowngradeFilter(apiVersionsService)); - var frontendHandler = new KafkaProxyFrontendHandler(netFilter, dp, binding.endpointGateway(), virtualCluster.getClusterName()); + var frontendHandler = new KafkaProxyFrontendHandler(netFilter, dp, binding, virtualCluster.getClusterName()); pipeline.addLast("netHandler", frontendHandler); addLoggingErrorHandler(pipeline); @@ -222,6 +235,11 @@ void addHandlers(SocketChannel ch, EndpointBinding binding) { LOGGER.debug("{}: Initial pipeline: {}", ch, pipeline); } + @SuppressWarnings("removal") + private DeprecatedDownstreamMessageMetrics deprecatedMessageMetricHandler(VirtualClusterModel virtualCluster) { + return new DeprecatedDownstreamMessageMetrics(virtualCluster.getClusterName()); + } + private static void addLoggingErrorHandler(ChannelPipeline pipeline) { pipeline.addLast(LOGGING_INBOUND_ERROR_HANDLER_NAME, LOGGING_INBOUND_ERROR_HANDLER); } @@ -298,4 +316,4 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { cause.getClass().getSimpleName(), cause.getMessage()); } } -} \ No newline at end of file +} diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/codec/KafkaRequestDecoder.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/codec/KafkaRequestDecoder.java index d847848920..04a54f6c6e 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/codec/KafkaRequestDecoder.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/codec/KafkaRequestDecoder.java @@ -5,8 +5,6 @@ */ package io.kroxylicious.proxy.internal.codec; -import java.util.List; - import org.apache.kafka.common.message.ApiVersionsRequestData; import org.apache.kafka.common.message.RequestHeaderData; import org.apache.kafka.common.protocol.ApiKeys; @@ -15,8 +13,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.Tag; import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; @@ -26,7 +22,6 @@ import io.kroxylicious.proxy.frame.RequestFrame; import io.kroxylicious.proxy.internal.ApiVersionsServiceImpl; import io.kroxylicious.proxy.internal.filter.ApiVersionsDowngradeFilter; -import io.kroxylicious.proxy.internal.util.Metrics; import edu.umd.cs.findbugs.annotations.NonNull; @@ -37,18 +32,11 @@ public class KafkaRequestDecoder extends KafkaMessageDecoder { private final DecodePredicate decodePredicate; private final ApiVersionsServiceImpl apiVersionsService; - private final Counter inboundMessageCounter; - private final Counter decodedMessagesCounter; - private final String clusterName; - public KafkaRequestDecoder(DecodePredicate decodePredicate, int socketFrameMaxSize, ApiVersionsServiceImpl apiVersionsService, String clusterName) { + public KafkaRequestDecoder(DecodePredicate decodePredicate, int socketFrameMaxSize, ApiVersionsServiceImpl apiVersionsService) { super(socketFrameMaxSize); this.decodePredicate = decodePredicate; this.apiVersionsService = apiVersionsService; - this.clusterName = clusterName; - List tags = Metrics.tags(Metrics.FLOWING_TAG, Metrics.DOWNSTREAM, Metrics.VIRTUAL_CLUSTER_TAG, clusterName); - inboundMessageCounter = Metrics.taggedCounter(Metrics.KROXYLICIOUS_INBOUND_DOWNSTREAM_MESSAGES, tags); - decodedMessagesCounter = Metrics.taggedCounter(Metrics.KROXYLICIOUS_INBOUND_DOWNSTREAM_DECODED_MESSAGES, tags); } @Override @@ -73,17 +61,16 @@ protected Frame decodeHeaderAndBody(ChannelHandlerContext ctx, ByteBuf in, final final int startOfMessage = in.readerIndex(); int correlationId = in.readInt(); LOGGER.debug("{}: {} downstream correlation id: {}", ctx, apiKey, correlationId); - RequestHeaderData header = null; - final ByteBufAccessorImpl accessor; - inboundMessageCounter.increment(); + var decodeRequest = decodePredicate.shouldDecodeRequest(apiKey, apiVersion); + LOGGER.debug("Decode {}/v{} request? {}, Predicate {} ", apiKey, apiVersion, decodeRequest, decodePredicate); boolean decodeResponse = decodePredicate.shouldDecodeResponse(apiKey, apiVersion); LOGGER.debug("Decode {}/v{} response? {}, Predicate {}", apiKey, apiVersion, decodeResponse, decodePredicate); short headerVersion = apiKey.requestHeaderVersion(apiVersion); + RequestHeaderData header = null; + final ByteBufAccessorImpl accessor; if (decodeRequest) { - decodedMessagesCounter.increment(); - Metrics.payloadSizeBytesUpstreamSummary(apiKey, apiVersion, clusterName).record(length); if (log().isTraceEnabled()) { // avoid boxing log().trace("{}: headerVersion {}", ctx, headerVersion); } @@ -131,7 +118,7 @@ protected Frame decodeHeaderAndBody(ChannelHandlerContext ctx, ByteBuf in, final hasResponse = acks != 0; } in.readerIndex(sof); - frame = opaqueFrame(in, correlationId, decodeResponse, length, hasResponse); + frame = opaqueFrame(in, apiId, apiVersion, correlationId, decodeResponse, length, hasResponse); in.readerIndex(sof + length); } return frame; @@ -193,12 +180,16 @@ else if (apiVersion <= 12) { // Flexible versions } private OpaqueRequestFrame opaqueFrame(ByteBuf in, + short apiKey, + short apiVersion, int correlationId, boolean decodeResponse, int length, boolean hasResponse) { return new OpaqueRequestFrame( in.readSlice(length).retain(), + apiKey, + apiVersion, correlationId, decodeResponse, length, @@ -209,4 +200,4 @@ private RequestHeaderData readHeader(short headerVersion, Readable accessor) { return new RequestHeaderData(accessor, headerVersion); } -} \ No newline at end of file +} diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/codec/KafkaResponseDecoder.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/codec/KafkaResponseDecoder.java index 2189bc436b..a0e0d87d3c 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/codec/KafkaResponseDecoder.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/codec/KafkaResponseDecoder.java @@ -21,7 +21,6 @@ import io.kroxylicious.proxy.frame.OpaqueFrame; import io.kroxylicious.proxy.frame.OpaqueResponseFrame; import io.kroxylicious.proxy.internal.InternalResponseFrame; -import io.kroxylicious.proxy.internal.util.Metrics; public class KafkaResponseDecoder extends KafkaMessageDecoder { @@ -72,7 +71,6 @@ else if (LOGGER.isDebugEnabled()) { ApiMessage body = BodyDecoder.decodeResponse(apiKey, apiVersion, accessor); log().trace("{}: Body: {}", ctx, body); Filter recipient = correlation.recipient(); - Metrics.payloadSizeBytesDownstreamSummary(apiKey, apiVersion, clusterName).record(length); if (recipient == null) { frame = new DecodedResponseFrame<>(apiVersion, correlationId, header, body); } @@ -81,14 +79,14 @@ else if (LOGGER.isDebugEnabled()) { } } else { - frame = opaqueFrame(in, correlationId, length); + frame = opaqueFrame(correlation.apiKey(), correlation.apiVersion(), in, correlationId, length); } log().trace("{}: Frame: {}", ctx, frame); return frame; } - private OpaqueFrame opaqueFrame(ByteBuf in, int correlationId, int length) { - return new OpaqueResponseFrame(in.readSlice(length).retain(), correlationId, length); + private OpaqueFrame opaqueFrame(short apiKeyId, short apiVersion, ByteBuf in, int correlationId, int length) { + return new OpaqueResponseFrame(apiKeyId, apiVersion, in.readSlice(length).retain(), correlationId, length); } private ResponseHeaderData readHeader(short headerVersion, Readable accessor) { diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedDownstreamMessageMetrics.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedDownstreamMessageMetrics.java new file mode 100644 index 0000000000..650a47a154 --- /dev/null +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedDownstreamMessageMetrics.java @@ -0,0 +1,51 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.proxy.internal.metrics; + +import java.util.List; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Tag; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; + +import io.kroxylicious.proxy.frame.DecodedRequestFrame; +import io.kroxylicious.proxy.internal.util.Metrics; + +@Deprecated(since = "0.13.0", forRemoval = true) +public class DeprecatedDownstreamMessageMetrics extends ChannelInboundHandlerAdapter { + private final String clusterName; + private final Counter inboundMessageCounter; + private final Counter decodedMessagesCounter; + + @SuppressWarnings("removal") + public DeprecatedDownstreamMessageMetrics(String clusterName) { + this.clusterName = clusterName; + List tags = Metrics.tags(Metrics.FLOWING_TAG, Metrics.DOWNSTREAM, Metrics.VIRTUAL_CLUSTER_TAG, clusterName); + inboundMessageCounter = Metrics.taggedCounter(Metrics.KROXYLICIOUS_INBOUND_DOWNSTREAM_MESSAGES, tags); + decodedMessagesCounter = Metrics.taggedCounter(Metrics.KROXYLICIOUS_INBOUND_DOWNSTREAM_DECODED_MESSAGES, tags); + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + + try { + inboundMessageCounter.increment(); + if (msg instanceof DecodedRequestFrame decodedRequestFrame) { + decodedMessagesCounter.increment(); + // TODO the request might be updated as it travels through the proxy + // so this avoids caching the size prematurely + int size = new DecodedRequestFrame<>(decodedRequestFrame.apiVersion(), decodedRequestFrame.correlationId(), decodedRequestFrame.decodeResponse(), + decodedRequestFrame.header(), decodedRequestFrame.body()).estimateEncodedSize(); + Metrics.payloadSizeBytesDownstreamSummary(decodedRequestFrame.apiKey(), decodedRequestFrame.apiVersion(), clusterName).record(size); + } + } + finally { + super.channelRead(ctx, msg); + } + } +} diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedUpstreamMessageMetrics.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedUpstreamMessageMetrics.java new file mode 100644 index 0000000000..b3d47a3f3c --- /dev/null +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedUpstreamMessageMetrics.java @@ -0,0 +1,41 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.proxy.internal.metrics; + +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; + +import io.kroxylicious.proxy.frame.DecodedResponseFrame; +import io.kroxylicious.proxy.internal.util.Metrics; + +@Deprecated(since = "0.13.0", forRemoval = true) +public class DeprecatedUpstreamMessageMetrics extends ChannelInboundHandlerAdapter { + private final String clusterName; + + @SuppressWarnings("removal") + public DeprecatedUpstreamMessageMetrics(String clusterName) { + this.clusterName = clusterName; + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + + try { + if (msg instanceof DecodedResponseFrame decodedResponseFrame) { + // TODO the request might be updated as it travels through the proxy + // so this avoids caching the size prematurely + int size = new DecodedResponseFrame<>(decodedResponseFrame.apiVersion(), decodedResponseFrame.correlationId(), decodedResponseFrame.header(), + decodedResponseFrame.body()).estimateEncodedSize(); + Metrics.payloadSizeBytesUpstreamSummary(decodedResponseFrame.apiKey(), decodedResponseFrame.apiVersion(), clusterName) + .record(size); + } + } + finally { + super.channelRead(ctx, msg); + } + } +} diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/MessageMetrics.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/MessageMetrics.java new file mode 100644 index 0000000000..98b06f96f3 --- /dev/null +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/MessageMetrics.java @@ -0,0 +1,69 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.proxy.internal.metrics; + +import java.util.Optional; + +import org.apache.kafka.common.protocol.ApiKeys; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Meter.MeterProvider; +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelPromise; + +import io.kroxylicious.proxy.frame.Frame; +import io.kroxylicious.proxy.internal.util.Metrics; + +public class MessageMetrics extends ChannelDuplexHandler { + private final MeterProvider readCounterProvider; + private final MeterProvider writeCounterProvider; + + public MessageMetrics(MeterProvider requestCounterProvider, MeterProvider responseCounterProvider) { + this.readCounterProvider = requestCounterProvider; + this.writeCounterProvider = responseCounterProvider; + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + try { + Optional.ofNullable(readCounterProvider) + .ifPresent(counter -> count(msg, counter)); + } + finally { + super.channelRead(ctx, msg); + } + + } + + @Override + public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { + + try { + Optional.ofNullable(writeCounterProvider) + .ifPresent(counter -> count(msg, counter)); + } + finally { + super.write(ctx, msg, promise); + } + } + + private void count(Object msg, MeterProvider counterProvider) { + if (msg instanceof Frame frame) { + var apiKey = ApiKeys.forId(frame.apiKeyId()); + short version = frame.apiVersion(); + boolean decoded = frame.isDecoded(); + counterProvider + .withTags(Metrics.DECODED_LABEL, Boolean.toString(decoded), + Metrics.API_KEY_LABEL, apiKey.name(), + Metrics.API_VERSION_LABEL, Short.toString(version)) + .increment(); + + } + } + +} diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java index 700a2c0d0e..23dc7ef286 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java @@ -9,42 +9,116 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.Optional; import org.apache.kafka.common.protocol.ApiKeys; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.DistributionSummary; +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.Meter.MeterProvider; import io.micrometer.core.instrument.Tag; import edu.umd.cs.findbugs.annotations.NonNull; import static io.micrometer.core.instrument.Metrics.counter; +import static io.micrometer.core.instrument.Metrics.globalRegistry; import static io.micrometer.core.instrument.Metrics.summary; public class Metrics { - // creating a constant for all Metrics in the one place so we can easily see what metrics there are + // Common Labels + public static final String VIRTUAL_CLUSTER_LABEL = "virtual_cluster"; + public static final String NODE_ID_LABEL = "node_id"; + public static final String API_KEY_LABEL = "api_key"; + public static final String API_VERSION_LABEL = "api_version"; + public static final String DECODED_LABEL = "decoded"; + + // Base Metric Names + + private static final String KROXYLICIOUS_CLIENT_TO_PROXY_REQUEST_BASE_METER_NAME = "kroxylicious_client_to_proxy_request"; + private static final String KROXYLICIOUS_PROXY_TO_SERVER_REQUEST_BASE_METER_NAME = "kroxylicious_proxy_to_server_request"; + private static final String KROXYLICIOUS_SERVER_TO_PROXY_RESPONSE_BASE_METER_NAME = "kroxylicious_server_to_proxy_response"; + private static final String KROXYLICIOUS_PROXY_TO_CLIENT_RESPONSE_BASE_METER_NAME = "kroxylicious_proxy_to_client_response"; + + // Meter Providers + // (Callers use the providers to create the meters they need, augmenting with any tags). + public static final ClusterNodeSpecificMetricProviderCreator KROXYLICIOUS_CLIENT_TO_PROXY_REQUEST_TOTAL_METER_PROVIDER = (virtualClusterName, + nodeId) -> Counter.builder( + KROXYLICIOUS_CLIENT_TO_PROXY_REQUEST_BASE_METER_NAME) + .description( + "Incremented by one every time a request arrives at the proxy from the downstream (client).") + .tag(VIRTUAL_CLUSTER_LABEL, + virtualClusterName) + .tag(NODE_ID_LABEL, + nodeIdToLabelValue( + nodeId)) + .withRegistry( + globalRegistry); + + public static final ClusterNodeSpecificMetricProviderCreator KROXYLICIOUS_PROXY_TO_SERVER_REQUEST_TOTAL_METER_PROVIDER = (virtualClusterName, + nodeId) -> Counter.builder( + KROXYLICIOUS_PROXY_TO_SERVER_REQUEST_BASE_METER_NAME) + .description( + "Incremented by one every time a request (#1) goes from the proxy to the upstream (server).") + .tag(VIRTUAL_CLUSTER_LABEL, + virtualClusterName) + .tag(NODE_ID_LABEL, + nodeIdToLabelValue( + nodeId)) + .withRegistry( + globalRegistry); + + public static final ClusterNodeSpecificMetricProviderCreator KROXYLICIOUS_SERVER_TO_PROXY_RESPONSE_TOTAL_METER_PROVIDER = (virtualClusterName, + nodeId) -> Counter.builder( + KROXYLICIOUS_SERVER_TO_PROXY_RESPONSE_BASE_METER_NAME) + .description( + "Incremented by one every time a response (#1) arrives at the proxy from the upstream (server).") + .tag(VIRTUAL_CLUSTER_LABEL, + virtualClusterName) + .tag(NODE_ID_LABEL, + nodeIdToLabelValue( + nodeId)) + .withRegistry( + globalRegistry); + + public static final ClusterNodeSpecificMetricProviderCreator KROXYLICIOUS_PROXY_TO_CLIENT_RESPONSE_TOTAL_METER_PROVIDER = (virtualClusterName, + nodeId) -> Counter.builder( + KROXYLICIOUS_PROXY_TO_CLIENT_RESPONSE_BASE_METER_NAME) + .description( + "Incremented by one every time a response goes from the proxy to the downstream (client).") + .tag(VIRTUAL_CLUSTER_LABEL, + virtualClusterName) + .tag(NODE_ID_LABEL, + nodeIdToLabelValue( + nodeId)) + .withRegistry( + globalRegistry); + + @Deprecated(since = "0.13.0", forRemoval = true) + public static final String KROXYLICIOUS_INBOUND_DOWNSTREAM_MESSAGES = "kroxylicious_inbound_downstream_messages"; + + @Deprecated(since = "0.13.0", forRemoval = true) + public static final String KROXYLICIOUS_INBOUND_DOWNSTREAM_DECODED_MESSAGES = "kroxylicious_inbound_downstream_decoded_messages"; private static final String KROXYLICIOUS_DOWNSTREAM = "kroxylicious_downstream_"; - private static final String KROXYLICIOUS_UPSTREAM = "kroxylicious_upstream_"; - public static final String KROXYLICIOUS_INBOUND_DOWNSTREAM_MESSAGES = "kroxylicious_inbound_downstream_messages"; + private static final String KROXYLICIOUS_UPSTREAM = "kroxylicious_upstream_"; public static final String KROXYLICIOUS_DOWNSTREAM_CONNECTIONS = KROXYLICIOUS_DOWNSTREAM + "connections"; - public static final String KROXYLICIOUS_DOWNSTREAM_ERRORS = KROXYLICIOUS_DOWNSTREAM + "errors"; + public static final String KROXYLICIOUS_DOWNSTREAM_ERRORS = KROXYLICIOUS_DOWNSTREAM + "errors"; public static final String KROXYLICIOUS_UPSTREAM_CONNECTIONS = KROXYLICIOUS_UPSTREAM + "connections"; + public static final String KROXYLICIOUS_UPSTREAM_CONNECTION_ATTEMPTS = KROXYLICIOUS_UPSTREAM + "connection_attempts"; public static final String KROXYLICIOUS_UPSTREAM_CONNECTION_FAILURES = KROXYLICIOUS_UPSTREAM + "connection_failures"; public static final String KROXYLICIOUS_UPSTREAM_ERRORS = KROXYLICIOUS_UPSTREAM + "errors"; - - public static final String KROXYLICIOUS_INBOUND_DOWNSTREAM_DECODED_MESSAGES = "kroxylicious_inbound_downstream_decoded_messages"; - public static final String KROXYLICIOUS_PAYLOAD_SIZE_BYTES = "kroxylicious_payload_size_bytes"; public static final String FLOWING_TAG = "flowing"; - public static final String VIRTUAL_CLUSTER_TAG = "virtualCluster"; + public static final String VIRTUAL_CLUSTER_TAG = "virtualCluster"; public static final String DOWNSTREAM = "downstream"; + public static final String UPSTREAM = "upstream"; @NonNull @@ -95,4 +169,15 @@ private static String required(String value) { } return value; } + + @NonNull + private static String nodeIdToLabelValue(Integer nodeId) { + return Optional.ofNullable(nodeId).map(Object::toString).orElse("bootstrap"); + } + + @FunctionalInterface + public interface ClusterNodeSpecificMetricProviderCreator { + + MeterProvider create(String virtualCluster, Integer nodeId); + } } \ No newline at end of file diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/FilterHarness.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/FilterHarness.java index 85e5cde488..5bb348e79c 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/FilterHarness.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/FilterHarness.java @@ -149,7 +149,7 @@ protected OpaqueRequestFrame writeArbitraryOpaqueRequest() { } protected OpaqueRequestFrame writeArbitraryOpaqueRequest(ByteBuf buffer) { - OpaqueRequestFrame frame = new OpaqueRequestFrame(buffer, 55, false, buffer.readableBytes(), false); + OpaqueRequestFrame frame = new OpaqueRequestFrame(buffer, ApiKeys.PRODUCE.id, ApiKeys.PRODUCE.latestVersion(), 55, false, buffer.readableBytes(), false); channel.writeOneOutbound(frame); return frame; } @@ -165,7 +165,7 @@ protected OpaqueResponseFrame writeArbitraryOpaqueResponse() { } protected OpaqueResponseFrame writeArbitraryOpaqueResponse(ByteBuf buffer) { - OpaqueResponseFrame frame = new OpaqueResponseFrame(buffer, 55, buffer.readableBytes()); + OpaqueResponseFrame frame = new OpaqueResponseFrame(ApiKeys.PRODUCE.id, ApiKeys.PRODUCE.latestVersion(), buffer, 55, buffer.readableBytes()); channel.writeOneInbound(frame); return frame; } diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaAuthnHandlerTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaAuthnHandlerTest.java index b860d2fed4..dc434b5c20 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaAuthnHandlerTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaAuthnHandlerTest.java @@ -578,6 +578,21 @@ public int estimateEncodedSize() { public void encode(ByteBufAccessor out) { } + @Override + public short apiKeyId() { + return 0; + } + + @Override + public short apiVersion() { + return 0; + } + + @Override + public boolean isDecoded() { + return false; + } + @Override public boolean decodeResponse() { return false; diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandlerMockCollaboratorsTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandlerMockCollaboratorsTest.java index e598297feb..ee56902c2d 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandlerMockCollaboratorsTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandlerMockCollaboratorsTest.java @@ -25,6 +25,7 @@ import io.netty.handler.codec.haproxy.HAProxyProxiedProtocol; import io.kroxylicious.proxy.filter.NetFilter; +import io.kroxylicious.proxy.internal.net.EndpointBinding; import io.kroxylicious.proxy.internal.net.EndpointGateway; import io.kroxylicious.proxy.model.VirtualClusterModel; @@ -50,6 +51,9 @@ class KafkaProxyFrontendHandlerMockCollaboratorsTest { @Mock VirtualClusterModel virtualCluster; + @Mock + EndpointBinding endpointBinding; + @Mock EndpointGateway endpointGateway; @@ -63,10 +67,11 @@ class KafkaProxyFrontendHandlerMockCollaboratorsTest { @BeforeEach void setUp() { when(endpointGateway.virtualCluster()).thenReturn(virtualCluster); + when(endpointBinding.endpointGateway()).thenReturn(endpointGateway); handler = new KafkaProxyFrontendHandler( netFilter, NO_SASL_DECODE_PREDICATE, - endpointGateway, + endpointBinding, proxyChannelStateMachine); } diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandlerTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandlerTest.java index 4a453db85b..2e87853a64 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandlerTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandlerTest.java @@ -58,6 +58,7 @@ import io.kroxylicious.proxy.internal.codec.FrameOversizedException; import io.kroxylicious.proxy.internal.codec.KafkaRequestDecoder; import io.kroxylicious.proxy.internal.codec.RequestDecoderTest; +import io.kroxylicious.proxy.internal.net.EndpointBinding; import io.kroxylicious.proxy.model.VirtualClusterModel; import io.kroxylicious.proxy.model.VirtualClusterModel.VirtualClusterGatewayModel; import io.kroxylicious.proxy.service.HostPort; @@ -157,9 +158,13 @@ public static List provideArgsForExpectedFlow() { @Test void testMessageHandledAfterConnectingBeforeConnected() { // Given + VirtualClusterModel virtualClusterModel = mock(VirtualClusterModel.class); + when(virtualClusterModel.getClusterName()).thenReturn("cluster"); VirtualClusterGatewayModel virtualClusterListenerModel = mock(VirtualClusterGatewayModel.class); - when(virtualClusterListenerModel.virtualCluster()).thenReturn(mock(VirtualClusterModel.class)); - KafkaProxyFrontendHandler handler = handler(this::getNetFilter, new SaslDecodePredicate(false), virtualClusterListenerModel); + when(virtualClusterListenerModel.virtualCluster()).thenReturn(virtualClusterModel); + EndpointBinding endpointBinding = mock(EndpointBinding.class); + when(endpointBinding.endpointGateway()).thenReturn(virtualClusterListenerModel); + KafkaProxyFrontendHandler handler = handler(this::getNetFilter, new SaslDecodePredicate(false), endpointBinding); givenHandlerIsConnecting(handler, "initial"); writeInboundApiVersionsRequest("post-connecting"); @@ -173,9 +178,13 @@ void testMessageHandledAfterConnectingBeforeConnected() { @Test void testMessageHandledAfterConnectedBeforeOutboundActive() { // Given + VirtualClusterModel virtualClusterModel = mock(VirtualClusterModel.class); + when(virtualClusterModel.getClusterName()).thenReturn("cluster"); VirtualClusterGatewayModel virtualClusterListenerModel = mock(VirtualClusterGatewayModel.class); - when(virtualClusterListenerModel.virtualCluster()).thenReturn(mock(VirtualClusterModel.class)); - KafkaProxyFrontendHandler handler = handler(this::getNetFilter, new SaslDecodePredicate(false), virtualClusterListenerModel); + when(virtualClusterListenerModel.virtualCluster()).thenReturn(virtualClusterModel); + EndpointBinding endpointBinding = mock(EndpointBinding.class); + when(endpointBinding.endpointGateway()).thenReturn(virtualClusterListenerModel); + KafkaProxyFrontendHandler handler = handler(this::getNetFilter, new SaslDecodePredicate(false), endpointBinding); givenHandlerIsConnected(handler); writeInboundApiVersionsRequest("post-connected"); @@ -189,9 +198,13 @@ void testMessageHandledAfterConnectedBeforeOutboundActive() { @Test void testUnexpectedMessageReceivedBeforeConnected() { // Given + VirtualClusterModel virtualClusterModel = mock(VirtualClusterModel.class); + when(virtualClusterModel.getClusterName()).thenReturn("cluster"); VirtualClusterGatewayModel virtualClusterListenerModel = mock(VirtualClusterGatewayModel.class); - when(virtualClusterListenerModel.virtualCluster()).thenReturn(mock(VirtualClusterModel.class)); - KafkaProxyFrontendHandler handler = handler(this::getNetFilter, new SaslDecodePredicate(false), virtualClusterListenerModel); + when(virtualClusterListenerModel.virtualCluster()).thenReturn(virtualClusterModel); + EndpointBinding endpointBinding = mock(EndpointBinding.class); + when(endpointBinding.endpointGateway()).thenReturn(virtualClusterListenerModel); + KafkaProxyFrontendHandler handler = handler(this::getNetFilter, new SaslDecodePredicate(false), endpointBinding); givenHandlerIsConnecting(handler, "initial"); // When @@ -208,11 +221,13 @@ void testHandleFrameOversizedExceptionDownstreamTlsDisabled() throws Exception { VirtualClusterModel virtualClusterModel = mock(VirtualClusterModel.class); VirtualClusterGatewayModel virtualClusterListenerModel = mock(VirtualClusterGatewayModel.class); + EndpointBinding endpointBinding = mock(EndpointBinding.class); + when(endpointBinding.endpointGateway()).thenReturn(virtualClusterListenerModel); when(virtualClusterListenerModel.virtualCluster()).thenReturn(virtualClusterModel); when(virtualClusterListenerModel.getDownstreamSslContext()).thenReturn(Optional.empty()); when(virtualClusterModel.gateways()).thenReturn(Map.of("default", virtualClusterListenerModel)); - KafkaProxyFrontendHandler handler = handler(connectContext::set, new SaslDecodePredicate(false), virtualClusterListenerModel); + KafkaProxyFrontendHandler handler = handler(connectContext::set, new SaslDecodePredicate(false), endpointBinding); ChannelPipeline pipeline = inboundChannel.pipeline(); pipeline.addLast(throwOnReadHandler(new DecoderException(new FrameOversizedException(5, 6)))); pipeline.addLast(handler); @@ -233,11 +248,13 @@ void testHandleFrameOversizedExceptionDownstreamTlsEnabled() throws Exception { // Given VirtualClusterModel virtualClusterModel = mock(VirtualClusterModel.class); VirtualClusterGatewayModel virtualClusterListenerModel = mock(VirtualClusterGatewayModel.class); + EndpointBinding endpointBinding = mock(EndpointBinding.class); + when(endpointBinding.endpointGateway()).thenReturn(virtualClusterListenerModel); when(virtualClusterListenerModel.virtualCluster()).thenReturn(virtualClusterModel); when(virtualClusterListenerModel.getDownstreamSslContext()).thenReturn(Optional.of(SslContextBuilder.forClient().build())); when(virtualClusterModel.gateways()).thenReturn(Map.of("default", virtualClusterListenerModel)); - KafkaProxyFrontendHandler handler = handler(connectContext::set, new SaslDecodePredicate(false), virtualClusterListenerModel); + KafkaProxyFrontendHandler handler = handler(connectContext::set, new SaslDecodePredicate(false), endpointBinding); ChannelPipeline pipeline = inboundChannel.pipeline(); pipeline.addLast(throwOnReadHandler(new DecoderException(new FrameOversizedException(5, 6)))); pipeline.addLast(handler); @@ -256,9 +273,13 @@ void testHandleFrameOversizedExceptionDownstreamTlsEnabled() throws Exception { @Test void testUnexpectedMessageReceivedBeforeOutboundActive() { // Given + VirtualClusterModel virtualClusterModel = mock(VirtualClusterModel.class); + when(virtualClusterModel.getClusterName()).thenReturn("cluster"); VirtualClusterGatewayModel virtualClusterListenerModel = mock(VirtualClusterGatewayModel.class); - when(virtualClusterListenerModel.virtualCluster()).thenReturn(mock(VirtualClusterModel.class)); - KafkaProxyFrontendHandler handler = handler(this::getNetFilter, new SaslDecodePredicate(false), virtualClusterListenerModel); + when(virtualClusterListenerModel.virtualCluster()).thenReturn(virtualClusterModel); + EndpointBinding endpointBinding = mock(EndpointBinding.class); + when(endpointBinding.endpointGateway()).thenReturn(virtualClusterListenerModel); + KafkaProxyFrontendHandler handler = handler(this::getNetFilter, new SaslDecodePredicate(false), endpointBinding); givenHandlerIsConnected(handler); // When @@ -274,8 +295,8 @@ private void writeInboundApiVersionsRequest(String clientSoftwareName) { .setClientSoftwareName(clientSoftwareName).setClientSoftwareVersion("1.0.0")); } - KafkaProxyFrontendHandler handler(NetFilter filter, SaslDecodePredicate dp, VirtualClusterGatewayModel gatewayModel) { - return new KafkaProxyFrontendHandler(filter, dp, gatewayModel, proxyChannelStateMachine) { + KafkaProxyFrontendHandler handler(NetFilter filter, SaslDecodePredicate dp, EndpointBinding endpointBinding) { + return new KafkaProxyFrontendHandler(filter, dp, endpointBinding, proxyChannelStateMachine) { @NonNull @Override @@ -342,8 +363,11 @@ void expectedFlow(boolean clientAuthConfigured, var dp = new SaslDecodePredicate(saslOffloadConfigured); ArgumentCaptor valueCapture = ArgumentCaptor.forClass(NetFilter.NetFilterContext.class); var filter = mock(NetFilter.class); - var virtualCluster = mock(VirtualClusterModel.class); + VirtualClusterModel virtualCluster = mock(VirtualClusterModel.class); + when(virtualCluster.getClusterName()).thenReturn("cluster"); var virtualClusterListenerModel = mock(VirtualClusterGatewayModel.class); + var endpointBinding = mock(EndpointBinding.class); + when(endpointBinding.endpointGateway()).thenReturn(virtualClusterListenerModel); when(virtualClusterListenerModel.virtualCluster()).thenReturn(virtualCluster); when(virtualCluster.getUpstreamSslContext()).thenReturn(Optional.empty()); @@ -382,7 +406,7 @@ void expectedFlow(boolean clientAuthConfigured, return null; }).when(filter).selectServer(valueCapture.capture()); - var handler = handler(filter, dp, virtualClusterListenerModel); + var handler = handler(filter, dp, endpointBinding); initialiseInboundChannel(handler); if (clientAuthConfigured) { @@ -478,10 +502,14 @@ void expectedFlow(boolean clientAuthConfigured, @MethodSource("requests") void shouldTransitionToFailedOnException(Short version, ApiMessage apiMessage) { // Given + VirtualClusterModel virtualClusterModel = mock(VirtualClusterModel.class); + when(virtualClusterModel.getClusterName()).thenReturn("cluster"); VirtualClusterGatewayModel virtualClusterListenerModel = mock(VirtualClusterGatewayModel.class); - when(virtualClusterListenerModel.virtualCluster()).thenReturn(mock(VirtualClusterModel.class)); + when(virtualClusterListenerModel.virtualCluster()).thenReturn(virtualClusterModel); + EndpointBinding endpointBinding = mock(EndpointBinding.class); + when(endpointBinding.endpointGateway()).thenReturn(virtualClusterListenerModel); - KafkaProxyFrontendHandler handler = handler(this::getNetFilter, new SaslDecodePredicate(false), virtualClusterListenerModel); + KafkaProxyFrontendHandler handler = handler(this::getNetFilter, new SaslDecodePredicate(false), endpointBinding); initialiseInboundChannel(handler); final RequestHeaderData header = new RequestHeaderData(); final int correlationId = 1234; @@ -508,7 +536,9 @@ void shouldTransitionToFailedOnExceptionForFrameOversizedException(Short version // Given VirtualClusterGatewayModel virtualClusterListenerModel = mock(VirtualClusterGatewayModel.class); when(virtualClusterListenerModel.virtualCluster()).thenReturn(mock(VirtualClusterModel.class)); - KafkaProxyFrontendHandler handler = handler(this::getNetFilter, new SaslDecodePredicate(false), virtualClusterListenerModel); + EndpointBinding endpointBinding = mock(EndpointBinding.class); + when(endpointBinding.endpointGateway()).thenReturn(virtualClusterListenerModel); + KafkaProxyFrontendHandler handler = handler(this::getNetFilter, new SaslDecodePredicate(false), endpointBinding); initialiseInboundChannel(handler); final RequestHeaderData header = new RequestHeaderData(); final int correlationId = 1234; @@ -574,7 +604,7 @@ private List> outboundApiVersionsFra while ((outboundMessage = outboundChannel.readOutbound()) != null) { assertThat(outboundMessage).isNotNull(); ArrayList objects = new ArrayList<>(); - new KafkaRequestDecoder(RequestDecoderTest.DECODE_EVERYTHING, DEFAULT_SOCKET_FRAME_MAX_SIZE_BYTES, new ApiVersionsServiceImpl(), "randomCluster") + new KafkaRequestDecoder(RequestDecoderTest.DECODE_EVERYTHING, DEFAULT_SOCKET_FRAME_MAX_SIZE_BYTES, new ApiVersionsServiceImpl()) .decode(outboundChannel.pipeline().firstContext(), outboundMessage, objects); assertThat(objects).hasSize(1); if (objects.get(0) instanceof DecodedRequestFrame f) { @@ -629,8 +659,10 @@ void transitionsFromStart() { var virtualClusterListenerModel = mock(VirtualClusterGatewayModel.class); when(virtualClusterListenerModel.virtualCluster()).thenReturn(virtualCluster); when(virtualCluster.getUpstreamSslContext()).thenReturn(Optional.empty()); + EndpointBinding endpointBinding = mock(EndpointBinding.class); + when(endpointBinding.endpointGateway()).thenReturn(virtualClusterListenerModel); - var handler = handler(filter, dp, virtualClusterListenerModel); + var handler = handler(filter, dp, endpointBinding); initialiseInboundChannel(handler); assertThat(proxyChannelStateMachine.state()).isExactlyInstanceOf(ProxyChannelState.ClientActive.class); diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyInitializerTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyInitializerTest.java index c144e8ff77..c7ccdc33af 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyInitializerTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyInitializerTest.java @@ -26,6 +26,7 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.channel.ChannelPipeline; @@ -343,6 +344,8 @@ private void verifyNetworkLoggerAdded(InOrder orderedVerifyer) { private void verifyEncoderAndOrdererAdded(InOrder orderedVerifyer) { orderedVerifyer.verify(channelPipeline).addLast(eq("requestDecoder"), any(ByteToMessageDecoder.class)); orderedVerifyer.verify(channelPipeline).addLast(eq("responseEncoder"), any(MessageToByteEncoder.class)); + orderedVerifyer.verify(channelPipeline).addLast(eq("downstreamMetrics"), any(ChannelHandler.class)); + orderedVerifyer.verify(channelPipeline).addLast(eq("deprecatedDownstreamMetrics"), any(ChannelHandler.class)); orderedVerifyer.verify(channelPipeline).addLast(eq("responseOrderer"), any(ResponseOrderer.class)); } } diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachineEndToEndTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachineEndToEndTest.java index c5ba1aad71..25a4b7ff1f 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachineEndToEndTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachineEndToEndTest.java @@ -61,6 +61,7 @@ import io.kroxylicious.proxy.frame.DecodedRequestFrame; import io.kroxylicious.proxy.frame.DecodedResponseFrame; import io.kroxylicious.proxy.internal.codec.FrameOversizedException; +import io.kroxylicious.proxy.internal.net.EndpointBinding; import io.kroxylicious.proxy.internal.net.EndpointGateway; import io.kroxylicious.proxy.model.VirtualClusterModel; import io.kroxylicious.proxy.service.HostPort; @@ -96,7 +97,7 @@ class ProxyChannelStateMachineEndToEndTest { private EmbeddedChannel outboundChannel; private final AtomicBoolean outboundClosed = new AtomicBoolean(false); - int correlectionId = 0; + int correlationId = 0; private ProxyChannelStateMachine proxyChannelStateMachine; private KafkaProxyFrontendHandler handler; private KafkaProxyBackendHandler backendHandler; @@ -613,7 +614,7 @@ private static DecodedRequestFrame decodedRequestFrame( } private int writeRequest(short apiVersion, ApiMessage body) { - int downstreamCorrelationId = correlectionId++; + int downstreamCorrelationId = correlationId++; DecodedRequestFrame apiMessageDecodedRequestFrame = decodedRequestFrame(apiVersion, body, downstreamCorrelationId); inboundChannel.writeInbound(apiMessageDecodedRequestFrame); @@ -643,8 +644,8 @@ private void writeInboundMetadataRequest() { private KafkaProxyFrontendHandler handler( NetFilter filter, SaslDecodePredicate dp, - EndpointGateway endpointGateway) { - return new KafkaProxyFrontendHandler(filter, dp, endpointGateway, proxyChannelStateMachine) { + EndpointBinding endpointBinding) { + return new KafkaProxyFrontendHandler(filter, dp, endpointBinding, proxyChannelStateMachine) { @NonNull @Override Bootstrap configureBootstrap(KafkaProxyBackendHandler capturedBackendHandler, Channel inboundChannel) { @@ -679,14 +680,17 @@ void buildFrontendHandler(boolean saslOffloadConfigured, boolean tlsConfigured, Answer filterSelectServerBehaviour) { this.inboundChannel = new EmbeddedChannel(); - this.correlectionId = 0; + this.correlationId = 0; var dp = new SaslDecodePredicate(saslOffloadConfigured); NetFilter filter = mock(NetFilter.class); doAnswer(filterSelectServerBehaviour).when(filter).selectServer(any()); VirtualClusterModel virtualClusterModel = mock(VirtualClusterModel.class); + when(virtualClusterModel.getClusterName()).thenReturn("cluster"); + EndpointBinding endpointBinding = mock(EndpointBinding.class); EndpointGateway endpointGateway = mock(EndpointGateway.class); when(endpointGateway.virtualCluster()).thenReturn(virtualClusterModel); + when(endpointBinding.endpointGateway()).thenReturn(endpointGateway); final Optional sslContext; try { sslContext = Optional.ofNullable(tlsConfigured ? SslContextBuilder.forClient().build() : null); @@ -696,7 +700,7 @@ void buildFrontendHandler(boolean saslOffloadConfigured, } when(virtualClusterModel.getUpstreamSslContext()).thenReturn(sslContext); - this.handler = handler(filter, dp, endpointGateway); + this.handler = handler(filter, dp, endpointBinding); this.inboundCtx = mock(ChannelHandlerContext.class); when(inboundCtx.channel()).thenReturn(inboundChannel); when(inboundCtx.pipeline()).thenReturn(inboundChannel.pipeline()); @@ -993,12 +997,12 @@ private DecodedRequestFrame apiKeyToMessage(ApiKeys firstMessage) { return switch (firstMessage) { case API_VERSIONS -> decodedRequestFrame(ApiVersionsRequestData.HIGHEST_SUPPORTED_VERSION, new ApiVersionsRequestData() .setClientSoftwareName(CLIENT_SOFTWARE_NAME) - .setClientSoftwareVersion(CLIENT_SOFTWARE_VERSION), correlectionId++); + .setClientSoftwareVersion(CLIENT_SOFTWARE_VERSION), correlationId++); case SASL_HANDSHAKE -> decodedRequestFrame(SaslHandshakeRequestData.HIGHEST_SUPPORTED_VERSION, new SaslHandshakeRequestData() - .setMechanism(KafkaAuthnHandler.SaslMechanism.PLAIN.mechanismName()), correlectionId++); + .setMechanism(KafkaAuthnHandler.SaslMechanism.PLAIN.mechanismName()), correlationId++); case SASL_AUTHENTICATE -> decodedRequestFrame(SaslAuthenticateRequestData.HIGHEST_SUPPORTED_VERSION, new SaslAuthenticateRequestData() - .setAuthBytes("pa55word".getBytes(StandardCharsets.UTF_8)), correlectionId++); - case METADATA -> decodedRequestFrame(MetadataRequestData.HIGHEST_SUPPORTED_VERSION, new MetadataRequestData(), correlectionId++); + .setAuthBytes("pa55word".getBytes(StandardCharsets.UTF_8)), correlationId++); + case METADATA -> decodedRequestFrame(MetadataRequestData.HIGHEST_SUPPORTED_VERSION, new MetadataRequestData(), correlationId++); default -> throw new IllegalArgumentException(); }; } @@ -1009,7 +1013,7 @@ private void assertNextClientResponseIsErrorFor(DecodedRequestFrame case SASL_HANDSHAKE -> assertNextClientResponseIsSaslHandshakeError(requestFrame.correlationId()); case SASL_AUTHENTICATE -> assertNextClientResponseIsSaslAuthenticateError(requestFrame.correlationId()); case METADATA -> assertNextClientResponseIsMetadataError(requestFrame.correlationId()); - default -> throw new UnsupportedOperationException("unsupported apiKey: " + requestFrame.apiKey()); + default -> throw new UnsupportedOperationException("unsupported apiKey: " + requestFrame.apiKeyId()); } } diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachineTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachineTest.java index d0c42127f7..628fe80bb3 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachineTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachineTest.java @@ -839,4 +839,4 @@ private static DecodedResponseFrame metadataResponse() { new ResponseHeaderData(), new MetadataResponseData()); } -} \ No newline at end of file +} diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/ResponseOrdererTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/ResponseOrdererTest.java index 5aaddafc03..88b5a378d7 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/ResponseOrdererTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/ResponseOrdererTest.java @@ -39,6 +39,21 @@ public int correlationId() { return correlationId; } + @Override + public short apiKeyId() { + return 0; + } + + @Override + public short apiVersion() { + return 0; + } + + @Override + public boolean isDecoded() { + return false; + } + } record TestRequestFrame(int correlationId, boolean hasResponse) implements RequestFrame { @@ -63,6 +78,21 @@ public boolean hasResponse() { return hasResponse; } + @Override + public short apiKeyId() { + return 0; + } + + @Override + public short apiVersion() { + return 0; + } + + @Override + public boolean isDecoded() { + return false; + } + @Override public boolean decodeResponse() { return false; diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/KafkaRequestDecoderTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/KafkaRequestDecoderTest.java index 16b09fa464..6f14f18bad 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/KafkaRequestDecoderTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/KafkaRequestDecoderTest.java @@ -32,10 +32,8 @@ class KafkaRequestDecoderTest { - private static final String CLUSTER_NAME = "randomCluster"; - @Test - void decodeUnknownApiVersionsRespectsOverridenLatestVersion() { + void decodeUnknownApiVersionsRespectsOverriddenLatestVersion() { short latestSupportedApiVersionsOverride = (short) 2; ApiVersionsServiceImpl apiVersionsService = new ApiVersionsServiceImpl(Map.of(ApiKeys.API_VERSIONS, latestSupportedApiVersionsOverride)); EmbeddedChannel embeddedChannel = newEmbeddedChannel(apiVersionsService); @@ -134,7 +132,7 @@ void throwsOnUnsupportedVersionOfNonApiVersionsRequests() { @NonNull private static EmbeddedChannel newEmbeddedChannel(ApiVersionsServiceImpl apiVersionsService) { return new EmbeddedChannel( - new KafkaRequestDecoder(RequestDecoderTest.DECODE_EVERYTHING, 1024, apiVersionsService, CLUSTER_NAME)); + new KafkaRequestDecoder(RequestDecoderTest.DECODE_EVERYTHING, 1024, apiVersionsService)); } private static @NonNull RequestHeaderData latestVersionHeaderWithAllFields(ApiKeys requestApiKey, short requestApiVersion) { diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/RequestDecoderTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/RequestDecoderTest.java index cbffa68f8d..c1449b0b66 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/RequestDecoderTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/RequestDecoderTest.java @@ -95,7 +95,7 @@ void testApiVersionsExactlyOneFrame_decoded(short apiVersion) { new KafkaRequestDecoder( DecodePredicate .forFilters(FilterAndInvoker.build(filter.getClass().getSimpleName(), filter)), - DEFAULT_SOCKET_FRAME_MAX_SIZE_BYTES, new ApiVersionsServiceImpl(), CLUSTER_NAME), + DEFAULT_SOCKET_FRAME_MAX_SIZE_BYTES, new ApiVersionsServiceImpl()), DecodedRequestFrame.class, (RequestHeaderData header) -> header, true), "Unexpected correlation id"); @@ -177,8 +177,7 @@ public CompletionStage onApiVersionsRequest(short apiVersio AbstractCodecTest::exampleRequestHeader, AbstractCodecTest::exampleApiVersionsRequest, new KafkaRequestDecoder(DecodePredicate.forFilters( - FilterAndInvoker.build(filter.getClass().getSimpleName(), filter)), DEFAULT_SOCKET_FRAME_MAX_SIZE_BYTES, new ApiVersionsServiceImpl(), - CLUSTER_NAME), + FilterAndInvoker.build(filter.getClass().getSimpleName(), filter)), DEFAULT_SOCKET_FRAME_MAX_SIZE_BYTES, new ApiVersionsServiceImpl()), OpaqueRequestFrame.class, true), "Unexpected correlation id"); } @@ -200,7 +199,7 @@ void testApiVersionsFrameLessOneByte(short apiVersion) { new KafkaRequestDecoder( DecodePredicate.forFilters( FilterAndInvoker.build(filter.getClass().getSimpleName(), filter)), - DEFAULT_SOCKET_FRAME_MAX_SIZE_BYTES, new ApiVersionsServiceImpl(), CLUSTER_NAME) + DEFAULT_SOCKET_FRAME_MAX_SIZE_BYTES, new ApiVersionsServiceImpl()) .decode(null, byteBuf, messages); assertEquals(List.of(), messageClasses(messages)); @@ -232,7 +231,7 @@ private void doTestApiVersionsFrameFirstNBytes(short apiVersion, int n, int expe private static KafkaRequestDecoder getKafkaRequestDecoder(DecodePredicate predicate, int socketFrameMaxSizeBytes) { return new KafkaRequestDecoder( predicate, - socketFrameMaxSizeBytes, new ApiVersionsServiceImpl(), CLUSTER_NAME); + socketFrameMaxSizeBytes, new ApiVersionsServiceImpl()); } @ParameterizedTest @@ -274,7 +273,7 @@ void testApiVersionsExactlyTwoFrames(short apiVersion) { new KafkaRequestDecoder( DecodePredicate.forFilters( FilterAndInvoker.build(filter.getClass().getSimpleName(), filter)), - DEFAULT_SOCKET_FRAME_MAX_SIZE_BYTES, new ApiVersionsServiceImpl(), CLUSTER_NAME) + DEFAULT_SOCKET_FRAME_MAX_SIZE_BYTES, new ApiVersionsServiceImpl()) .decode(null, byteBuf, messages); assertEquals(List.of(DecodedRequestFrame.class, DecodedRequestFrame.class), messageClasses(messages)); diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/RequestEncoderTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/RequestEncoderTest.java index 9859837e2f..864ba381b2 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/RequestEncoderTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/RequestEncoderTest.java @@ -69,7 +69,7 @@ void testApiVersions(short apiVersion) throws Exception { @MethodSource("testResponseFrames") void testRequestsWithNoResponseArentStoredInCorrelationManager() throws Exception { - givenRequestFrame frame = createRequestFrame(false); + GivenRequestFrame frame = createRequestFrame(false); var correlationManager = new CorrelationManager(78); whenRequestEncoded(frame, correlationManager); @@ -82,7 +82,7 @@ void testRequestsWithNoResponseArentStoredInCorrelationManager() throws Exceptio @MethodSource("testResponseFrames") void testRequestsWithResponseAreStoredInCorrelationManager() throws Exception { - givenRequestFrame frame = createRequestFrame(true); + GivenRequestFrame frame = createRequestFrame(true); var correlationManager = new CorrelationManager(78); whenRequestEncoded(frame, correlationManager); @@ -93,19 +93,20 @@ void testRequestsWithResponseAreStoredInCorrelationManager() throws Exception { "Expect request with response to have a correlation"); } - private static void whenRequestEncoded(givenRequestFrame result, CorrelationManager correlationManager) throws Exception { + private static void whenRequestEncoded(GivenRequestFrame result, CorrelationManager correlationManager) throws Exception { ByteBuf out = Unpooled.buffer(result.byteBuffer().capacity() + 4); new KafkaRequestEncoder(correlationManager).encode(null, result.frame(), out); } @NonNull - private static givenRequestFrame createRequestFrame(boolean hasResponse) { - short produceVersion = ApiKeys.PRODUCE.latestVersion(); + private static GivenRequestFrame createRequestFrame(boolean hasResponse) { + var produceKey = ApiKeys.PRODUCE; + short produceVersion = produceKey.latestVersion(); var header = new RequestHeaderData() - .setRequestApiKey(ApiKeys.PRODUCE.id) + .setRequestApiKey(produceKey.id) .setRequestApiVersion(produceVersion) .setCorrelationId(45); - short headerVersion = ApiKeys.PRODUCE.requestHeaderVersion(produceVersion); + short headerVersion = produceKey.requestHeaderVersion(produceVersion); if (headerVersion >= 1) { header.setClientId("323423"); } @@ -120,10 +121,9 @@ private static givenRequestFrame createRequestFrame(boolean hasResponse) { ByteBuf buf = Unpooled.copiedBuffer(byteBuffer); - var frame = new OpaqueRequestFrame(buf, 12, true, frameSize, hasResponse); - givenRequestFrame result = new givenRequestFrame(byteBuffer, frame); - return result; + var frame = new OpaqueRequestFrame(buf, produceKey.id, produceVersion, 12, true, frameSize, hasResponse); + return new GivenRequestFrame(byteBuffer, frame); } - private record givenRequestFrame(ByteBuffer byteBuffer, OpaqueRequestFrame frame) {} + private record GivenRequestFrame(ByteBuffer byteBuffer, OpaqueRequestFrame frame) {} } diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/ResponseEncoderTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/ResponseEncoderTest.java index 84885cb3d2..9d7fba694a 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/ResponseEncoderTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/ResponseEncoderTest.java @@ -23,6 +23,7 @@ void testApiVersions(short apiVersion) throws Exception { ApiVersionsResponseData exampleBody = exampleApiVersionsResponse(); short headerVersion = ApiKeys.API_VERSIONS.responseHeaderVersion(apiVersion); ByteBuffer expected = serializeUsingKafkaApis(headerVersion, exampleHeader, apiVersion, exampleBody); - testEncode(expected, new DecodedResponseFrame<>(apiVersion, exampleHeader.correlationId(), exampleHeader, exampleBody), new KafkaResponseEncoder()); + testEncode(expected, new DecodedResponseFrame<>(apiVersion, exampleHeader.correlationId(), exampleHeader, exampleBody), + new KafkaResponseEncoder()); } } From 8720ace6f81e4d4018fade504a022c60cfa854e6 Mon Sep 17 00:00:00 2001 From: Keith Wall Date: Mon, 2 Jun 2025 17:17:22 +0100 Subject: [PATCH 14/83] Use api key/version from Frame rather than re-reading the written response Signed-off-by: Keith Wall Signed-off-by: Ujjwal --- .../proxy/internal/codec/KafkaRequestEncoder.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/codec/KafkaRequestEncoder.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/codec/KafkaRequestEncoder.java index 3a8d45914a..62c5fcb2b2 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/codec/KafkaRequestEncoder.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/codec/KafkaRequestEncoder.java @@ -36,13 +36,9 @@ protected Logger log() { @Override protected void encode(ChannelHandlerContext ctx, RequestFrame frame, ByteBuf out) throws Exception { super.encode(ctx, frame, out); - // TODO re-reading from the encoded buffer like this is ugly - // probably better to just include apiKey and apiVersion in the frame - var ri = out.readerIndex(); var wi = out.writerIndex(); - out.readerIndex(LENGTH); - short apiKey = out.readShort(); - short apiVersion = out.readShort(); + short apiKey = frame.apiKeyId(); + short apiVersion = frame.apiVersion(); boolean hasResponse = frame.hasResponse(); boolean decodeResponse = frame.decodeResponse(); int downstreamCorrelationId = frame.correlationId(); @@ -58,7 +54,6 @@ protected void encode(ChannelHandlerContext ctx, RequestFrame frame, ByteBuf out LOGGER.debug("{}: {} downstream correlation id {} assigned upstream correlation id: {}", ctx, ApiKeys.forId(apiKey), downstreamCorrelationId, upstreamCorrelationId); } - out.readerIndex(ri); out.writerIndex(wi); } From 3fa403180ca2b166f4a322875fb383f4d76aa99a Mon Sep 17 00:00:00 2001 From: Keith Wall Date: Mon, 2 Jun 2025 17:27:16 +0100 Subject: [PATCH 15/83] sonar Signed-off-by: Keith Wall Signed-off-by: Ujjwal --- .../proxy/internal/KafkaProxyFrontendHandler.java | 3 +-- .../proxy/internal/codec/KafkaResponseDecoder.java | 4 +--- .../metrics/DeprecatedDownstreamMessageMetrics.java | 7 +++++++ .../internal/metrics/DeprecatedUpstreamMessageMetrics.java | 5 +++++ .../proxy/internal/codec/ResponseDecoderTest.java | 2 +- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java index f879372277..48299b7a21 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java @@ -520,8 +520,7 @@ void inConnecting( pipeline.addFirst("upstreamMetrics", new MessageMetrics(serverToProxyCounterProvider, proxyToServerCounterProvider)); pipeline.addFirst("deprecatedUpstreamMetrics", getDeprecatedUpstreamMessageMetrics(this.virtualClusterModel.getClusterName())); - pipeline.addFirst("responseDecoder", new KafkaResponseDecoder(correlationManager, virtualClusterModel.socketFrameMaxSizeBytes(), - virtualClusterModel.getClusterName())); + pipeline.addFirst("responseDecoder", new KafkaResponseDecoder(correlationManager, virtualClusterModel.socketFrameMaxSizeBytes())); pipeline.addFirst("requestEncoder", new KafkaRequestEncoder(correlationManager)); if (logNetwork) { pipeline.addFirst("networkLogger", new LoggingHandler("io.kroxylicious.proxy.internal.UpstreamNetworkLogger")); diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/codec/KafkaResponseDecoder.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/codec/KafkaResponseDecoder.java index a0e0d87d3c..07a1fbbffc 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/codec/KafkaResponseDecoder.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/codec/KafkaResponseDecoder.java @@ -27,12 +27,10 @@ public class KafkaResponseDecoder extends KafkaMessageDecoder { private static final Logger LOGGER = LoggerFactory.getLogger(KafkaResponseDecoder.class); private final CorrelationManager correlationManager; - private final String clusterName; - public KafkaResponseDecoder(CorrelationManager correlationManager, int socketRequestMaxSizeBytes, String clusterName) { + public KafkaResponseDecoder(CorrelationManager correlationManager, int socketRequestMaxSizeBytes) { super(socketRequestMaxSizeBytes); this.correlationManager = correlationManager; - this.clusterName = clusterName; } @Override diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedDownstreamMessageMetrics.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedDownstreamMessageMetrics.java index 650a47a154..e1a837983a 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedDownstreamMessageMetrics.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedDownstreamMessageMetrics.java @@ -16,6 +16,13 @@ import io.kroxylicious.proxy.frame.DecodedRequestFrame; import io.kroxylicious.proxy.internal.util.Metrics; +/** + * Emits the deprecated downstream metrics kroxylicious_inbound_downstream_messages, + * kroxylicious_inbound_downstream_messages, kroxylicious_inbound_downstream_decoded_messages, and + * kroxylicious_payload_size_bytes. + * + * @deprecated use metrics emitted by {@link MessageMetrics} instead. + */ @Deprecated(since = "0.13.0", forRemoval = true) public class DeprecatedDownstreamMessageMetrics extends ChannelInboundHandlerAdapter { private final String clusterName; diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedUpstreamMessageMetrics.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedUpstreamMessageMetrics.java index b3d47a3f3c..17992de9b2 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedUpstreamMessageMetrics.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedUpstreamMessageMetrics.java @@ -12,6 +12,11 @@ import io.kroxylicious.proxy.frame.DecodedResponseFrame; import io.kroxylicious.proxy.internal.util.Metrics; +/** + * Emits the deprecated upstream metric kroxylicious_payload_size_bytes + * + * @deprecated use metrics emitted by {@link MessageMetrics} instead. + */ @Deprecated(since = "0.13.0", forRemoval = true) public class DeprecatedUpstreamMessageMetrics extends ChannelInboundHandlerAdapter { private final String clusterName; diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/ResponseDecoderTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/ResponseDecoderTest.java index a11763a686..e5c82c56e4 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/ResponseDecoderTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/ResponseDecoderTest.java @@ -74,7 +74,7 @@ void testApiVersionsExactlyOneFrame_opaque(short apiVersion) throws Exception { @NonNull private static KafkaResponseDecoder createResponseDecoder(CorrelationManager mgr, int socketFrameMaxSizeBytes) { - return new KafkaResponseDecoder(mgr, socketFrameMaxSizeBytes, "vc"); + return new KafkaResponseDecoder(mgr, socketFrameMaxSizeBytes); } @Test From c0465ccfddd83026c2080b815a7c81c0bfd85a35 Mon Sep 17 00:00:00 2001 From: Keith Wall Date: Mon, 2 Jun 2025 21:04:11 +0100 Subject: [PATCH 16/83] bug fix - deprecated metrics wrong way about Signed-off-by: Keith Wall Signed-off-by: Ujjwal --- .../internal/metrics/DeprecatedDownstreamMessageMetrics.java | 2 +- .../internal/metrics/DeprecatedUpstreamMessageMetrics.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedDownstreamMessageMetrics.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedDownstreamMessageMetrics.java index e1a837983a..52d839a757 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedDownstreamMessageMetrics.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedDownstreamMessageMetrics.java @@ -48,7 +48,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception // so this avoids caching the size prematurely int size = new DecodedRequestFrame<>(decodedRequestFrame.apiVersion(), decodedRequestFrame.correlationId(), decodedRequestFrame.decodeResponse(), decodedRequestFrame.header(), decodedRequestFrame.body()).estimateEncodedSize(); - Metrics.payloadSizeBytesDownstreamSummary(decodedRequestFrame.apiKey(), decodedRequestFrame.apiVersion(), clusterName).record(size); + Metrics.payloadSizeBytesUpstreamSummary(decodedRequestFrame.apiKey(), decodedRequestFrame.apiVersion(), clusterName).record(size); } } finally { diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedUpstreamMessageMetrics.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedUpstreamMessageMetrics.java index 17992de9b2..d9020cb13f 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedUpstreamMessageMetrics.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedUpstreamMessageMetrics.java @@ -35,7 +35,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception // so this avoids caching the size prematurely int size = new DecodedResponseFrame<>(decodedResponseFrame.apiVersion(), decodedResponseFrame.correlationId(), decodedResponseFrame.header(), decodedResponseFrame.body()).estimateEncodedSize(); - Metrics.payloadSizeBytesUpstreamSummary(decodedResponseFrame.apiKey(), decodedResponseFrame.apiVersion(), clusterName) + Metrics.payloadSizeBytesDownstreamSummary(decodedResponseFrame.apiKey(), decodedResponseFrame.apiVersion(), clusterName) .record(size); } } From 30da5d385e3cba702d002635b7f0d920e59519ce Mon Sep 17 00:00:00 2001 From: Keith Wall Date: Tue, 3 Jun 2025 09:05:21 +0100 Subject: [PATCH 17/83] sonar Signed-off-by: Keith Wall Signed-off-by: Ujjwal --- .../kroxylicious/test/tester/SimpleMetricAssert.java | 10 ++++------ .../proxy/internal/codec/RequestDecoderTest.java | 1 - 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/tester/SimpleMetricAssert.java b/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/tester/SimpleMetricAssert.java index b000dcebe5..9330216b56 100644 --- a/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/tester/SimpleMetricAssert.java +++ b/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/tester/SimpleMetricAssert.java @@ -8,7 +8,6 @@ import java.util.List; import java.util.Objects; -import java.util.Optional; import org.assertj.core.api.AbstractAssert; import org.assertj.core.api.AbstractDoubleAssert; @@ -29,21 +28,19 @@ public static SimpleMetricListAssert assertThat(List actual) { return new SimpleMetricListAssert(actual); } - public static SimpleMetricAssert assertThat(Optional actual) { - return new SimpleMetricAssert(actual.orElse(null)); - } - private SimpleMetricAssert(SimpleMetric simpleMetric) { super(simpleMetric, SimpleMetricAssert.class); describedAs(simpleMetric == null ? "null metric" : "metric"); } + @SuppressWarnings("java:S1452") public AbstractDoubleAssert value() { isNotNull(); return Assertions.assertThat(actual.value()) .describedAs("metric value"); } + @SuppressWarnings("java:S1452") public AbstractStringAssert name() { isNotNull(); return Assertions.assertThat(actual.name()) @@ -56,8 +53,9 @@ public MapAssert labels() { .describedAs("metric labels"); } + @SuppressWarnings("java:S110") // Ignoring "This class has 6 parents which is greater than 5 authorized" as this is the Assert-J public API. public static class SimpleMetricListAssert extends FactoryBasedNavigableListAssert, SimpleMetric, SimpleMetricAssert> { - SimpleMetricListAssert(List simpleMetrics) { + private SimpleMetricListAssert(List simpleMetrics) { super(simpleMetrics, SimpleMetricListAssert.class, SimpleMetricAssert::new); describedAs(simpleMetrics == null ? "empty metric list" : "metrics"); diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/RequestDecoderTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/RequestDecoderTest.java index c1449b0b66..03c5e52c91 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/RequestDecoderTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/codec/RequestDecoderTest.java @@ -66,7 +66,6 @@ public boolean shouldDecodeResponse(ApiKeys apiKey, short apiVersion) { } }; public static final KafkaRequestDecoder MAX_FRAME_SIZE_10_BYTES_DECODER = getKafkaRequestDecoder(DECODE_EVERYTHING, 10); - private static final String CLUSTER_NAME = "randomVCluster"; static List produceRequestApiVersions() { List produceVersions = requestApiVersions(ApiMessageType.PRODUCE).toList(); From f09da5f1591996e3f6c4e7ea1a5088d3d108879c Mon Sep 17 00:00:00 2001 From: Keith Wall Date: Tue, 3 Jun 2025 09:12:18 +0100 Subject: [PATCH 18/83] more sonar Signed-off-by: Keith Wall Signed-off-by: Ujjwal --- .../metrics/DeprecatedDownstreamMessageMetrics.java | 2 +- .../internal/metrics/DeprecatedUpstreamMessageMetrics.java | 4 ++-- .../java/io/kroxylicious/proxy/internal/util/Metrics.java | 6 ++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedDownstreamMessageMetrics.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedDownstreamMessageMetrics.java index 52d839a757..37a8555628 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedDownstreamMessageMetrics.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedDownstreamMessageMetrics.java @@ -45,7 +45,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception if (msg instanceof DecodedRequestFrame decodedRequestFrame) { decodedMessagesCounter.increment(); // TODO the request might be updated as it travels through the proxy - // so this avoids caching the size prematurely + // so this avoids causing the size to be cached prematurely int size = new DecodedRequestFrame<>(decodedRequestFrame.apiVersion(), decodedRequestFrame.correlationId(), decodedRequestFrame.decodeResponse(), decodedRequestFrame.header(), decodedRequestFrame.body()).estimateEncodedSize(); Metrics.payloadSizeBytesUpstreamSummary(decodedRequestFrame.apiKey(), decodedRequestFrame.apiVersion(), clusterName).record(size); diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedUpstreamMessageMetrics.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedUpstreamMessageMetrics.java index d9020cb13f..2a5f9e9c6c 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedUpstreamMessageMetrics.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedUpstreamMessageMetrics.java @@ -31,8 +31,8 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception try { if (msg instanceof DecodedResponseFrame decodedResponseFrame) { - // TODO the request might be updated as it travels through the proxy - // so this avoids caching the size prematurely + // TODO the response might be updated as it travels through the proxy + // so this avoids causing the size to be cached prematurely int size = new DecodedResponseFrame<>(decodedResponseFrame.apiVersion(), decodedResponseFrame.correlationId(), decodedResponseFrame.header(), decodedResponseFrame.body()).estimateEncodedSize(); Metrics.payloadSizeBytesDownstreamSummary(decodedResponseFrame.apiKey(), decodedResponseFrame.apiVersion(), clusterName) diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java index 23dc7ef286..70d736b7a3 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java @@ -95,9 +95,15 @@ public class Metrics { .withRegistry( globalRegistry); + /** + * @deprecated use kroxylicious_client_to_proxy_request_count instead. + */ @Deprecated(since = "0.13.0", forRemoval = true) public static final String KROXYLICIOUS_INBOUND_DOWNSTREAM_MESSAGES = "kroxylicious_inbound_downstream_messages"; + /** + * @deprecated use kroxylicious_client_to_proxy_request_count instead. + */ @Deprecated(since = "0.13.0", forRemoval = true) public static final String KROXYLICIOUS_INBOUND_DOWNSTREAM_DECODED_MESSAGES = "kroxylicious_inbound_downstream_decoded_messages"; From 2e14c802c0797f4efd2d616e582e375a9716d575 Mon Sep 17 00:00:00 2001 From: Keith Wall Date: Tue, 3 Jun 2025 09:56:03 +0100 Subject: [PATCH 19/83] add unit test for new metrics handler Signed-off-by: Keith Wall Signed-off-by: Ujjwal --- .../DeprecatedDownstreamMessageMetrics.java | 2 +- .../DeprecatedUpstreamMessageMetrics.java | 2 +- .../internal/metrics/MessageMetrics.java | 10 +- .../internal/metrics/MessageMetricsTest.java | 104 ++++++++++++++++++ 4 files changed, 112 insertions(+), 6 deletions(-) create mode 100644 kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/metrics/MessageMetricsTest.java diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedDownstreamMessageMetrics.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedDownstreamMessageMetrics.java index 37a8555628..a3ddfa3ab4 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedDownstreamMessageMetrics.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedDownstreamMessageMetrics.java @@ -44,7 +44,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception inboundMessageCounter.increment(); if (msg instanceof DecodedRequestFrame decodedRequestFrame) { decodedMessagesCounter.increment(); - // TODO the request might be updated as it travels through the proxy + // The request might be updated by the filters as it travels through the proxy // so this avoids causing the size to be cached prematurely int size = new DecodedRequestFrame<>(decodedRequestFrame.apiVersion(), decodedRequestFrame.correlationId(), decodedRequestFrame.decodeResponse(), decodedRequestFrame.header(), decodedRequestFrame.body()).estimateEncodedSize(); diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedUpstreamMessageMetrics.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedUpstreamMessageMetrics.java index 2a5f9e9c6c..772051a809 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedUpstreamMessageMetrics.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/DeprecatedUpstreamMessageMetrics.java @@ -31,7 +31,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception try { if (msg instanceof DecodedResponseFrame decodedResponseFrame) { - // TODO the response might be updated as it travels through the proxy + // The response might be updated by the filters as it travels through the proxy // so this avoids causing the size to be cached prematurely int size = new DecodedResponseFrame<>(decodedResponseFrame.apiVersion(), decodedResponseFrame.correlationId(), decodedResponseFrame.header(), decodedResponseFrame.body()).estimateEncodedSize(); diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/MessageMetrics.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/MessageMetrics.java index 98b06f96f3..1c524f69ef 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/MessageMetrics.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/MessageMetrics.java @@ -19,13 +19,16 @@ import io.kroxylicious.proxy.frame.Frame; import io.kroxylicious.proxy.internal.util.Metrics; +/** + * Responsible for emitting message metrics. + */ public class MessageMetrics extends ChannelDuplexHandler { private final MeterProvider readCounterProvider; private final MeterProvider writeCounterProvider; - public MessageMetrics(MeterProvider requestCounterProvider, MeterProvider responseCounterProvider) { - this.readCounterProvider = requestCounterProvider; - this.writeCounterProvider = responseCounterProvider; + public MessageMetrics(MeterProvider readCounterProvider, MeterProvider writeCounterProvider) { + this.readCounterProvider = readCounterProvider; + this.writeCounterProvider = writeCounterProvider; } @Override @@ -42,7 +45,6 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception @Override public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { - try { Optional.ofNullable(writeCounterProvider) .ifPresent(counter -> count(msg, counter)); diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/metrics/MessageMetricsTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/metrics/MessageMetricsTest.java new file mode 100644 index 0000000000..e9a15c4c10 --- /dev/null +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/metrics/MessageMetricsTest.java @@ -0,0 +1,104 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.proxy.internal.metrics; + +import org.apache.kafka.common.protocol.ApiKeys; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Meter; +import io.netty.channel.embedded.EmbeddedChannel; + +import io.kroxylicious.proxy.frame.Frame; +import io.kroxylicious.proxy.internal.util.Metrics; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class MessageMetricsTest { + + private EmbeddedChannel embeddedChannel; + + @Mock + Frame message; + @Mock + private Meter.MeterProvider provider; + + @Mock + Counter counter; + + @Test + void shouldTickMetricOnFrameRead() { + // Given + when(provider.withTags(any(String[].class))).thenReturn(counter); + when(message.apiKeyId()).thenReturn(ApiKeys.PRODUCE.id); + + var messageMetrics = new MessageMetrics(provider, null); + embeddedChannel = new EmbeddedChannel(messageMetrics); + + // When + embeddedChannel.writeInbound(message); + + // Then + verify(provider).withTags(Metrics.DECODED_LABEL, "false", + Metrics.API_KEY_LABEL, ApiKeys.PRODUCE.name(), + Metrics.API_VERSION_LABEL, "0"); + verify(counter).increment(); + } + + @Test + void shouldNotTickOnNonFrameRead() { + // Given + var messageMetrics = new MessageMetrics(provider, null); + embeddedChannel = new EmbeddedChannel(messageMetrics); + + // When + embeddedChannel.writeInbound(new Object()); + + // Then + verifyNoInteractions(provider, counter); + } + + @Test + void shouldTickMetricOnFrameWrite() { + // Given + when(provider.withTags(any(String[].class))).thenReturn(counter); + when(message.apiKeyId()).thenReturn(ApiKeys.PRODUCE.id); + + var messageMetrics = new MessageMetrics(null, provider); + embeddedChannel = new EmbeddedChannel(messageMetrics); + + // When + embeddedChannel.writeOutbound(message); + + // Then + verify(provider).withTags(Metrics.DECODED_LABEL, "false", + Metrics.API_KEY_LABEL, ApiKeys.PRODUCE.name(), + Metrics.API_VERSION_LABEL, "0"); + verify(counter).increment(); + } + + @Test + void shouldNotTickOnNonFrameWrite() { + // Given + var messageMetrics = new MessageMetrics(null, provider); + embeddedChannel = new EmbeddedChannel(messageMetrics); + + // When + embeddedChannel.writeOutbound(new Object()); + + // Then + verifyNoInteractions(provider, counter); + } + +} \ No newline at end of file From cc0616a1c6a8d2a14149d230ae1b1b2beda9144b Mon Sep 17 00:00:00 2001 From: Keith Wall Date: Tue, 3 Jun 2025 10:10:39 +0100 Subject: [PATCH 20/83] Addressing code review comments. Co-authored-by: Sam Barker Signed-off-by: Keith Wall Signed-off-by: Ujjwal --- .../proxy/frame/DecodedRequestFrame.java | 3 +-- .../proxy/internal/metrics/MessageMetrics.java | 17 +++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/DecodedRequestFrame.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/DecodedRequestFrame.java index bbca35a235..9e4071f223 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/DecodedRequestFrame.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/frame/DecodedRequestFrame.java @@ -7,7 +7,6 @@ import org.apache.kafka.common.message.ProduceRequestData; import org.apache.kafka.common.message.RequestHeaderData; -import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ApiMessage; import static org.apache.kafka.common.protocol.ApiKeys.PRODUCE; @@ -32,7 +31,7 @@ public DecodedRequestFrame(short apiVersion, @Override public short headerVersion() { - return ApiKeys.forId(apiKeyId()).messageType.requestHeaderVersion(apiVersion); + return apiKey().messageType.requestHeaderVersion(apiVersion); } @Override diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/MessageMetrics.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/MessageMetrics.java index 1c524f69ef..d0810e32f0 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/MessageMetrics.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/metrics/MessageMetrics.java @@ -23,19 +23,21 @@ * Responsible for emitting message metrics. */ public class MessageMetrics extends ChannelDuplexHandler { - private final MeterProvider readCounterProvider; - private final MeterProvider writeCounterProvider; + + @SuppressWarnings("OptionalUsedAsFieldOrParameterType") + private final Optional> readCounterProvider; + @SuppressWarnings("OptionalUsedAsFieldOrParameterType") + private final Optional> writeCounterProvider; public MessageMetrics(MeterProvider readCounterProvider, MeterProvider writeCounterProvider) { - this.readCounterProvider = readCounterProvider; - this.writeCounterProvider = writeCounterProvider; + this.readCounterProvider = Optional.ofNullable(readCounterProvider); + this.writeCounterProvider = Optional.ofNullable(writeCounterProvider); } @Override public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { try { - Optional.ofNullable(readCounterProvider) - .ifPresent(counter -> count(msg, counter)); + readCounterProvider.ifPresent(provider -> count(msg, provider)); } finally { super.channelRead(ctx, msg); @@ -46,8 +48,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception @Override public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception { try { - Optional.ofNullable(writeCounterProvider) - .ifPresent(counter -> count(msg, counter)); + writeCounterProvider.ifPresent(provider -> count(msg, provider)); } finally { super.write(ctx, msg, promise); From de686a940c664846c61bec810749f41c4e0dc573 Mon Sep 17 00:00:00 2001 From: Keith Wall Date: Tue, 3 Jun 2025 10:51:30 +0100 Subject: [PATCH 21/83] Make io.kroxylicious.proxy.internal.net.EndpointBinding#nodeId part of API Signed-off-by: Keith Wall Signed-off-by: Ujjwal --- .../proxy/internal/KafkaProxyFrontendHandler.java | 7 +++---- .../proxy/internal/KafkaProxyInitializer.java | 6 ++---- .../internal/net/BootstrapEndpointBinding.java | 9 ++++++++- .../proxy/internal/net/BrokerEndpointBinding.java | 13 +++++++++++-- .../proxy/internal/net/EndpointBinding.java | 10 ++++++++++ 5 files changed, 34 insertions(+), 11 deletions(-) diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java index 48299b7a21..6ea2ac75f2 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java @@ -52,7 +52,6 @@ import io.kroxylicious.proxy.internal.codec.KafkaResponseDecoder; import io.kroxylicious.proxy.internal.metrics.DeprecatedUpstreamMessageMetrics; import io.kroxylicious.proxy.internal.metrics.MessageMetrics; -import io.kroxylicious.proxy.internal.net.BrokerEndpointBinding; import io.kroxylicious.proxy.internal.net.EndpointBinding; import io.kroxylicious.proxy.internal.util.Metrics; import io.kroxylicious.proxy.model.VirtualClusterModel; @@ -513,10 +512,10 @@ void inConnecting( pipeline.addFirst("frameLogger", new LoggingHandler("io.kroxylicious.proxy.internal.UpstreamFrameLogger")); } addFiltersToPipeline(filters, pipeline, inboundChannel); - Integer nodeId = endpointBinding instanceof BrokerEndpointBinding brokerEndpointBinding ? brokerEndpointBinding.nodeId() : null; var proxyToServerCounterProvider = Metrics.KROXYLICIOUS_PROXY_TO_SERVER_REQUEST_TOTAL_METER_PROVIDER - .create(this.virtualClusterModel.getClusterName(), nodeId); - var serverToProxyCounterProvider = Metrics.KROXYLICIOUS_SERVER_TO_PROXY_RESPONSE_TOTAL_METER_PROVIDER.create(virtualClusterModel.getClusterName(), nodeId); + .create(this.virtualClusterModel.getClusterName(), endpointBinding.nodeId()); + var serverToProxyCounterProvider = Metrics.KROXYLICIOUS_SERVER_TO_PROXY_RESPONSE_TOTAL_METER_PROVIDER.create(virtualClusterModel.getClusterName(), + endpointBinding.nodeId()); pipeline.addFirst("upstreamMetrics", new MessageMetrics(serverToProxyCounterProvider, proxyToServerCounterProvider)); pipeline.addFirst("deprecatedUpstreamMetrics", getDeprecatedUpstreamMessageMetrics(this.virtualClusterModel.getClusterName())); diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java index 33ed7c8836..56e2988a50 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java @@ -40,7 +40,6 @@ import io.kroxylicious.proxy.internal.filter.NettyFilterContext; import io.kroxylicious.proxy.internal.metrics.DeprecatedDownstreamMessageMetrics; import io.kroxylicious.proxy.internal.metrics.MessageMetrics; -import io.kroxylicious.proxy.internal.net.BrokerEndpointBinding; import io.kroxylicious.proxy.internal.net.Endpoint; import io.kroxylicious.proxy.internal.net.EndpointBinding; import io.kroxylicious.proxy.internal.net.EndpointBindingResolver; @@ -197,14 +196,13 @@ void addHandlers(SocketChannel ch, EndpointBinding binding) { var dp = new SaslDecodePredicate(!authnHandlers.isEmpty()); // The decoder, this only cares about the filters // because it needs to know whether to decode requests - Integer nodeId = binding instanceof BrokerEndpointBinding brokerEndpointBinding ? brokerEndpointBinding.nodeId() : null; KafkaRequestDecoder decoder = new KafkaRequestDecoder(dp, virtualCluster.socketFrameMaxSizeBytes(), apiVersionsService); pipeline.addLast("requestDecoder", decoder); pipeline.addLast("responseEncoder", new KafkaResponseEncoder()); - var clientToProxyCounterProvider = Metrics.KROXYLICIOUS_CLIENT_TO_PROXY_REQUEST_TOTAL_METER_PROVIDER.create(virtualCluster.getClusterName(), nodeId); - var proxyToClientCounterProvider = Metrics.KROXYLICIOUS_PROXY_TO_CLIENT_RESPONSE_TOTAL_METER_PROVIDER.create(virtualCluster.getClusterName(), nodeId); + var clientToProxyCounterProvider = Metrics.KROXYLICIOUS_CLIENT_TO_PROXY_REQUEST_TOTAL_METER_PROVIDER.create(virtualCluster.getClusterName(), binding.nodeId()); + var proxyToClientCounterProvider = Metrics.KROXYLICIOUS_PROXY_TO_CLIENT_RESPONSE_TOTAL_METER_PROVIDER.create(virtualCluster.getClusterName(), binding.nodeId()); pipeline.addLast("downstreamMetrics", new MessageMetrics(clientToProxyCounterProvider, proxyToClientCounterProvider)); pipeline.addLast("deprecatedDownstreamMetrics", deprecatedMessageMetricHandler(virtualCluster)); diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/net/BootstrapEndpointBinding.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/net/BootstrapEndpointBinding.java index cd9f7feafc..e3e722978f 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/net/BootstrapEndpointBinding.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/net/BootstrapEndpointBinding.java @@ -10,6 +10,8 @@ import io.kroxylicious.proxy.service.HostPort; +import edu.umd.cs.findbugs.annotations.Nullable; + /** * A bootstrap binding. * @@ -23,11 +25,16 @@ public record BootstrapEndpointBinding(EndpointGateway endpointGateway, HostPort Objects.requireNonNull(upstreamTarget, "upstreamTarget cannot be null"); } + @Nullable + @Override + public Integer nodeId() { + return null; + } + @Override public String toString() { return "BootstrapEndpointBinding[" + "endpointGateway=" + this.endpointGateway() + ", " + "upstreamTarget=" + this.upstreamTarget() + ']'; } - } diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/net/BrokerEndpointBinding.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/net/BrokerEndpointBinding.java index 10e733f19b..7bad6187d2 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/net/BrokerEndpointBinding.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/net/BrokerEndpointBinding.java @@ -10,6 +10,8 @@ import io.kroxylicious.proxy.service.HostPort; +import edu.umd.cs.findbugs.annotations.NonNull; + /** * A broker specific endpoint binding. * @@ -18,11 +20,18 @@ * @param nodeId kafka nodeId of the target broker * @param restrictUpstreamToMetadataDiscovery true if the upstreamTarget corresponds to a broker, false if it points at a bootstrap. */ -public record BrokerEndpointBinding(EndpointGateway endpointGateway, HostPort upstreamTarget, int nodeId, boolean restrictUpstreamToMetadataDiscovery) +public record BrokerEndpointBinding(EndpointGateway endpointGateway, HostPort upstreamTarget, Integer nodeId, boolean restrictUpstreamToMetadataDiscovery) implements EndpointBinding { public BrokerEndpointBinding { Objects.requireNonNull(endpointGateway, "endpointGateway must not be null"); Objects.requireNonNull(upstreamTarget, "upstreamTarget must not be null"); + Objects.requireNonNull(nodeId, "nodeId must not be null"); + } + + @Override + @NonNull + public Integer nodeId() { + return nodeId; } @Override @@ -35,7 +44,7 @@ public String toString() { } public boolean refersToSameVirtualClusterAndNode(BrokerEndpointBinding other) { - return other != null && other.nodeId == this.nodeId && Objects.equals(other.endpointGateway, this.endpointGateway); + return Objects.equals(other.nodeId, nodeId) && Objects.equals(other.endpointGateway, this.endpointGateway); } } diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/net/EndpointBinding.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/net/EndpointBinding.java index ed439ff202..45e8ee3c43 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/net/EndpointBinding.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/net/EndpointBinding.java @@ -6,6 +6,8 @@ package io.kroxylicious.proxy.internal.net; +import javax.annotation.Nullable; + import io.kroxylicious.proxy.service.HostPort; /** @@ -34,4 +36,12 @@ public interface EndpointBinding { default boolean restrictUpstreamToMetadataDiscovery() { return false; } + + /** + * Returns the broker node id associated with this endpoint. If the endpoint + * is being used for bootstrapping, null will be returned instead. + * @return node id or null. + */ + @Nullable + Integer nodeId(); } From 8df2041bc2188f96c6ec381107df07a1dc8ed19a Mon Sep 17 00:00:00 2001 From: Keith Wall Date: Tue, 3 Jun 2025 11:40:27 +0100 Subject: [PATCH 22/83] adding short circuit IT Signed-off-by: Keith Wall Signed-off-by: Ujjwal --- .../java/io/kroxylicious/proxy/MetricsIT.java | 47 +++++++++++++++---- .../filter/RejectingCreateTopicFilter.java | 24 +++++++--- 2 files changed, 54 insertions(+), 17 deletions(-) diff --git a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java index 103470ce4e..22262f97b6 100644 --- a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java +++ b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java @@ -178,6 +178,8 @@ void prometheusMetricsWithCommonTags(KafkaCluster cluster) { static Stream transitScenarios() { var decodeAll = new NamedFilterDefinitionBuilder("decodeAll", "DecodeAll").build(); + var rejectCreateTopic = new NamedFilterDefinitionBuilder("rejectCreateTopic", "RejectingCreateTopicFilterFactory").withConfig("respondWithError", Boolean.FALSE) + .build(); return Stream.of( argumentSet("counts opaque requests from client", (UnaryOperator) builder -> builder, @@ -196,9 +198,7 @@ static Stream transitScenarios() { .value() .isEqualTo(1.0)), argumentSet("counts decoded requests from client", - (UnaryOperator) builder -> builder - .addToFilterDefinitions(decodeAll) - .addToDefaultFilters(decodeAll.name()), + (UnaryOperator) builder -> addFilterToConfig(builder, decodeAll), (Consumer>) metricList -> assertThat(metricList) .filterByName("kroxylicious_client_to_proxy_request_total") .filterByTag(API_KEY_LABEL, ApiKeys.CREATE_TOPICS.name()) @@ -229,9 +229,7 @@ static Stream transitScenarios() { .value() .isEqualTo(1.0)), argumentSet("counts decoded requests to server", - (UnaryOperator) builder -> builder - .addToFilterDefinitions(decodeAll) - .addToDefaultFilters(decodeAll.name()), + (UnaryOperator) builder -> addFilterToConfig(builder, decodeAll), (Consumer>) metricList -> assertThat(metricList) .filterByName("kroxylicious_proxy_to_server_request_total") .filterByTag(API_KEY_LABEL, ApiKeys.CREATE_TOPICS.name()) @@ -283,9 +281,7 @@ static Stream transitScenarios() { .value() .isEqualTo(1.0)), argumentSet("counts decoded responses to client", - (UnaryOperator) builder -> builder - .addToFilterDefinitions(decodeAll) - .addToDefaultFilters(decodeAll.name()), + (UnaryOperator) builder -> addFilterToConfig(builder, decodeAll), (Consumer>) metricList -> assertThat(metricList) .filterByName("kroxylicious_proxy_to_client_response_total") .filterByTag(API_KEY_LABEL, ApiKeys.CREATE_TOPICS.name()) @@ -299,7 +295,38 @@ static Stream transitScenarios() { .filterByTag(DECODED_LABEL, "true") .singleElement() .value() - .isEqualTo(1.0))); + .isEqualTo(1.0)), + argumentSet("short circuited request does not tick upstream", + (UnaryOperator) builder -> addFilterToConfig(builder, rejectCreateTopic), + (Consumer>) metricList -> { + }, + (Consumer>) metricList -> { + assertThat(metricList) + .filterByName("kroxylicious_client_to_proxy_request_total") + .filterByTag(API_KEY_LABEL, ApiKeys.CREATE_TOPICS.name()) + .singleElement() + .value() + .isEqualTo(1.0); + assertThat(metricList) + .filterByName("kroxylicious_proxy_to_client_response_total") + .filterByTag(API_KEY_LABEL, ApiKeys.CREATE_TOPICS.name()) + .singleElement() + .value() + .isEqualTo(1.0); + assertThat(metricList) + .describedAs("create topic should not have gone upstream") + .filterByTag(API_KEY_LABEL, ApiKeys.CREATE_TOPICS.name()) + .extracting(SimpleMetric::name) + .doesNotContain("kroxylicious_proxy_to_server_request_total", "kroxylicious_server_to_proxy_response_total"); + }) + + ); + } + + private static ConfigurationBuilder addFilterToConfig(ConfigurationBuilder builder, NamedFilterDefinition filterDefinition) { + return builder + .addToFilterDefinitions(filterDefinition) + .addToDefaultFilters(filterDefinition.name()); } @ParameterizedTest diff --git a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/filter/RejectingCreateTopicFilter.java b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/filter/RejectingCreateTopicFilter.java index 96b215731d..898da34186 100644 --- a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/filter/RejectingCreateTopicFilter.java +++ b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/filter/RejectingCreateTopicFilter.java @@ -6,6 +6,7 @@ package io.kroxylicious.proxy.filter; +import java.util.Optional; import java.util.concurrent.CompletionStage; import org.apache.kafka.common.message.CreateTopicsRequestData; @@ -26,12 +27,14 @@ public class RejectingCreateTopicFilter implements CreateTopicsRequestFilter { private final ForwardingStyle forwardingStyle; private final boolean withCloseConnection; private final FilterFactoryContext constructionContext; + private final Boolean respondWithError; public RejectingCreateTopicFilter(FilterFactoryContext constructionContext, RejectingCreateTopicFilterConfig config) { this.constructionContext = constructionContext; - config = config == null ? new RejectingCreateTopicFilterConfig(false, ForwardingStyle.SYNCHRONOUS) : config; + config = config == null ? new RejectingCreateTopicFilterConfig(false, ForwardingStyle.SYNCHRONOUS, true) : config; this.withCloseConnection = config.withCloseConnection(); this.forwardingStyle = config.forwardingStyle(); + this.respondWithError = config.respondWithError; } @Override @@ -44,7 +47,9 @@ public CompletionStage onCreateTopicsRequest(short apiVersi allocateByteBufToTestKroxyliciousReleasesIt(context); request.topics().forEach(creatableTopic -> { CreateTopicsResponseData.CreatableTopicResult result = new CreateTopicsResponseData.CreatableTopicResult(); - result.setErrorCode(Errors.INVALID_TOPIC_EXCEPTION.code()).setErrorMessage(ERROR_MESSAGE); + if (respondWithError) { + result.setErrorCode(Errors.INVALID_TOPIC_EXCEPTION.code()).setErrorMessage(ERROR_MESSAGE); + } result.setName(creatableTopic.name()); topics.add(result); }); @@ -63,15 +68,20 @@ private static void allocateByteBufToTestKroxyliciousReleasesIt(FilterContext co } /** - * @param withCloseConnection - * If true, rejection will also close the connection */ - public record RejectingCreateTopicFilterConfig(boolean withCloseConnection, ForwardingStyle forwardingStyle) { + * @param withCloseConnection If true, rejection will also close the connection + * @param forwardingStyle forward style to use, allows the response to be delayed + * @param respondWithError if true, a response containing an error will be returned to the client, otherwise + * the client will receive an apparent success. + */ + public record RejectingCreateTopicFilterConfig(boolean withCloseConnection, ForwardingStyle forwardingStyle, Boolean respondWithError) { @JsonCreator public RejectingCreateTopicFilterConfig(@JsonProperty(value = "withCloseConnection") boolean withCloseConnection, - @JsonProperty(value = "forwardingStyle") ForwardingStyle forwardingStyle) { + @JsonProperty(value = "forwardingStyle") ForwardingStyle forwardingStyle, + @JsonProperty(value = "respondWithError") Boolean respondWithError) { this.withCloseConnection = withCloseConnection; - this.forwardingStyle = forwardingStyle == null ? ForwardingStyle.SYNCHRONOUS : forwardingStyle; + this.forwardingStyle = Optional.ofNullable(forwardingStyle).orElse(ForwardingStyle.SYNCHRONOUS); + this.respondWithError = Optional.ofNullable(respondWithError).orElse(true); } } } From 63daeb9856ab820d7541be3dca1555f3348a4d82 Mon Sep 17 00:00:00 2001 From: Keith Wall Date: Tue, 3 Jun 2025 11:54:09 +0100 Subject: [PATCH 23/83] tidy it Signed-off-by: Keith Wall Signed-off-by: Ujjwal --- .../java/io/kroxylicious/proxy/MetricsIT.java | 112 +++++------------- 1 file changed, 28 insertions(+), 84 deletions(-) diff --git a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java index 22262f97b6..44fe72767f 100644 --- a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java +++ b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java @@ -71,13 +71,7 @@ void afterEach() { @Test void nonexistentEndpointGives404(KafkaCluster cluster) { - var config = proxy(cluster) - .withNewManagement() - .withNewEndpoints() - .withNewPrometheus() - .endPrometheus() - .endEndpoints() - .endManagement(); + var config = configWithMetrics(cluster); try (var tester = kroxyliciousTester(config); var ahc = tester.getManagementClient()) { @@ -89,13 +83,7 @@ void nonexistentEndpointGives404(KafkaCluster cluster) { @Test void scrapeEndpointExists(KafkaCluster cluster) { - var config = proxy(cluster) - .withNewManagement() - .withNewEndpoints() - .withNewPrometheus() - .endPrometheus() - .endEndpoints() - .endManagement(); + var config = configWithMetrics(cluster); try (var tester = kroxyliciousTester(config); var ahc = tester.getManagementClient()) { @@ -109,13 +97,7 @@ void scrapeEndpointExists(KafkaCluster cluster) { @Test void knownPrometheusMetricPresent(KafkaCluster cluster) { - var config = proxy(cluster) - .withNewManagement() - .withNewEndpoints() - .withNewPrometheus() - .endPrometheus() - .endEndpoints() - .endManagement(); + var config = configWithMetrics(cluster); try (var tester = kroxyliciousTester(config); var ahc = tester.getManagementClient()) { @@ -132,15 +114,9 @@ void knownPrometheusMetricPresent(KafkaCluster cluster) { @Test void prometheusMetricFromNamedBinder(KafkaCluster cluster) { - var config = proxy(cluster) + var config = configWithMetrics(cluster) .addToMicrometer( - new MicrometerDefinitionBuilder(StandardBindersHook.class.getName()).withConfig("binderNames", List.of("JvmGcMetrics")).build()) - .withNewManagement() - .withNewEndpoints() - .withNewPrometheus() - .endPrometheus() - .endEndpoints() - .endManagement(); + new MicrometerDefinitionBuilder(StandardBindersHook.class.getName()).withConfig("binderNames", List.of("JvmGcMetrics")).build()); try (var tester = kroxyliciousTester(config); var ahc = tester.getManagementClient()) { @@ -153,14 +129,8 @@ void prometheusMetricFromNamedBinder(KafkaCluster cluster) { @Test void prometheusMetricsWithCommonTags(KafkaCluster cluster) { - var config = proxy(cluster) - .addToMicrometer(new MicrometerDefinitionBuilder(CommonTagsHook.class.getName()).withConfig("commonTags", Map.of("a", "b")).build()) - .withNewManagement() - .withNewEndpoints() - .withNewPrometheus() - .endPrometheus() - .endEndpoints() - .endManagement(); + var config = configWithMetrics(cluster) + .addToMicrometer(new MicrometerDefinitionBuilder(CommonTagsHook.class.getName()).withConfig("commonTags", Map.of("a", "b")).build()); try (var tester = kroxyliciousTester(config); var ahc = tester.getManagementClient()) { @@ -323,25 +293,13 @@ static Stream transitScenarios() { ); } - private static ConfigurationBuilder addFilterToConfig(ConfigurationBuilder builder, NamedFilterDefinition filterDefinition) { - return builder - .addToFilterDefinitions(filterDefinition) - .addToDefaultFilters(filterDefinition.name()); - } - @ParameterizedTest @MethodSource("transitScenarios") void shouldIncrementCountOnMessageTransit(UnaryOperator builder, Consumer> beforeAssertion, Consumer> afterAssertion, KafkaCluster cluster) { - var config = proxy(cluster) - .withNewManagement() - .withNewEndpoints() - .withNewPrometheus() - .endPrometheus() - .endEndpoints() - .endManagement(); + var config = configWithMetrics(cluster); // Given try (var tester = kroxyliciousTester(builder.apply(config)); @@ -365,13 +323,7 @@ void shouldIncrementCountOnMessageTransit(UnaryOperator bu @Test @Deprecated(since = "0.13.0", forRemoval = true) void shouldIncrementDownstreamMessagesOnProduceRequestWithoutFilter(KafkaCluster cluster, Topic topic) throws ExecutionException, InterruptedException { - var config = proxy(cluster) - .withNewManagement() - .withNewEndpoints() - .withNewPrometheus() - .endPrometheus() - .endEndpoints() - .endManagement(); + var config = configWithMetrics(cluster); // Given try (var tester = kroxyliciousTester(config); @@ -404,15 +356,7 @@ void shouldIncrementDownstreamMessagesOnProduceRequestWithFilter(KafkaCluster cl NamedFilterDefinition namedFilterDefinition = new NamedFilterDefinitionBuilder("filter", CreateTopicRequest.class.getName()).withConfig("configInstanceId", configInstance).build(); - var config = proxy(cluster) - .addToFilterDefinitions(namedFilterDefinition) - .addToDefaultFilters(namedFilterDefinition.name()) - .withNewManagement() - .withNewEndpoints() - .withNewPrometheus() - .endPrometheus() - .endEndpoints() - .endManagement(); + var config = addFilterToConfig(configWithMetrics(cluster), namedFilterDefinition); // Given try (var tester = kroxyliciousTester(config); @@ -437,15 +381,7 @@ void shouldIncrementDownstreamMessagesOnProduceRequestWithFilter(KafkaCluster cl @Test void shouldIncrementConnectionMetrics(KafkaCluster cluster, Topic topic) throws ExecutionException, InterruptedException { - var config = proxy(cluster) - .addToFilterDefinitions() - .addToDefaultFilters() - .withNewManagement() - .withNewEndpoints() - .withNewPrometheus() - .endPrometheus() - .endEndpoints() - .endManagement(); + var config = configWithMetrics(cluster); // Given try (var tester = kroxyliciousTester(config); @@ -468,15 +404,7 @@ void shouldIncrementConnectionMetrics(KafkaCluster cluster, Topic topic) throws @Test void shouldIncrementPayloadSizeBytesMetricsOnProduceRequest(KafkaCluster cluster, Topic topic) throws ExecutionException, InterruptedException { - var config = proxy(cluster) - .addToFilterDefinitions() - .addToDefaultFilters() - .withNewManagement() - .withNewEndpoints() - .withNewPrometheus() - .endPrometheus() - .endEndpoints() - .endManagement(); + var config = configWithMetrics(cluster); // Given try (var tester = kroxyliciousTester(config); @@ -537,4 +465,20 @@ void assertMetricsWithValue(List metricList, String metricsName, A private String getRandomCounterName() { return "test_metric_" + Math.abs(new Random().nextLong()) + "_total"; } + + private ConfigurationBuilder configWithMetrics(KafkaCluster cluster) { + return proxy(cluster) + .withNewManagement() + .withNewEndpoints() + .withNewPrometheus() + .endPrometheus() + .endEndpoints() + .endManagement(); + } + + private static ConfigurationBuilder addFilterToConfig(ConfigurationBuilder builder, NamedFilterDefinition filterDefinition) { + return builder + .addToFilterDefinitions(filterDefinition) + .addToDefaultFilters(filterDefinition.name()); + } } From 47544a6be3b50dde3b0977f65e465648028a62b9 Mon Sep 17 00:00:00 2001 From: Keith Wall Date: Tue, 3 Jun 2025 12:30:45 +0100 Subject: [PATCH 24/83] adding IT checking metrics are discriminated by node_id sonar Signed-off-by: Keith Wall Signed-off-by: Ujjwal --- .../java/io/kroxylicious/proxy/MetricsIT.java | 56 +++++++++++++++++++ .../internal/net/BrokerEndpointBinding.java | 1 + 2 files changed, 57 insertions(+) diff --git a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java index 44fe72767f..abceacc98f 100644 --- a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java +++ b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java @@ -19,6 +19,8 @@ import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.TopicPartitionInfo; import org.apache.kafka.common.protocol.ApiKeys; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterEach; @@ -41,8 +43,10 @@ import io.kroxylicious.proxy.micrometer.StandardBindersHook; import io.kroxylicious.test.tester.SimpleMetric; import io.kroxylicious.testing.kafka.api.KafkaCluster; +import io.kroxylicious.testing.kafka.common.BrokerCluster; import io.kroxylicious.testing.kafka.junit5ext.KafkaClusterExtension; import io.kroxylicious.testing.kafka.junit5ext.Topic; +import io.kroxylicious.testing.kafka.junit5ext.TopicPartitions; import edu.umd.cs.findbugs.annotations.NonNull; @@ -320,6 +324,54 @@ void shouldIncrementCountOnMessageTransit(UnaryOperator bu } } + @Test + void countMetricsShouldBeDiscriminatedByNodeId(@BrokerCluster(numBrokers = 2) KafkaCluster cluster, @TopicPartitions(2) Topic topic) + throws ExecutionException, InterruptedException { + var config = configWithMetrics(cluster); + + try (var tester = kroxyliciousTester(config); + var managementClient = tester.getManagementClient(); + var producer = tester.producer(); + var admin = tester.admin()) { + + // Given + var describeFuture = admin.describeTopics(List.of(topic.name())).allTopicNames(); + assertThat(describeFuture).succeedsWithin(Duration.ofSeconds(5)); + + var partitions = describeFuture.get().get(topic.name()).partitions(); + var partitionZeroNodeId = getNodeIdForPartition(partitions, 0); + var partitionOneNodeId = getNodeIdForPartition(partitions, 1); + assertThat(partitionZeroNodeId) + .describedAs("expecting topic partitions to be distributed amongst brokers") + .isNotEqualTo(partitionOneNodeId); + + // When + List.of(new ProducerRecord<>(topic.name(), 0, "my-key", "0-1"), + new ProducerRecord<>(topic.name(), 0, "my-key", "0-2"), + new ProducerRecord<>(topic.name(), 1, "my-key", "0-2")) + .forEach(pr -> assertThat(producer.send(pr)).succeedsWithin(Duration.ofSeconds(5))); + + // Then + var metricList = managementClient.scrapeMetrics(); + + assertThat(metricList) + .filterByName("kroxylicious_client_to_proxy_request_total") + .filterByTag(API_KEY_LABEL, ApiKeys.PRODUCE.name()) + .filterByTag(NODE_ID_LABEL, "" + partitionZeroNodeId) + .singleElement() + .value() + .isEqualTo(2.0); + + assertThat(metricList) + .filterByName("kroxylicious_client_to_proxy_request_total") + .filterByTag(API_KEY_LABEL, ApiKeys.PRODUCE.name()) + .filterByTag(NODE_ID_LABEL, "" + partitionOneNodeId) + .singleElement() + .value() + .isEqualTo(1.0); + } + } + @Test @Deprecated(since = "0.13.0", forRemoval = true) void shouldIncrementDownstreamMessagesOnProduceRequestWithoutFilter(KafkaCluster cluster, Topic topic) throws ExecutionException, InterruptedException { @@ -426,6 +478,10 @@ void shouldIncrementPayloadSizeBytesMetricsOnProduceRequest(KafkaCluster cluster } } + private int getNodeIdForPartition(List partitions, int partitionId) { + return partitions.stream().filter(p -> p.partition() == partitionId).map(TopicPartitionInfo::leader).map(Node::id).findFirst().orElseThrow(); + } + void assertMetricsDoesNotExist(List metricList, String metricsName, ApiKeys apiKey) { Assertions.assertThat(metricList) .hasSizeGreaterThan(0) diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/net/BrokerEndpointBinding.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/net/BrokerEndpointBinding.java index 7bad6187d2..2acba3ca0d 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/net/BrokerEndpointBinding.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/net/BrokerEndpointBinding.java @@ -30,6 +30,7 @@ public record BrokerEndpointBinding(EndpointGateway endpointGateway, HostPort up @Override @NonNull + @SuppressWarnings("java:S6207") // method's return annotation differs from that of the interface public Integer nodeId() { return nodeId; } From 8cc665d72bc445ef5850c5ed7b7f0fdb20a76b0a Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Tue, 27 May 2025 21:59:53 +1200 Subject: [PATCH 25/83] Cert type thing in the system test Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- kroxylicious-systemtests/pom.xml | 5 ++ .../OperatorChangeDetectionST.java | 53 +++++++++++++++++-- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/kroxylicious-systemtests/pom.xml b/kroxylicious-systemtests/pom.xml index ce3625a91e..755cccdf45 100644 --- a/kroxylicious-systemtests/pom.xml +++ b/kroxylicious-systemtests/pom.xml @@ -66,6 +66,11 @@ kubernetes-model-rbac compile + + io.fabric8 + certmanager-model + test + io.fabric8 kubernetes-client diff --git a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java index d920000fea..7745cf3440 100644 --- a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java +++ b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java @@ -6,6 +6,7 @@ package io.kroxylicious.systemtests; +import java.io.IOException; import java.time.Duration; import java.util.List; import java.util.Map; @@ -20,6 +21,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import io.fabric8.certmanager.api.model.v1.CertificateBuilder; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.PodTemplateSpecBuilder; @@ -27,7 +29,6 @@ import io.fabric8.kubernetes.api.model.SecretBuilder; import io.fabric8.kubernetes.api.model.apps.Deployment; import io.fabric8.kubernetes.client.utils.KubernetesResourceUtil; -import io.skodjob.testframe.resources.KubeResourceManager; import io.kroxylicious.kubernetes.api.common.FilterRef; import io.kroxylicious.kubernetes.api.common.FilterRefBuilder; @@ -39,6 +40,7 @@ import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilter; import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilterBuilder; import io.kroxylicious.kubernetes.operator.assertj.OperatorAssertions; +import io.kroxylicious.systemtests.installation.kroxylicious.CertManager; import io.kroxylicious.systemtests.installation.kroxylicious.Kroxylicious; import io.kroxylicious.systemtests.installation.kroxylicious.KroxyliciousOperator; import io.kroxylicious.systemtests.k8s.KubeClient; @@ -96,20 +98,61 @@ void shouldUpdateDeploymentWhenVirtualKafkaClusterChanges(String namespace) { String originalChecksum = getInitialChecksum(namespace, kubeClient); + FilterRef filterRef = new FilterRefBuilder().withName("arbitrary-filter").build(); VirtualKafkaCluster virtualKafkaCluster = kubeClient.getClient().resources(VirtualKafkaCluster.class).inNamespace(namespace) .withName("test-vkc").get(); - FilterRef filterRef = new FilterRefBuilder().withName("arbitrary-filter").build(); - VirtualKafkaCluster updatedVirtualCluster = virtualKafkaCluster.edit().editSpec().withFilterRefs(filterRef).endSpec().build(); - // When - KubeResourceManager.get().createOrUpdateResourceWithWait(updatedVirtualCluster); + resourceManager.replaceResourceWithRetries(virtualKafkaCluster, vkc -> vkc.getSpec().getFilterRefs().add(filterRef)); LOGGER.info("virtual cluster edited"); // Then assertDeploymentUpdated(namespace, kubeClient, originalChecksum); } + @Test + void shouldUpdateDeploymentWhenDownstreamTlsCertUpdated(String namespace) throws IOException { + // Given + CertManager certManager = new CertManager(); + certManager.deploy(); + //@formatter:off + CertificateBuilder cert = new CertificateBuilder() + .withNewMetadata() + .withName("server-certificate") + .withNamespace(namespace) + .endMetadata() + .withNewSpec() + .withCommonName("my-cluster-cluster-ip.my-proxy.svc.cluster.local") + .withSecretName("server-certificate") + .withNewPrivateKey() + .withAlgorithm("RSA") + .withEncoding("PKCS8") + .withSize(4096) + .endPrivateKey() + .withDnsNames("my-cluster-cluster-ip.my-proxy.svc.cluster.local") + .withUsages("server auth") + .withNewIssuerRef() + .withName("selfsinged-issuer") + .withKind("Issuer") + .withGroup("cert-manager.io") + .endIssuerRef() + .endSpec(); + //@formatter:on + resourceManager.createOrUpdateResourceWithWait(cert); + kroxylicious.deployPortIdentifiesNodeWithNoFilters("test-vkc"); + KubeClient kubeClient = kubeClient(namespace); + + String originalChecksum = getInitialChecksum(namespace, kubeClient); + + // When + resourceManager.replaceResourceWithRetries(cert.build(), + certToPatch -> certToPatch.getSpec().getDnsNames().add("test-vkc-cluster-ip.my-proxy.svc.cluster.local")); + LOGGER.info("SAN added to downstream tls cert"); + + // Then + assertDeploymentUpdated(namespace, kubeClient, originalChecksum); + } + @Test void shouldUpdateWhenFilterConfigurationChanges(String namespace) { // Given From 3ceeea39c0cc4b2817851dd10c9d3fd300bab51d Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Wed, 28 May 2025 17:22:40 +1200 Subject: [PATCH 26/83] Updating the cert causes the podTemplate updated. Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- kroxylicious-systemtests/pom.xml | 2 +- .../kroxylicious/CertManager.java | 48 +++++++++++++++++++ .../kroxylicious/Kroxylicious.java | 44 +++++++++++++++-- .../KroxyliciousKafkaClusterRefTemplates.java | 25 ++++++---- ...roxyliciousKafkaProxyIngressTemplates.java | 28 +++++++++++ ...xyliciousVirtualKafkaClusterTemplates.java | 24 ++++++++++ .../OperatorChangeDetectionST.java | 41 +++++----------- 7 files changed, 171 insertions(+), 41 deletions(-) diff --git a/kroxylicious-systemtests/pom.xml b/kroxylicious-systemtests/pom.xml index 755cccdf45..1de3378135 100644 --- a/kroxylicious-systemtests/pom.xml +++ b/kroxylicious-systemtests/pom.xml @@ -69,7 +69,7 @@ io.fabric8 certmanager-model - test + compile io.fabric8 diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java index a966473d6c..90f62fc8e2 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java @@ -7,10 +7,14 @@ package io.kroxylicious.systemtests.installation.kroxylicious; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import io.fabric8.certmanager.api.model.v1.CertificateBuilder; +import io.fabric8.certmanager.api.model.v1.IssuerBuilder; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.client.dsl.NamespaceListVisitFromServerGetDeleteRecreateWaitApplicable; @@ -25,6 +29,7 @@ */ public class CertManager { private static final Logger LOGGER = LoggerFactory.getLogger(CertManager.class); + public static final String SELF_SINGED_ISSUER_NAME = "self-signed-issuer"; private final NamespaceListVisitFromServerGetDeleteRecreateWaitApplicable deployment; private boolean deleteCertManager = true; @@ -39,6 +44,49 @@ public CertManager() throws IOException { .load(DeploymentUtils.getDeploymentFileFromURL(Constants.CERT_MANAGER_URL)); } + public IssuerBuilder issuer(String namespace) { + return new IssuerBuilder() + .withNewMetadata() + .withName(SELF_SINGED_ISSUER_NAME) + .withNamespace(namespace) + .endMetadata() + .withNewSpec() + .withNewSelfSigned() + .endSelfSigned() + .endSpec(); + } + + public CertificateBuilder certFor(String namespace, String commonName, String... dnsNames) { + List dnsAliases = new ArrayList<>(List.of(dnsNames)); + if (!dnsAliases.contains(commonName)) { + dnsAliases.add(0, commonName); + } + + //@formatter:off + return new CertificateBuilder() + .withNewMetadata() + .withName("server-certificate") + .withNamespace(namespace) + .endMetadata() + .withNewSpec() + .withCommonName(commonName) + .withSecretName("server-certificate") + .withNewPrivateKey() + .withAlgorithm("RSA") + .withEncoding("PKCS8") + .withSize(4096) + .endPrivateKey() + .withDnsNames(dnsAliases) + .withUsages("server auth") + .withNewIssuerRef() + .withName(SELF_SINGED_ISSUER_NAME) + .withKind("Issuer") + .withGroup("cert-manager.io") + .endIssuerRef() + .endSpec(); + //@formatter:on + } + private boolean isDeployed() { return !kubeClient().getClient().apps().deployments().inAnyNamespace().withLabelSelector("app=cert-manager").list().getItems().isEmpty(); } diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/Kroxylicious.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/Kroxylicious.java index 858c2fbd5d..92ed1c4fc3 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/Kroxylicious.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/Kroxylicious.java @@ -16,8 +16,12 @@ import io.strimzi.api.kafka.model.kafka.listener.ListenerStatus; import io.kroxylicious.kms.service.TestKmsFacade; +import io.kroxylicious.kubernetes.api.common.CertificateRefBuilder; +import io.kroxylicious.kubernetes.api.common.TrustAnchorRefBuilder; import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaCluster; import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaClusterStatus; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaservicespec.Tls; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaservicespec.TlsBuilder; import io.kroxylicious.kubernetes.api.v1alpha1.virtualkafkaclusterstatus.Ingresses; import io.kroxylicious.systemtests.Constants; import io.kroxylicious.systemtests.resources.kms.ExperimentalKmsConfig; @@ -92,17 +96,27 @@ public void deployPortIdentifiesNodeWithNoFilters(String clusterName) { * @param clusterName the cluster name */ public void deployPortIdentifiesNodeWithTlsAndNoFilters(String clusterName) { - createCertificateConfigMap(deploymentNamespace); + Tls tls = createCertificateConfigMapFromListener(deploymentNamespace); resourceManager.createResourceFromBuilder( KroxyliciousKafkaProxyTemplates.defaultKafkaProxyCR(deploymentNamespace, Constants.KROXYLICIOUS_PROXY_SIMPLE_NAME), KroxyliciousKafkaProxyIngressTemplates.defaultKafkaProxyIngressCR(deploymentNamespace, Constants.KROXYLICIOUS_INGRESS_CLUSTER_IP, Constants.KROXYLICIOUS_PROXY_SIMPLE_NAME), - KroxyliciousKafkaClusterRefTemplates.kafkaClusterRefCRWithTls(deploymentNamespace, clusterName), + KroxyliciousKafkaClusterRefTemplates.kafkaClusterRefCRWithTls(deploymentNamespace, clusterName, tls), KroxyliciousVirtualKafkaClusterTemplates.defaultVirtualKafkaClusterCR(deploymentNamespace, clusterName, Constants.KROXYLICIOUS_PROXY_SIMPLE_NAME, clusterName, Constants.KROXYLICIOUS_INGRESS_CLUSTER_IP)); } - private void createCertificateConfigMap(String namespace) { + public void deployPortIdentifiesNodeWithDownstreamTlsAndNoFilters(String clusterName, Tls tls) { + resourceManager.createResourceFromBuilder( + KroxyliciousKafkaProxyTemplates.defaultKafkaProxyCR(deploymentNamespace, Constants.KROXYLICIOUS_PROXY_SIMPLE_NAME), + KroxyliciousKafkaProxyIngressTemplates.tlsKafkaProxyIngressCR(deploymentNamespace, Constants.KROXYLICIOUS_INGRESS_CLUSTER_IP, + Constants.KROXYLICIOUS_PROXY_SIMPLE_NAME), + KroxyliciousKafkaClusterRefTemplates.kafkaClusterRefCRWithTls(deploymentNamespace, clusterName, tls), + KroxyliciousVirtualKafkaClusterTemplates.defaultVirtualKafkaClusterWithTlsCR(deploymentNamespace, clusterName, Constants.KROXYLICIOUS_PROXY_SIMPLE_NAME, + clusterName, Constants.KROXYLICIOUS_INGRESS_CLUSTER_IP, tls)); + } + + public Tls createCertificateConfigMapFromListener(String namespace) { // wait for listeners to contain data var tlsListenerStatus = KafkaUtils.getKafkaListenerStatus("tls"); @@ -112,6 +126,30 @@ private void createCertificateConfigMap(String namespace) { resourceManager.createResourceFromBuilder(KroxyliciousConfigMapTemplates.getClusterCaConfigMap(namespace, Constants.KROXYLICIOUS_TLS_CLIENT_CA_CERT, cert.get(0))); + //@formatter:off + return new TlsBuilder() + .withTrustAnchorRef( + new TrustAnchorRefBuilder() + .withNewRef() + .withName(Constants.KROXYLICIOUS_TLS_CLIENT_CA_CERT) + .withKind(Constants.CONFIG_MAP) + .endRef() + .withKey(Constants.KROXYLICIOUS_TLS_CA_NAME) + .build()) + .build(); + //@formatter:on + } + + public Tls tlsConfigFromCert(String certNane) { + //@formatter:off + return new TlsBuilder() + .withCertificateRef( + new CertificateRefBuilder() + .withName(certNane) + .withKind("Secret") + .build()) + .build(); + //@formatter:on } /** diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaClusterRefTemplates.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaClusterRefTemplates.java index c0ee80437a..6b73bb2aaa 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaClusterRefTemplates.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaClusterRefTemplates.java @@ -8,8 +8,10 @@ import io.strimzi.api.kafka.model.kafka.listener.ListenerStatus; +import io.kroxylicious.kubernetes.api.common.TrustAnchorRef; import io.kroxylicious.kubernetes.api.common.TrustAnchorRefBuilder; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaServiceBuilder; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaservicespec.Tls; import io.kroxylicious.kubernetes.api.v1alpha1.kafkaservicespec.TlsBuilder; import io.kroxylicious.systemtests.Constants; import io.kroxylicious.systemtests.utils.KafkaUtils; @@ -47,19 +49,24 @@ public static KafkaServiceBuilder defaultKafkaClusterRefCR(String namespaceName, * @return the kafka service builder */ public static KafkaServiceBuilder kafkaClusterRefCRWithTls(String namespaceName, String clusterRefName) { + final TrustAnchorRef trustAnchorRef = new TrustAnchorRefBuilder() + .withNewRef() + .withName(Constants.KROXYLICIOUS_TLS_CLIENT_CA_CERT) + .withKind(Constants.CONFIG_MAP) + .endRef() + .withKey(Constants.KROXYLICIOUS_TLS_CA_NAME) + .build(); + return kafkaClusterRefCRWithTls(namespaceName, clusterRefName, new TlsBuilder() + .withTrustAnchorRef(trustAnchorRef) + .build()); + } + + public static KafkaServiceBuilder kafkaClusterRefCRWithTls(String namespaceName, String clusterRefName, Tls tls) { // @formatter:off return defaultKafkaClusterRefCR(namespaceName, clusterRefName) .editSpec() .withBootstrapServers(getKafkaBootstrap("tls", clusterRefName)) - .withTls(new TlsBuilder() - .withTrustAnchorRef(new TrustAnchorRefBuilder() - .withNewRef() - .withName(Constants.KROXYLICIOUS_TLS_CLIENT_CA_CERT) - .withKind(Constants.CONFIG_MAP) - .endRef() - .withKey(Constants.KROXYLICIOUS_TLS_CA_NAME) - .build()) - .build()) + .withTls(tls) .endSpec(); // @formatter:on } diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaProxyIngressTemplates.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaProxyIngressTemplates.java index 79b425c4da..6def8a51fc 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaProxyIngressTemplates.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaProxyIngressTemplates.java @@ -7,8 +7,10 @@ package io.kroxylicious.systemtests.templates.kroxylicious; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngressBuilder; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP; import static io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP.Protocol.TCP; +import static io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP.Protocol.TLS; public class KroxyliciousKafkaProxyIngressTemplates { @@ -40,4 +42,30 @@ public static KafkaProxyIngressBuilder defaultKafkaProxyIngressCR(String namespa .endSpec(); // @formatter:on } + + /** + * Default kafka proxy ingress CR. + * + * @param namespaceName the namespace name + * @param ingressName the ingress name + * @param proxyName the name of the proxy to reference + * @return the kafka proxy ingress builder + */ + public static KafkaProxyIngressBuilder tlsKafkaProxyIngressCR(String namespaceName, String ingressName, String proxyName) { + // @formatter:off + return new KafkaProxyIngressBuilder() + .withNewMetadata() + .withName(ingressName) + .withNamespace(namespaceName) + .endMetadata() + .withNewSpec() + .withNewClusterIP() + .withProtocol(TLS) + .endClusterIP() + .withNewProxyRef() + .withName(proxyName) + .endProxyRef() + .endSpec(); + // @formatter:on + } } diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousVirtualKafkaClusterTemplates.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousVirtualKafkaClusterTemplates.java index da71efce78..cfa81ea358 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousVirtualKafkaClusterTemplates.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousVirtualKafkaClusterTemplates.java @@ -14,6 +14,7 @@ import io.kroxylicious.kubernetes.api.common.FilterRefBuilder; import io.kroxylicious.kubernetes.api.common.KafkaServiceRefBuilder; import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaClusterBuilder; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaservicespec.Tls; public class KroxyliciousVirtualKafkaClusterTemplates { @@ -59,6 +60,29 @@ public static VirtualKafkaClusterBuilder defaultVirtualKafkaClusterCR(String nam return baseVirtualKafkaClusterCR(namespaceName, clusterName, proxyName, clusterRefName, ingressName); } + /** + * Default virtual kafka cluster CR. + * + * @param namespaceName the namespace name + * @param clusterName the cluster name + * @param proxyName the proxy name + * @param clusterRefName the cluster ref name + * @param ingressName the ingress name + * @param tls + * @return the virtual kafka cluster builder + */ + public static VirtualKafkaClusterBuilder defaultVirtualKafkaClusterWithTlsCR(String namespaceName, String clusterName, String proxyName, String clusterRefName, + String ingressName, Tls tls) { + return baseVirtualKafkaClusterCR(namespaceName, clusterName, proxyName, clusterRefName, ingressName) + .editSpec() + .editIngress(0) + .withNewTls() + .withCertificateRef(tls.getCertificateRef()) + .endTls() + .endIngress() + .endSpec(); + } + /** * Default virtual kafka cluster CR. * diff --git a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java index 7745cf3440..c4d198026d 100644 --- a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java +++ b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java @@ -21,7 +21,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import io.fabric8.certmanager.api.model.v1.CertificateBuilder; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.PodTemplateSpecBuilder; @@ -32,11 +31,13 @@ import io.kroxylicious.kubernetes.api.common.FilterRef; import io.kroxylicious.kubernetes.api.common.FilterRefBuilder; +import io.kroxylicious.kubernetes.api.common.TrustAnchorRef; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxy; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyBuilder; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngress; import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaCluster; import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaservicespec.TlsBuilder; import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilter; import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilterBuilder; import io.kroxylicious.kubernetes.operator.assertj.OperatorAssertions; @@ -115,31 +116,14 @@ void shouldUpdateDeploymentWhenDownstreamTlsCertUpdated(String namespace) throws // Given CertManager certManager = new CertManager(); certManager.deploy(); - //@formatter:off - CertificateBuilder cert = new CertificateBuilder() - .withNewMetadata() - .withName("server-certificate") - .withNamespace(namespace) - .endMetadata() - .withNewSpec() - .withCommonName("my-cluster-cluster-ip.my-proxy.svc.cluster.local") - .withSecretName("server-certificate") - .withNewPrivateKey() - .withAlgorithm("RSA") - .withEncoding("PKCS8") - .withSize(4096) - .endPrivateKey() - .withDnsNames("my-cluster-cluster-ip.my-proxy.svc.cluster.local") - .withUsages("server auth") - .withNewIssuerRef() - .withName("selfsinged-issuer") - .withKind("Issuer") - .withGroup("cert-manager.io") - .endIssuerRef() - .endSpec(); - //@formatter:on - resourceManager.createOrUpdateResourceWithWait(cert); - kroxylicious.deployPortIdentifiesNodeWithNoFilters("test-vkc"); + + var issuer = certManager.issuer(namespace); + var cert = certManager.certFor(namespace, "my-cluster-cluster-ip." + namespace + ".svc.cluster.local"); + + resourceManager.createOrUpdateResourceWithWait(issuer, cert); + + var tls = kroxylicious.tlsConfigFromCert("server-certificate"); + kroxylicious.deployPortIdentifiesNodeWithDownstreamTlsAndNoFilters("test-vkc", tls); KubeClient kubeClient = kubeClient(namespace); String originalChecksum = getInitialChecksum(namespace, kubeClient); @@ -248,7 +232,7 @@ void shouldUpdateDeploymentWhenSecretChanges(String namespace) { private static void assertDeploymentUpdated(String namespace, KubeClient kubeClient, String originalChecksum) { await().atMost(Duration.ofSeconds(90)).untilAsserted(() -> { Deployment proxyDeployment = kubeClient.getDeployment(namespace, Constants.KROXYLICIOUS_PROXY_SIMPLE_NAME); - Assertions.assertThat(proxyDeployment).isNotNull(); + assertThat(proxyDeployment).isNotNull(); OperatorAssertions.assertThat(proxyDeployment.getSpec().getTemplate().getMetadata()).hasAnnotationSatisfying("kroxylicious.io/referent-checksum", value -> assertThat(value).isNotEqualTo(originalChecksum)); }); @@ -284,13 +268,14 @@ private String getInitialChecksum(String namespace, KubeClient kubeClient) { .hasAnnotationSatisfying("kroxylicious.io/referent-checksum", value -> assertThat(value).isNotBlank())); String checksumFromPod = getChecksumFromAnnotation(proxyPods.get(0)); - Deployment proxyDeployment = kubeClient.getDeployment(namespace, "simple"); + Deployment proxyDeployment = kubeClient.getDeployment(namespace, Constants.KROXYLICIOUS_PROXY_SIMPLE_NAME); OperatorAssertions.assertThat(proxyDeployment.getSpec().getTemplate().getMetadata()) .hasAnnotationSatisfying( "kroxylicious.io/referent-checksum", value -> assertThat(value).isEqualTo(checksumFromPod)); checksumFromAnnotation.set(checksumFromPod); }); + LOGGER.info("initial checksum: '{}'", checksumFromAnnotation); return checksumFromAnnotation.get(); } From f1c3b7ddba794ae3765cb9c76d49f092d1cc0242 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Wed, 28 May 2025 20:27:50 +1200 Subject: [PATCH 27/83] Formatting! Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../systemtests/installation/kroxylicious/CertManager.java | 2 ++ .../kroxylicious/KroxyliciousKafkaProxyIngressTemplates.java | 1 - .../KroxyliciousVirtualKafkaClusterTemplates.java | 4 +++- .../kroxylicious/systemtests/OperatorChangeDetectionST.java | 3 --- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java index 90f62fc8e2..955562f337 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java @@ -45,6 +45,7 @@ public CertManager() throws IOException { } public IssuerBuilder issuer(String namespace) { + //@formatter:off return new IssuerBuilder() .withNewMetadata() .withName(SELF_SINGED_ISSUER_NAME) @@ -54,6 +55,7 @@ public IssuerBuilder issuer(String namespace) { .withNewSelfSigned() .endSelfSigned() .endSpec(); + //@formatter:on } public CertificateBuilder certFor(String namespace, String commonName, String... dnsNames) { diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaProxyIngressTemplates.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaProxyIngressTemplates.java index 6def8a51fc..49d2e93ab5 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaProxyIngressTemplates.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaProxyIngressTemplates.java @@ -7,7 +7,6 @@ package io.kroxylicious.systemtests.templates.kroxylicious; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngressBuilder; -import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP; import static io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP.Protocol.TCP; import static io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP.Protocol.TLS; diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousVirtualKafkaClusterTemplates.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousVirtualKafkaClusterTemplates.java index cfa81ea358..731f1ec923 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousVirtualKafkaClusterTemplates.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousVirtualKafkaClusterTemplates.java @@ -68,11 +68,12 @@ public static VirtualKafkaClusterBuilder defaultVirtualKafkaClusterCR(String nam * @param proxyName the proxy name * @param clusterRefName the cluster ref name * @param ingressName the ingress name - * @param tls + * @param tls the TLS config with a certificate references * @return the virtual kafka cluster builder */ public static VirtualKafkaClusterBuilder defaultVirtualKafkaClusterWithTlsCR(String namespaceName, String clusterName, String proxyName, String clusterRefName, String ingressName, Tls tls) { + //@formatter:off return baseVirtualKafkaClusterCR(namespaceName, clusterName, proxyName, clusterRefName, ingressName) .editSpec() .editIngress(0) @@ -81,6 +82,7 @@ public static VirtualKafkaClusterBuilder defaultVirtualKafkaClusterWithTlsCR(Str .endTls() .endIngress() .endSpec(); + //@formatter:on } /** diff --git a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java index c4d198026d..3766f9fed2 100644 --- a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java +++ b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java @@ -12,7 +12,6 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; -import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -31,13 +30,11 @@ import io.kroxylicious.kubernetes.api.common.FilterRef; import io.kroxylicious.kubernetes.api.common.FilterRefBuilder; -import io.kroxylicious.kubernetes.api.common.TrustAnchorRef; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxy; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyBuilder; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngress; import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaCluster; import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP; -import io.kroxylicious.kubernetes.api.v1alpha1.kafkaservicespec.TlsBuilder; import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilter; import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilterBuilder; import io.kroxylicious.kubernetes.operator.assertj.OperatorAssertions; From 904889953f5e590ee0854ed0aae0d77703e96bbb Mon Sep 17 00:00:00 2001 From: Francisco Vila Date: Wed, 28 May 2025 12:45:59 +0200 Subject: [PATCH 28/83] some fixes to the 2211 tactical PR Signed-off-by: Francisco Vila Signed-off-by: Ujjwal --- .../kroxylicious/CertManager.java | 4 +-- .../OperatorChangeDetectionST.java | 27 ++++++++++++++++--- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java index 955562f337..d5136f3118 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java @@ -109,15 +109,15 @@ public void deploy() { /** * Delete cert manager. - * @throws IOException the io exception */ - public void delete() throws IOException { + public void delete() { if (!deleteCertManager) { LOGGER.warn("Skipping cert manager deletion. It was previously installed"); return; } LOGGER.info("Deleting Cert Manager in {} namespace", Constants.CERT_MANAGER_NAMESPACE); deployment.withGracePeriod(0).delete(); + DeploymentUtils.waitForDeploymentDeletion(Constants.CERT_MANAGER_NAMESPACE, "cert-manager-webhook"); NamespaceUtils.deleteNamespaceWithWait(Constants.CERT_MANAGER_NAMESPACE); } } diff --git a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java index 3766f9fed2..f53e59709b 100644 --- a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java +++ b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java @@ -10,9 +10,11 @@ import java.time.Duration; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; @@ -27,6 +29,9 @@ import io.fabric8.kubernetes.api.model.SecretBuilder; import io.fabric8.kubernetes.api.model.apps.Deployment; import io.fabric8.kubernetes.client.utils.KubernetesResourceUtil; +import io.skodjob.testframe.resources.KubeResourceManager; +import io.strimzi.api.kafka.model.kafka.Kafka; +import io.strimzi.api.kafka.model.kafka.KafkaStatus; import io.kroxylicious.kubernetes.api.common.FilterRef; import io.kroxylicious.kubernetes.api.common.FilterRefBuilder; @@ -34,6 +39,7 @@ import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyBuilder; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngress; import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaCluster; +import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaClusterSpec; import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP; import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilter; import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilterBuilder; @@ -42,6 +48,7 @@ import io.kroxylicious.systemtests.installation.kroxylicious.Kroxylicious; import io.kroxylicious.systemtests.installation.kroxylicious.KroxyliciousOperator; import io.kroxylicious.systemtests.k8s.KubeClient; +import io.kroxylicious.systemtests.resources.kroxylicious.KroxyliciousResource; import io.kroxylicious.systemtests.resources.manager.ResourceManager; import io.kroxylicious.systemtests.templates.kroxylicious.KroxyliciousFilterTemplates; @@ -58,6 +65,7 @@ class OperatorChangeDetectionST extends AbstractST { private static final Logger LOGGER = LoggerFactory.getLogger(OperatorChangeDetectionST.class); private static Kroxylicious kroxylicious; + private static CertManager certManager; private final String kafkaClusterName = "my-cluster"; private KroxyliciousOperator kroxyliciousOperator; @@ -101,7 +109,7 @@ void shouldUpdateDeploymentWhenVirtualKafkaClusterChanges(String namespace) { .withName("test-vkc").get(); // When - resourceManager.replaceResourceWithRetries(virtualKafkaCluster, vkc -> vkc.getSpec().getFilterRefs().add(filterRef)); + resourceManager.replaceResourceWithRetries(virtualKafkaCluster, vkc -> vkc.getSpec().setFilterRefs(List.of(filterRef))); LOGGER.info("virtual cluster edited"); // Then @@ -111,7 +119,7 @@ void shouldUpdateDeploymentWhenVirtualKafkaClusterChanges(String namespace) { @Test void shouldUpdateDeploymentWhenDownstreamTlsCertUpdated(String namespace) throws IOException { // Given - CertManager certManager = new CertManager(); + certManager = new CertManager(); certManager.deploy(); var issuer = certManager.issuer(namespace); @@ -227,12 +235,17 @@ void shouldUpdateDeploymentWhenSecretChanges(String namespace) { } private static void assertDeploymentUpdated(String namespace, KubeClient kubeClient, String originalChecksum) { + AtomicReference newChecksumFromAnnotation = new AtomicReference<>(); await().atMost(Duration.ofSeconds(90)).untilAsserted(() -> { Deployment proxyDeployment = kubeClient.getDeployment(namespace, Constants.KROXYLICIOUS_PROXY_SIMPLE_NAME); assertThat(proxyDeployment).isNotNull(); OperatorAssertions.assertThat(proxyDeployment.getSpec().getTemplate().getMetadata()).hasAnnotationSatisfying("kroxylicious.io/referent-checksum", - value -> assertThat(value).isNotEqualTo(originalChecksum)); + value -> { + newChecksumFromAnnotation.set(value); + assertThat(value).isNotEqualTo(originalChecksum); + }); }); + LOGGER.info("New checksum: {}", newChecksumFromAnnotation); } @BeforeEach @@ -240,6 +253,14 @@ void setUp(String namespace) { kroxylicious = new Kroxylicious(namespace); } + @AfterEach + void cleanAfterEach() { + if (certManager != null) { + certManager.delete(); + } + KubeResourceManager.get().deleteResources(true); + } + @AfterAll void cleanUp() { if (kroxyliciousOperator != null) { From f6f49a9cd27bb32450d8cfb1b48c4cc0570329b7 Mon Sep 17 00:00:00 2001 From: Francisco Vila Date: Wed, 28 May 2025 14:03:25 +0200 Subject: [PATCH 29/83] revert some changes Signed-off-by: Francisco Vila Signed-off-by: Ujjwal --- .../systemtests/installation/kroxylicious/CertManager.java | 2 ++ .../io/kroxylicious/systemtests/OperatorChangeDetectionST.java | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java index d5136f3118..4e3360c7e8 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java @@ -17,6 +17,7 @@ import io.fabric8.certmanager.api.model.v1.IssuerBuilder; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.client.dsl.NamespaceListVisitFromServerGetDeleteRecreateWaitApplicable; +import io.skodjob.testframe.resources.KubeResourceManager; import io.kroxylicious.systemtests.Constants; import io.kroxylicious.systemtests.utils.DeploymentUtils; @@ -101,6 +102,7 @@ public void deploy() { if (isDeployed()) { LOGGER.warn("Skipping cert manager deployment. It is already deployed!"); deleteCertManager = false; + var list = kubeClient().getClient().apps().deployments().inAnyNamespace().withLabelSelector("app=cert-manager").list().getItems(); return; } deployment.create(); diff --git a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java index f53e59709b..bd6b611ea0 100644 --- a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java +++ b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java @@ -22,6 +22,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import io.fabric8.certmanager.api.model.v1.Certificate; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.PodTemplateSpecBuilder; @@ -193,7 +194,6 @@ void shouldUpdateDeploymentWhenKafkaProxyChanges(String namespace) { .editOrNewSpec() .withPodTemplate(new PodTemplateSpecBuilder().build()) .endSpec(); - // @formatter:on // When @@ -258,7 +258,6 @@ void cleanAfterEach() { if (certManager != null) { certManager.delete(); } - KubeResourceManager.get().deleteResources(true); } @AfterAll From 376ce541d2aebb1d4ccdb13dbf36ec6d165f76d6 Mon Sep 17 00:00:00 2001 From: Francisco Vila Date: Wed, 28 May 2025 14:05:48 +0200 Subject: [PATCH 30/83] remove unused imports Signed-off-by: Francisco Vila Signed-off-by: Ujjwal --- .../systemtests/installation/kroxylicious/CertManager.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java index 4e3360c7e8..d5136f3118 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java @@ -17,7 +17,6 @@ import io.fabric8.certmanager.api.model.v1.IssuerBuilder; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.client.dsl.NamespaceListVisitFromServerGetDeleteRecreateWaitApplicable; -import io.skodjob.testframe.resources.KubeResourceManager; import io.kroxylicious.systemtests.Constants; import io.kroxylicious.systemtests.utils.DeploymentUtils; @@ -102,7 +101,6 @@ public void deploy() { if (isDeployed()) { LOGGER.warn("Skipping cert manager deployment. It is already deployed!"); deleteCertManager = false; - var list = kubeClient().getClient().apps().deployments().inAnyNamespace().withLabelSelector("app=cert-manager").list().getItems(); return; } deployment.create(); From 3ebf256032d6ebb096759eb0c99f6919ef5017c4 Mon Sep 17 00:00:00 2001 From: Francisco Vila Date: Wed, 28 May 2025 14:54:59 +0200 Subject: [PATCH 31/83] =?UTF-8?q?sort=20imports=C3=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Francisco Vila Signed-off-by: Ujjwal --- .../systemtests/OperatorChangeDetectionST.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java index bd6b611ea0..34395991ba 100644 --- a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java +++ b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java @@ -10,7 +10,6 @@ import java.time.Duration; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.AfterAll; @@ -22,7 +21,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import io.fabric8.certmanager.api.model.v1.Certificate; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.PodTemplateSpecBuilder; @@ -30,9 +28,6 @@ import io.fabric8.kubernetes.api.model.SecretBuilder; import io.fabric8.kubernetes.api.model.apps.Deployment; import io.fabric8.kubernetes.client.utils.KubernetesResourceUtil; -import io.skodjob.testframe.resources.KubeResourceManager; -import io.strimzi.api.kafka.model.kafka.Kafka; -import io.strimzi.api.kafka.model.kafka.KafkaStatus; import io.kroxylicious.kubernetes.api.common.FilterRef; import io.kroxylicious.kubernetes.api.common.FilterRefBuilder; @@ -40,7 +35,6 @@ import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyBuilder; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngress; import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaCluster; -import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaClusterSpec; import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP; import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilter; import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilterBuilder; @@ -49,7 +43,6 @@ import io.kroxylicious.systemtests.installation.kroxylicious.Kroxylicious; import io.kroxylicious.systemtests.installation.kroxylicious.KroxyliciousOperator; import io.kroxylicious.systemtests.k8s.KubeClient; -import io.kroxylicious.systemtests.resources.kroxylicious.KroxyliciousResource; import io.kroxylicious.systemtests.resources.manager.ResourceManager; import io.kroxylicious.systemtests.templates.kroxylicious.KroxyliciousFilterTemplates; From bff131da867c57cd8eb1cbdd61b854f7afe6e9b6 Mon Sep 17 00:00:00 2001 From: Francisco Vila Date: Wed, 28 May 2025 15:37:00 +0200 Subject: [PATCH 32/83] adding cert-manager namespace to the logCollector Signed-off-by: Francisco Vila Signed-off-by: Ujjwal --- .../systemtests/installation/kroxylicious/CertManager.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java index d5136f3118..0a82a991ac 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java @@ -19,6 +19,7 @@ import io.fabric8.kubernetes.client.dsl.NamespaceListVisitFromServerGetDeleteRecreateWaitApplicable; import io.kroxylicious.systemtests.Constants; +import io.kroxylicious.systemtests.resources.manager.ResourceManager; import io.kroxylicious.systemtests.utils.DeploymentUtils; import io.kroxylicious.systemtests.utils.NamespaceUtils; @@ -105,6 +106,7 @@ public void deploy() { } deployment.create(); DeploymentUtils.waitForDeploymentReady(Constants.CERT_MANAGER_NAMESPACE, "cert-manager-webhook"); + NamespaceUtils.addNamespaceToSet(Constants.CERT_MANAGER_NAMESPACE, ResourceManager.getTestContext().getTestClass().get().getName()); } /** @@ -118,6 +120,6 @@ public void delete() { LOGGER.info("Deleting Cert Manager in {} namespace", Constants.CERT_MANAGER_NAMESPACE); deployment.withGracePeriod(0).delete(); DeploymentUtils.waitForDeploymentDeletion(Constants.CERT_MANAGER_NAMESPACE, "cert-manager-webhook"); - NamespaceUtils.deleteNamespaceWithWait(Constants.CERT_MANAGER_NAMESPACE); + NamespaceUtils.deleteNamespaceWithWaitAndRemoveFromSet(Constants.CERT_MANAGER_NAMESPACE, ResourceManager.getTestContext().getTestClass().get().getName()); } } From 57ba25ed1ff89520479b352d1d6567883a8cfd63 Mon Sep 17 00:00:00 2001 From: Francisco Vila Date: Wed, 28 May 2025 15:38:06 +0200 Subject: [PATCH 33/83] adding cert-manager namespace to the logCollector Signed-off-by: Francisco Vila Signed-off-by: Ujjwal --- .../systemtests/installation/kroxylicious/CertManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java index 0a82a991ac..f5681e5d17 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java @@ -106,7 +106,7 @@ public void deploy() { } deployment.create(); DeploymentUtils.waitForDeploymentReady(Constants.CERT_MANAGER_NAMESPACE, "cert-manager-webhook"); - NamespaceUtils.addNamespaceToSet(Constants.CERT_MANAGER_NAMESPACE, ResourceManager.getTestContext().getTestClass().get().getName()); + NamespaceUtils.addNamespaceToSet(Constants.CERT_MANAGER_NAMESPACE, ResourceManager.getTestContext().getRequiredTestClass().getName()); } /** @@ -120,6 +120,6 @@ public void delete() { LOGGER.info("Deleting Cert Manager in {} namespace", Constants.CERT_MANAGER_NAMESPACE); deployment.withGracePeriod(0).delete(); DeploymentUtils.waitForDeploymentDeletion(Constants.CERT_MANAGER_NAMESPACE, "cert-manager-webhook"); - NamespaceUtils.deleteNamespaceWithWaitAndRemoveFromSet(Constants.CERT_MANAGER_NAMESPACE, ResourceManager.getTestContext().getTestClass().get().getName()); + NamespaceUtils.deleteNamespaceWithWaitAndRemoveFromSet(Constants.CERT_MANAGER_NAMESPACE, ResourceManager.getTestContext().getRequiredTestClass().getName()); } } From a351812651ed82fd4dda706576119fc20a86f8c1 Mon Sep 17 00:00:00 2001 From: Francisco Vila Date: Wed, 28 May 2025 16:37:40 +0200 Subject: [PATCH 34/83] move cleaup afterEach to afterAll Signed-off-by: Francisco Vila Signed-off-by: Ujjwal --- .../systemtests/OperatorChangeDetectionST.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java index 34395991ba..b0517d6182 100644 --- a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java +++ b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java @@ -246,18 +246,14 @@ void setUp(String namespace) { kroxylicious = new Kroxylicious(namespace); } - @AfterEach - void cleanAfterEach() { - if (certManager != null) { - certManager.delete(); - } - } - @AfterAll void cleanUp() { if (kroxyliciousOperator != null) { kroxyliciousOperator.delete(); } + if (certManager != null) { + certManager.delete(); + } } @BeforeAll From 85bcd1244e793174db3ffaed129729940724907f Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Thu, 29 May 2025 14:33:06 +1200 Subject: [PATCH 35/83] Fix formatting Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../io/kroxylicious/systemtests/OperatorChangeDetectionST.java | 1 - 1 file changed, 1 deletion(-) diff --git a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java index b0517d6182..68584bb9d4 100644 --- a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java +++ b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java @@ -13,7 +13,6 @@ import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; From a70a8a102ff0878beda76fa6b87ff961c2c83302 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Thu, 29 May 2025 14:55:32 +1200 Subject: [PATCH 36/83] Cope with optional lists Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../systemtests/OperatorChangeDetectionST.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java index 68584bb9d4..e6b4e07b62 100644 --- a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java +++ b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java @@ -8,8 +8,10 @@ import java.io.IOException; import java.time.Duration; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.AfterAll; @@ -102,7 +104,12 @@ void shouldUpdateDeploymentWhenVirtualKafkaClusterChanges(String namespace) { .withName("test-vkc").get(); // When - resourceManager.replaceResourceWithRetries(virtualKafkaCluster, vkc -> vkc.getSpec().setFilterRefs(List.of(filterRef))); + resourceManager.replaceResourceWithRetries(virtualKafkaCluster, vkc -> { + var filterRefs = Optional.ofNullable(vkc.getSpec().getFilterRefs()).orElse(new ArrayList<>()); + filterRefs.add(filterRef); + vkc.getSpec().setFilterRefs(filterRefs); + + }); LOGGER.info("virtual cluster edited"); // Then @@ -128,7 +135,11 @@ void shouldUpdateDeploymentWhenDownstreamTlsCertUpdated(String namespace) throws // When resourceManager.replaceResourceWithRetries(cert.build(), - certToPatch -> certToPatch.getSpec().getDnsNames().add("test-vkc-cluster-ip.my-proxy.svc.cluster.local")); + certToPatch -> { + var dnsNames = Optional.ofNullable(certToPatch.getSpec().getDnsNames()).orElse(new ArrayList<>()); + dnsNames.add("test-vkc-cluster-ip.my-proxy.svc.cluster.local"); + certToPatch.getSpec().setDnsNames(dnsNames); + }); LOGGER.info("SAN added to downstream tls cert"); // Then From 25dae621f79955bd782704c085d925e6cef54510 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Tue, 27 May 2025 21:59:53 +1200 Subject: [PATCH 37/83] Cert type thing in the system test Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../io/kroxylicious/systemtests/OperatorChangeDetectionST.java | 1 + 1 file changed, 1 insertion(+) diff --git a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java index e6b4e07b62..74aba199b4 100644 --- a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java +++ b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java @@ -22,6 +22,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import io.fabric8.certmanager.api.model.v1.CertificateBuilder; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.PodTemplateSpecBuilder; From ccb2396023c242de5de3c92c1c39024ef26c402e Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Wed, 28 May 2025 17:22:40 +1200 Subject: [PATCH 38/83] Updating the cert causes the podTemplate updated. Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../kroxylicious/KroxyliciousKafkaProxyIngressTemplates.java | 1 + .../io/kroxylicious/systemtests/OperatorChangeDetectionST.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaProxyIngressTemplates.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaProxyIngressTemplates.java index 49d2e93ab5..6def8a51fc 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaProxyIngressTemplates.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaProxyIngressTemplates.java @@ -7,6 +7,7 @@ package io.kroxylicious.systemtests.templates.kroxylicious; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngressBuilder; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP; import static io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP.Protocol.TCP; import static io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP.Protocol.TLS; diff --git a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java index 74aba199b4..f0a3913d52 100644 --- a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java +++ b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java @@ -22,7 +22,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import io.fabric8.certmanager.api.model.v1.CertificateBuilder; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.PodTemplateSpecBuilder; @@ -33,11 +32,13 @@ import io.kroxylicious.kubernetes.api.common.FilterRef; import io.kroxylicious.kubernetes.api.common.FilterRefBuilder; +import io.kroxylicious.kubernetes.api.common.TrustAnchorRef; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxy; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyBuilder; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngress; import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaCluster; import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaservicespec.TlsBuilder; import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilter; import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilterBuilder; import io.kroxylicious.kubernetes.operator.assertj.OperatorAssertions; From 55ee77614c054cccb0a0fcfb3b8f5a826094eff2 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Wed, 28 May 2025 20:27:50 +1200 Subject: [PATCH 39/83] Formatting! Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../kroxylicious/KroxyliciousKafkaProxyIngressTemplates.java | 1 - .../io/kroxylicious/systemtests/OperatorChangeDetectionST.java | 2 -- 2 files changed, 3 deletions(-) diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaProxyIngressTemplates.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaProxyIngressTemplates.java index 6def8a51fc..49d2e93ab5 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaProxyIngressTemplates.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaProxyIngressTemplates.java @@ -7,7 +7,6 @@ package io.kroxylicious.systemtests.templates.kroxylicious; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngressBuilder; -import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP; import static io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP.Protocol.TCP; import static io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP.Protocol.TLS; diff --git a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java index f0a3913d52..e6b4e07b62 100644 --- a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java +++ b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java @@ -32,13 +32,11 @@ import io.kroxylicious.kubernetes.api.common.FilterRef; import io.kroxylicious.kubernetes.api.common.FilterRefBuilder; -import io.kroxylicious.kubernetes.api.common.TrustAnchorRef; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxy; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyBuilder; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngress; import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaCluster; import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP; -import io.kroxylicious.kubernetes.api.v1alpha1.kafkaservicespec.TlsBuilder; import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilter; import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilterBuilder; import io.kroxylicious.kubernetes.operator.assertj.OperatorAssertions; From 0cff548874b23881c53354ee4f3a5f3a111aa949 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Thu, 29 May 2025 10:14:13 +1200 Subject: [PATCH 40/83] downstream only TLS Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../systemtests/installation/kroxylicious/Kroxylicious.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/Kroxylicious.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/Kroxylicious.java index 92ed1c4fc3..6438a0d9da 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/Kroxylicious.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/Kroxylicious.java @@ -111,7 +111,7 @@ public void deployPortIdentifiesNodeWithDownstreamTlsAndNoFilters(String cluster KroxyliciousKafkaProxyTemplates.defaultKafkaProxyCR(deploymentNamespace, Constants.KROXYLICIOUS_PROXY_SIMPLE_NAME), KroxyliciousKafkaProxyIngressTemplates.tlsKafkaProxyIngressCR(deploymentNamespace, Constants.KROXYLICIOUS_INGRESS_CLUSTER_IP, Constants.KROXYLICIOUS_PROXY_SIMPLE_NAME), - KroxyliciousKafkaClusterRefTemplates.kafkaClusterRefCRWithTls(deploymentNamespace, clusterName, tls), + KroxyliciousKafkaClusterRefTemplates.defaultKafkaClusterRefCR(deploymentNamespace, clusterName), KroxyliciousVirtualKafkaClusterTemplates.defaultVirtualKafkaClusterWithTlsCR(deploymentNamespace, clusterName, Constants.KROXYLICIOUS_PROXY_SIMPLE_NAME, clusterName, Constants.KROXYLICIOUS_INGRESS_CLUSTER_IP, tls)); } From f27206404f8fcdc427e2cda7ba594754998ed3c0 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Fri, 30 May 2025 11:15:55 +1200 Subject: [PATCH 41/83] Include downstream TLS certs directlu Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../VirtualKafkaClusterReconciler.java | 21 ++++- .../VirtualKafkaClusterReconcilerTest.java | 83 ++++++++++++++----- 2 files changed, 82 insertions(+), 22 deletions(-) diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java index fb8e2f40a4..0f7fa567da 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java @@ -61,6 +61,7 @@ import io.kroxylicious.kubernetes.operator.resolver.DependencyResolver; import io.kroxylicious.proxy.tag.VisibleForTesting; +import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import static io.fabric8.kubernetes.api.model.HasMetadata.getKind; @@ -113,8 +114,13 @@ public UpdateControl reconcile(VirtualKafkaCluster cluster, VirtualKafkaCluster updatedCluster = checkClusterIngressTlsSettings(cluster, context); if (updatedCluster == null) { updatedCluster = maybeCombineStatusWithClusterConfigMap(cluster, context); - Crc32ChecksumGenerator checksumGenerator = new Crc32ChecksumGenerator(); + MetadataChecksumGenerator checksumGenerator = context.managedWorkflowAndDependentResourceContext() + .get(MetadataChecksumGenerator.CHECKSUM_CONTEXT_KEY, MetadataChecksumGenerator.class) + .orElse(new Crc32ChecksumGenerator()); clusterResolutionResult.allResolvedReferents().forEach(checksumGenerator::appendMetadata); + + appendSecretsFromCertificateRefs(context, cluster, checksumGenerator); + KubernetesResourceUtil.getOrCreateAnnotations(updatedCluster) .put(MetadataChecksumGenerator.REFERENT_CHECKSUM_ANNOTATION, checksumGenerator.encode()); @@ -133,6 +139,19 @@ public UpdateControl reconcile(VirtualKafkaCluster cluster, return reconciliationResult; } + private static void appendSecretsFromCertificateRefs(Context context, VirtualKafkaCluster updatedCluster, + @NonNull MetadataChecksumGenerator checksumGenerator) { + LOGGER.debug("Including secrets from ingress TLS in checksum"); + updatedCluster.getSpec().getIngresses().stream() + .map(Ingresses::getTls) + .filter(Objects::nonNull) + .map(Tls::getCertificateRef) + .flatMap(certificateRef -> context.getSecondaryResourcesAsStream(Secret.class) + .filter(secret -> secret.getMetadata().getName().equals(certificateRef.getName()))) + + .forEach(checksumGenerator::appendMetadata); + } + /** * Checks the validity of the cluster's ingress TLS settings. Specifically it checks whether * the TLS specification given by the KafkaProxyIngress matches what has been provided in diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java index 9fbdb147cd..9341beb8e8 100644 --- a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java @@ -15,6 +15,7 @@ import java.util.Set; import java.util.UUID; import java.util.function.Consumer; +import java.util.stream.Stream; import org.assertj.core.api.InstanceOfAssertFactories; import org.junit.jupiter.api.BeforeEach; @@ -22,6 +23,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; import io.fabric8.kubernetes.api.model.ConfigMap; import io.fabric8.kubernetes.api.model.ConfigMapBuilder; @@ -39,6 +41,7 @@ import io.javaoperatorsdk.operator.api.reconciler.Context; import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; import io.javaoperatorsdk.operator.api.reconciler.UpdateControl; +import io.javaoperatorsdk.operator.api.reconciler.dependent.managed.ManagedWorkflowAndDependentResourceContext; import io.javaoperatorsdk.operator.processing.event.ResourceID; import io.javaoperatorsdk.operator.processing.event.source.SecondaryToPrimaryMapper; @@ -62,6 +65,7 @@ import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilterBuilder; import io.kroxylicious.kubernetes.operator.assertj.ConditionListAssert; import io.kroxylicious.kubernetes.operator.assertj.VirtualKafkaClusterStatusAssert; +import io.kroxylicious.kubernetes.operator.checksum.MetadataChecksumGenerator; import io.kroxylicious.kubernetes.operator.resolver.DependencyResolver; import edu.umd.cs.findbugs.annotations.NonNull; @@ -70,6 +74,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; class VirtualKafkaClusterReconcilerTest { @@ -242,21 +247,23 @@ class VirtualKafkaClusterReconcilerTest { .endMetadata() .addToData("ca-bundle.pem", "value") .build(); - // @formatter:on - + @Mock + private static ManagedWorkflowAndDependentResourceContext workflowContext; private VirtualKafkaClusterReconciler virtualKafkaClusterReconciler; @BeforeEach void setUp() { virtualKafkaClusterReconciler = new VirtualKafkaClusterReconciler(Clock.systemUTC(), DependencyResolver.create()); + workflowContext = mock(ManagedWorkflowAndDependentResourceContext.class); } static List shouldSetResolvedRefs() { List result = new ArrayList<>(); { - Context> context = mock(); + Context> context = mockContext(); + when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); @@ -271,7 +278,7 @@ static List shouldSetResolvedRefs() { } { - Context> context = mock(); + Context> context = mockContext(); when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); @@ -289,7 +296,7 @@ static List shouldSetResolvedRefs() { { - Context> context = mock(); + Context> context = mockContext(); when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); @@ -308,7 +315,7 @@ static List shouldSetResolvedRefs() { } { - Context> context = mock(); + Context> context = mockContext(); when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); result.add(Arguments.argumentSet("proxy not found", @@ -322,7 +329,7 @@ static List shouldSetResolvedRefs() { } { - Context> context = mock(); + Context> context = mockContext(); when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); result.add(Arguments.argumentSet("service not found", @@ -336,7 +343,7 @@ static List shouldSetResolvedRefs() { } { - Context> context = mock(); + Context> context = mockContext(); when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn( Set.of(INGRESS.edit().editSpec().withNewProxyRef().withName("not-my-proxy").endProxyRef().endSpec().build())); @@ -352,7 +359,7 @@ static List shouldSetResolvedRefs() { } { - Context> context = mock(); + Context> context = mockContext(); when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(new KafkaServiceBuilder(SERVICE).withNewStatus().addNewCondition() @@ -374,7 +381,7 @@ static List shouldSetResolvedRefs() { } { - Context> context = mock(); + Context> context = mockContext(); when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); result.add(Arguments.argumentSet("ingress not found", @@ -388,7 +395,7 @@ static List shouldSetResolvedRefs() { } { - Context> context = mock(); + Context> context = mockContext(); when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn( @@ -411,7 +418,7 @@ static List shouldSetResolvedRefs() { } { - Context> context = mock(); + Context> context = mockContext(); when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); @@ -426,7 +433,7 @@ static List shouldSetResolvedRefs() { } { - Context> context = mock(); + Context> context = mockContext(); when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); @@ -451,7 +458,7 @@ static List shouldSetResolvedRefs() { } { - Context> context = mock(); + Context> context = mockContext(); mockGetSecret(context, Optional.of(KUBE_TLS_CERT_SECRET)); when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); @@ -467,7 +474,7 @@ static List shouldSetResolvedRefs() { } { - Context> context = mock(); + Context> context = mockContext(); mockGetSecret(context, Optional.empty()); when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); @@ -484,7 +491,7 @@ static List shouldSetResolvedRefs() { } { - Context> context = mock(); + Context> context = mockContext(); mockGetSecret(context, Optional.of(NON_KUBE_TLS_CERT_SECRET)); when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); @@ -501,7 +508,7 @@ static List shouldSetResolvedRefs() { } { - Context> context = mock(); + Context> context = mockContext(); mockGetSecret(context, Optional.empty()); when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); @@ -518,7 +525,7 @@ static List shouldSetResolvedRefs() { } { - Context> context = mock(); + Context> context = mockContext(); mockGetSecret(context, Optional.of(KUBE_TLS_CERT_SECRET)); mockGetConfigMap(context, Optional.of(PEM_CONFIG_MAP)); when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); @@ -535,7 +542,7 @@ static List shouldSetResolvedRefs() { } { - Context> context = mock(); + Context> context = mockContext(); mockGetSecret(context, Optional.of(KUBE_TLS_CERT_SECRET)); mockGetConfigMap(context, Optional.empty()); @@ -553,7 +560,7 @@ static List shouldSetResolvedRefs() { } { - Context> context = mock(); + Context> context = mockContext(); when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); @@ -569,7 +576,7 @@ static List shouldSetResolvedRefs() { } { - Context> context = mock(); + Context> context = mockContext(); when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); @@ -587,6 +594,13 @@ static List shouldSetResolvedRefs() { return result; } + @NonNull + private static Context> mockContext() { + Context> context = mock(); + when(context.managedWorkflowAndDependentResourceContext()).thenReturn(workflowContext); + return context; + } + @NonNull private static ConfigMap buildProxyConfigMapWithPatch(VirtualKafkaCluster clusterOneFilter) { // @formatter:off @@ -645,6 +659,8 @@ void shouldSetIngressStatusForClusterIPIngress() { var reconciler = new VirtualKafkaClusterReconciler(TEST_CLOCK, DependencyResolver.create()); Context context = mock(); + ManagedWorkflowAndDependentResourceContext workflowContext = mock(ManagedWorkflowAndDependentResourceContext.class); + when(context.managedWorkflowAndDependentResourceContext()).thenReturn(workflowContext); when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); when(context.getSecondaryResource(ConfigMap.class)).thenReturn(Optional.of(buildProxyConfigMapWithPatch(CLUSTER_NO_FILTERS))); @@ -679,6 +695,8 @@ void shouldOmitIngressIfKubernetesServiceNotPresent() { var reconciler = new VirtualKafkaClusterReconciler(TEST_CLOCK, DependencyResolver.create()); Context context = mock(); + ManagedWorkflowAndDependentResourceContext workflowContext = mock(ManagedWorkflowAndDependentResourceContext.class); + when(context.managedWorkflowAndDependentResourceContext()).thenReturn(workflowContext); when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); when(context.getSecondaryResource(ConfigMap.class)).thenReturn(Optional.of(buildProxyConfigMapWithPatch(CLUSTER_NO_FILTERS))); @@ -981,6 +999,29 @@ void kafkaServiceSecondaryToPrimaryMapperIgnoresServiceWithStaleStatus() { assertThat(primaryResourceIDs).isEmpty(); } + @Test + void shouldIncludeDownstreamTlsSecretInChecksum() { + // Given + Context context = mock(); + MetadataChecksumGenerator checksumGenerator = mock(MetadataChecksumGenerator.class); + when(workflowContext.get(MetadataChecksumGenerator.CHECKSUM_CONTEXT_KEY, MetadataChecksumGenerator.class)).thenReturn(Optional.of(checksumGenerator)); + when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(context.getSecondaryResource(ConfigMap.class)).thenReturn(Optional.of(buildProxyConfigMapWithPatch(CLUSTER_TLS_NO_FILTERS))); + when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); + when(context.getSecondaryResources(KafkaProtocolFilter.class)).thenReturn(Set.of()); + when(context.getSecondaryResources(Service.class)).thenReturn(Set.of(KUBERNETES_INGRESS_SERVICES)); + when(context.getSecondaryResource(Secret.class, "secrets")).thenReturn(Optional.of(KUBE_TLS_CERT_SECRET)); + when(context.getSecondaryResourcesAsStream(Secret.class)).thenReturn(Stream.of(KUBE_TLS_CERT_SECRET)); + when(context.managedWorkflowAndDependentResourceContext()).thenReturn(workflowContext); + + // When + virtualKafkaClusterReconciler.reconcile(CLUSTER_TLS_NO_FILTERS, context); + + // Then + verify(checksumGenerator).appendMetadata(KUBE_TLS_CERT_SECRET); + } + private static EventSourceContext mockContextContaining(VirtualKafkaCluster cluster) { EventSourceContext eventSourceContext = mock(); KubernetesClient client = mock(); From 11df083609eee35496f5641fe1e52edfd6f4eff5 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Fri, 30 May 2025 11:18:54 +1200 Subject: [PATCH 42/83] Warning cleanup Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../VirtualKafkaClusterReconciler.java | 15 +++--- .../VirtualKafkaClusterReconcilerTest.java | 51 ++++++++----------- 2 files changed, 26 insertions(+), 40 deletions(-) diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java index 0f7fa567da..60d3cedca4 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java @@ -418,15 +418,12 @@ public List> prepareEventSources(EventSource Service.class, VirtualKafkaCluster.class) .withName(KUBERNETES_SERVICES_EVENT_SOURCE_NAME) - .withPrimaryToSecondaryMapper((VirtualKafkaCluster cluster) -> { - var name = cluster.getMetadata().getName(); - return cluster.getSpec().getIngresses() - .stream() - .map(Ingresses::getIngressRef) - .flatMap(ir -> ResourcesUtil - .localRefAsResourceId(cluster, new AnyLocalRefBuilder().withName(bootstrapServiceName(cluster, ir.getName())).build()).stream()) - .collect(Collectors.toSet()); - }) + .withPrimaryToSecondaryMapper((VirtualKafkaCluster cluster) -> cluster.getSpec().getIngresses() + .stream() + .map(Ingresses::getIngressRef) + .flatMap(ir -> ResourcesUtil + .localRefAsResourceId(cluster, new AnyLocalRefBuilder().withName(bootstrapServiceName(cluster, ir.getName())).build()).stream()) + .collect(Collectors.toSet())) .withSecondaryToPrimaryMapper(kubenetesService -> Optional.of(kubenetesService) .flatMap(service -> extractOwnerRefFromKubernetesService(service, VIRTUAL_KAFKA_CLUSTER_KIND)) .map(ownerRef -> new ResourceID(ownerRef.getName(), kubenetesService.getMetadata().getNamespace())) diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java index 9341beb8e8..abd72ec95b 100644 --- a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java @@ -27,7 +27,6 @@ import io.fabric8.kubernetes.api.model.ConfigMap; import io.fabric8.kubernetes.api.model.ConfigMapBuilder; -import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.api.model.KubernetesResourceList; import io.fabric8.kubernetes.api.model.Secret; import io.fabric8.kubernetes.api.model.SecretBuilder; @@ -270,11 +269,9 @@ static List shouldSetResolvedRefs() { result.add(Arguments.argumentSet("no filter", CLUSTER_NO_FILTERS, context, - (Consumer) conditionList -> { - conditionList - .singleElement() - .isResolvedRefsTrue(); - })); + (Consumer) conditionList -> conditionList + .singleElement() + .isResolvedRefsTrue())); } { @@ -287,11 +284,9 @@ static List shouldSetResolvedRefs() { result.add(Arguments.argumentSet("one filter", CLUSTER_ONE_FILTER, context, - (Consumer) conditionList -> { - conditionList - .singleElement() - .isResolvedRefsTrue(); - })); + (Consumer) conditionList -> conditionList + .singleElement() + .isResolvedRefsTrue())); } { @@ -307,11 +302,9 @@ static List shouldSetResolvedRefs() { new VirtualKafkaClusterBuilder(CLUSTER_ONE_FILTER).editOrNewStatus().withObservedGeneration(ResourcesUtil.generation(CLUSTER_NO_FILTERS)) .endStatus().build(), context, - (Consumer) conditionList -> { - conditionList - .singleElement() - .isResolvedRefsTrue(); - })); + (Consumer) conditionList -> conditionList + .singleElement() + .isResolvedRefsTrue())); } { @@ -466,11 +459,9 @@ static List shouldSetResolvedRefs() { result.add(Arguments.argumentSet("cluster with tls", CLUSTER_TLS_NO_FILTERS, context, - (Consumer) conditionList -> { - conditionList - .singleElement() - .isResolvedRefsTrue(); - })); + (Consumer) conditionList -> conditionList + .singleElement() + .isResolvedRefsTrue())); } { @@ -534,11 +525,9 @@ static List shouldSetResolvedRefs() { result.add(Arguments.argumentSet("cluster with tls with trust anchor", CLUSTER_TLS_NO_FILTERS_WITH_TRUST_ANCHOR, context, - (Consumer) conditionList -> { - conditionList - .singleElement() - .isResolvedRefsTrue(); - })); + (Consumer) conditionList -> conditionList + .singleElement() + .isResolvedRefsTrue())); } { @@ -1026,15 +1015,15 @@ private static EventSourceContext mockContextContaining(Vir EventSourceContext eventSourceContext = mock(); KubernetesClient client = mock(); when(eventSourceContext.getClient()).thenReturn(client); - KubernetesResourceList mockList = mockListOperation(client, VirtualKafkaCluster.class); + KubernetesResourceList mockList = mockListVirtualClustersOperation(client); when(mockList.getItems()).thenReturn(List.of(cluster)); return eventSourceContext; } - private static KubernetesResourceList mockListOperation(KubernetesClient client, Class clazz) { - MixedOperation, Resource> mockOperation = mock(); - when(client.resources(clazz)).thenReturn(mockOperation); - KubernetesResourceList mockList = mock(); + private static KubernetesResourceList mockListVirtualClustersOperation(KubernetesClient client) { + MixedOperation, Resource> mockOperation = mock(); + when(client.resources(VirtualKafkaCluster.class)).thenReturn(mockOperation); + KubernetesResourceList mockList = mock(); when(mockOperation.list()).thenReturn(mockList); when(mockOperation.inNamespace(any())).thenReturn(mockOperation); return mockList; From 64e3935c8c142d1504b3e5d9101e1f1bbd584f12 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Fri, 30 May 2025 12:03:00 +1200 Subject: [PATCH 43/83] use field instead of local var Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../kubernetes/operator/VirtualKafkaClusterReconcilerTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java index abd72ec95b..d73c665823 100644 --- a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java @@ -648,7 +648,6 @@ void shouldSetIngressStatusForClusterIPIngress() { var reconciler = new VirtualKafkaClusterReconciler(TEST_CLOCK, DependencyResolver.create()); Context context = mock(); - ManagedWorkflowAndDependentResourceContext workflowContext = mock(ManagedWorkflowAndDependentResourceContext.class); when(context.managedWorkflowAndDependentResourceContext()).thenReturn(workflowContext); when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); @@ -684,7 +683,6 @@ void shouldOmitIngressIfKubernetesServiceNotPresent() { var reconciler = new VirtualKafkaClusterReconciler(TEST_CLOCK, DependencyResolver.create()); Context context = mock(); - ManagedWorkflowAndDependentResourceContext workflowContext = mock(ManagedWorkflowAndDependentResourceContext.class); when(context.managedWorkflowAndDependentResourceContext()).thenReturn(workflowContext); when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); From 11316826f129f11a67c547a29113412c4b9ec0e6 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Fri, 30 May 2025 12:04:32 +1200 Subject: [PATCH 44/83] Failing system test for downstream trust anchor Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../kroxylicious/Kroxylicious.java | 46 ++++++++++++------- ...xyliciousVirtualKafkaClusterTemplates.java | 1 + .../OperatorChangeDetectionST.java | 32 +++++++++++++ 3 files changed, 62 insertions(+), 17 deletions(-) diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/Kroxylicious.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/Kroxylicious.java index 6438a0d9da..813fa1c9d7 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/Kroxylicious.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/Kroxylicious.java @@ -16,7 +16,7 @@ import io.strimzi.api.kafka.model.kafka.listener.ListenerStatus; import io.kroxylicious.kms.service.TestKmsFacade; -import io.kroxylicious.kubernetes.api.common.CertificateRefBuilder; +import io.kroxylicious.kubernetes.api.common.TrustAnchorRef; import io.kroxylicious.kubernetes.api.common.TrustAnchorRefBuilder; import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaCluster; import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaClusterStatus; @@ -34,6 +34,8 @@ import io.kroxylicious.systemtests.templates.kroxylicious.KroxyliciousVirtualKafkaClusterTemplates; import io.kroxylicious.systemtests.utils.KafkaUtils; +import edu.umd.cs.findbugs.annotations.NonNull; + import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient; import static org.awaitility.Awaitility.await; @@ -129,27 +131,37 @@ public Tls createCertificateConfigMapFromListener(String namespace) { //@formatter:off return new TlsBuilder() .withTrustAnchorRef( - new TrustAnchorRefBuilder() - .withNewRef() - .withName(Constants.KROXYLICIOUS_TLS_CLIENT_CA_CERT) - .withKind(Constants.CONFIG_MAP) - .endRef() - .withKey(Constants.KROXYLICIOUS_TLS_CA_NAME) - .build()) + buildTrustAnchorRef()) .build(); //@formatter:on } - public Tls tlsConfigFromCert(String certNane) { - //@formatter:off - return new TlsBuilder() - .withCertificateRef( - new CertificateRefBuilder() - .withName(certNane) - .withKind("Secret") - .build()) + @NonNull + private static TrustAnchorRef buildTrustAnchorRef() { + //formatter:off + return new TrustAnchorRefBuilder() + .withNewRef() + .withName(Constants.KROXYLICIOUS_TLS_CLIENT_CA_CERT) + .withKind(Constants.CONFIG_MAP) + .endRef() + .withKey(Constants.KROXYLICIOUS_TLS_CA_NAME) .build(); - //@formatter:on + //formatter:on + } + + public Tls tlsConfigFromCert(String certNane) { + TlsBuilder tlsBuilder = new TlsBuilder(); + if (certNane != null) { + //formatter:off + tlsBuilder + .withNewCertificateRef() + .withName(certNane) + .withKind("Secret") + .endCertificateRef(); + //formatter:on + } + tlsBuilder.withTrustAnchorRef(buildTrustAnchorRef()); + return tlsBuilder.build(); } /** diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousVirtualKafkaClusterTemplates.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousVirtualKafkaClusterTemplates.java index 731f1ec923..59a02902bf 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousVirtualKafkaClusterTemplates.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousVirtualKafkaClusterTemplates.java @@ -79,6 +79,7 @@ public static VirtualKafkaClusterBuilder defaultVirtualKafkaClusterWithTlsCR(Str .editIngress(0) .withNewTls() .withCertificateRef(tls.getCertificateRef()) + .withTrustAnchorRef(tls.getTrustAnchorRef()) .endTls() .endIngress() .endSpec(); diff --git a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java index e6b4e07b62..d1fa74b118 100644 --- a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java +++ b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java @@ -22,6 +22,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import io.fabric8.kubernetes.api.model.ConfigMap; +import io.fabric8.kubernetes.api.model.ConfigMapBuilder; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.PodTemplateSpecBuilder; @@ -146,6 +148,36 @@ void shouldUpdateDeploymentWhenDownstreamTlsCertUpdated(String namespace) throws assertDeploymentUpdated(namespace, kubeClient, originalChecksum); } + @Test + void shouldUpdateDeploymentWhenDownstreamTrustUpdated(String namespace) throws IOException { + // Given + certManager = new CertManager(); + certManager.deploy(); + + var issuer = certManager.issuer(namespace); + var cert = certManager.certFor(namespace, "my-cluster-cluster-ip." + namespace + ".svc.cluster.local"); + + resourceManager.createOrUpdateResourceWithWait(issuer, cert); + + var tls = kroxylicious.tlsConfigFromCert("server-certificate"); + + kroxylicious.deployPortIdentifiesNodeWithDownstreamTlsAndNoFilters("test-vkc", tls); + KubeClient kubeClient = kubeClient(namespace); + + String originalChecksum = getInitialChecksum(namespace, kubeClient); + ConfigMap trustAnchorConfig = new ConfigMapBuilder().withNewMetadata().withName(Constants.KROXYLICIOUS_TLS_CLIENT_CA_CERT).withNamespace(namespace).endMetadata() + .build(); + // When + resourceManager.replaceResourceWithRetries(trustAnchorConfig, + trustConfigMap -> { + trustAnchorConfig.setData(Map.of(Constants.KROXYLICIOUS_TLS_CA_NAME, "server-certificate")); + }); + LOGGER.info("Downstream trust updated"); + + // Then + assertDeploymentUpdated(namespace, kubeClient, originalChecksum); + } + @Test void shouldUpdateWhenFilterConfigurationChanges(String namespace) { // Given From 41b0376fef367ed99b5c6c7087f597e98193659f Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Fri, 30 May 2025 22:17:32 +1200 Subject: [PATCH 45/83] Add the ability to generate a slug from just metadata Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../kubernetes/operator/ResourcesUtil.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ResourcesUtil.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ResourcesUtil.java index 7f9bf46e5e..4a31af0c3d 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ResourcesUtil.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ResourcesUtil.java @@ -304,6 +304,10 @@ public static String namespacedSlug(LocalRef ref, HasMetadata resource) { return slug(ref) + " in namespace '" + namespace(resource) + "'"; } + public static String namespacedSlug(HasMetadata resource) { + return slug(resource) + " in namespace '" + namespace(resource) + "'"; + } + private static String slug(LocalRef ref) { String group = ref.getGroup(); String name = ref.getName(); @@ -311,6 +315,13 @@ private static String slug(LocalRef ref) { return ref.getKind().toLowerCase(Locale.ROOT) + groupString + "/" + name; } + private static String slug(HasMetadata ref) { + String group = group(ref); + String name = name(ref); + String groupString = group.isEmpty() ? "" : "." + group; + return ref.getKind().toLowerCase(Locale.ROOT) + groupString + "/" + name; + } + /** * Checks that the status observedGeneration is equal to the metadata generation. Indicating * that the current {@code spec} of the resource has been reconciled. From 396c894378cbad80f8cb1bd6ebc4455c0c09af22 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Fri, 30 May 2025 22:17:52 +1200 Subject: [PATCH 46/83] Only log creation if it's happening. Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../systemtests/installation/kroxylicious/CertManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java index f5681e5d17..cf0b740884 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/CertManager.java @@ -98,12 +98,12 @@ private boolean isDeployed() { * Deploy cert manager. */ public void deploy() { - LOGGER.info("Deploy cert manager in {} namespace", Constants.CERT_MANAGER_NAMESPACE); if (isDeployed()) { LOGGER.warn("Skipping cert manager deployment. It is already deployed!"); deleteCertManager = false; return; } + LOGGER.info("Deploy cert manager in {} namespace", Constants.CERT_MANAGER_NAMESPACE); deployment.create(); DeploymentUtils.waitForDeploymentReady(Constants.CERT_MANAGER_NAMESPACE, "cert-manager-webhook"); NamespaceUtils.addNamespaceToSet(Constants.CERT_MANAGER_NAMESPACE, ResourceManager.getTestContext().getRequiredTestClass().getName()); From 89875e2671d7dc5a1ce9e9fee02fc25e1f2a088f Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Fri, 30 May 2025 22:18:33 +1200 Subject: [PATCH 47/83] Debug logging. Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../operator/checksum/MetadataChecksumGenerator.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/checksum/MetadataChecksumGenerator.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/checksum/MetadataChecksumGenerator.java index 158f903c9a..b9079bd42d 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/checksum/MetadataChecksumGenerator.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/checksum/MetadataChecksumGenerator.java @@ -9,14 +9,19 @@ import java.util.Map; import java.util.Objects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.client.utils.KubernetesResourceUtil; +import io.kroxylicious.kubernetes.operator.ResourcesUtil; + import edu.umd.cs.findbugs.annotations.Nullable; public interface MetadataChecksumGenerator { - + Logger LOGGER = LoggerFactory.getLogger(MetadataChecksumGenerator.class); String REFERENT_CHECKSUM_ANNOTATION = "kroxylicious.io/referent-checksum"; String CHECKSUM_CONTEXT_KEY = "kroxylicious.io/referent-checksum-generator"; String NO_CHECKSUM_SPECIFIED = ""; @@ -31,6 +36,7 @@ default void appendMetadata(ObjectMeta objectMeta) { } default void appendMetadata(HasMetadata entity) { + LOGGER.debug("appendMetadata for: {}", ResourcesUtil.namespacedSlug(entity)); appendMetadata(entity.getMetadata()); } From 353bcc47c4ab8d3beebf9afcadba8c65126f855c Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Fri, 30 May 2025 22:19:45 +1200 Subject: [PATCH 48/83] Tactical fix to include TLS config in VKC reconcilation. Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../VirtualKafkaClusterReconciler.java | 18 ++++++++++--- .../VirtualKafkaClusterReconcilerTest.java | 25 +++++++++++++++++++ .../kroxylicious/Kroxylicious.java | 17 +++++++------ .../resources/manager/ResourceManager.java | 6 +++-- .../KroxyliciousConfigMapTemplates.java | 19 ++++++++++++++ .../OperatorChangeDetectionST.java | 5 ++-- 6 files changed, 74 insertions(+), 16 deletions(-) diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java index 60d3cedca4..378d702f26 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java @@ -141,14 +141,26 @@ public UpdateControl reconcile(VirtualKafkaCluster cluster, private static void appendSecretsFromCertificateRefs(Context context, VirtualKafkaCluster updatedCluster, @NonNull MetadataChecksumGenerator checksumGenerator) { - LOGGER.debug("Including secrets from ingress TLS in checksum"); - updatedCluster.getSpec().getIngresses().stream() + LOGGER.info("Including secrets from ingress TLS in checksum"); + updatedCluster.getSpec() + .getIngresses() + .stream() .map(Ingresses::getTls) .filter(Objects::nonNull) .map(Tls::getCertificateRef) .flatMap(certificateRef -> context.getSecondaryResourcesAsStream(Secret.class) - .filter(secret -> secret.getMetadata().getName().equals(certificateRef.getName()))) + .filter(secret -> KubernetesResourceUtil.getName(secret).equals(certificateRef.getName()))) + .forEach(checksumGenerator::appendMetadata); + LOGGER.info("Including TrustAnchors from ingress TLS in checksum"); + updatedCluster.getSpec() + .getIngresses() + .stream() + .map(Ingresses::getTls) + .filter(Objects::nonNull) + .map(Tls::getTrustAnchorRef) + .flatMap(trustAnchorRef -> context.getSecondaryResourcesAsStream(ConfigMap.class) + .filter(cm -> KubernetesResourceUtil.getName(cm).equals(trustAnchorRef.getRef().getName()))) .forEach(checksumGenerator::appendMetadata); } diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java index d73c665823..fb2e6e7161 100644 --- a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java @@ -1009,6 +1009,31 @@ void shouldIncludeDownstreamTlsSecretInChecksum() { verify(checksumGenerator).appendMetadata(KUBE_TLS_CERT_SECRET); } + @Test + void shouldIncludeDownstreamTlsTrustAnchorInChecksum() { + // Given + Context context = mock(); + MetadataChecksumGenerator checksumGenerator = mock(MetadataChecksumGenerator.class); + when(workflowContext.get(MetadataChecksumGenerator.CHECKSUM_CONTEXT_KEY, MetadataChecksumGenerator.class)).thenReturn(Optional.of(checksumGenerator)); + when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(context.getSecondaryResource(ConfigMap.class)).thenReturn(Optional.of(buildProxyConfigMapWithPatch(CLUSTER_TLS_NO_FILTERS_WITH_TRUST_ANCHOR))); + when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); + when(context.getSecondaryResources(KafkaProtocolFilter.class)).thenReturn(Set.of()); + when(context.getSecondaryResources(Service.class)).thenReturn(Set.of(KUBERNETES_INGRESS_SERVICES)); + when(context.getSecondaryResource(Secret.class, "secrets")).thenReturn(Optional.of(KUBE_TLS_CERT_SECRET)); + when(context.getSecondaryResource(ConfigMap.class, "configmaps")).thenReturn(Optional.of(PEM_CONFIG_MAP)); + when(context.getSecondaryResourcesAsStream(Secret.class)).thenReturn(Stream.of(KUBE_TLS_CERT_SECRET)); + when(context.getSecondaryResourcesAsStream(ConfigMap.class)).thenReturn(Stream.of(PEM_CONFIG_MAP)); + when(context.managedWorkflowAndDependentResourceContext()).thenReturn(workflowContext); + + // When + virtualKafkaClusterReconciler.reconcile(CLUSTER_TLS_NO_FILTERS_WITH_TRUST_ANCHOR, context); + + // Then + verify(checksumGenerator).appendMetadata(PEM_CONFIG_MAP); + } + private static EventSourceContext mockContextContaining(VirtualKafkaCluster cluster) { EventSourceContext eventSourceContext = mock(); KubernetesClient client = mock(); diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/Kroxylicious.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/Kroxylicious.java index 813fa1c9d7..ca782c3cb8 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/Kroxylicious.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/Kroxylicious.java @@ -114,6 +114,7 @@ public void deployPortIdentifiesNodeWithDownstreamTlsAndNoFilters(String cluster KroxyliciousKafkaProxyIngressTemplates.tlsKafkaProxyIngressCR(deploymentNamespace, Constants.KROXYLICIOUS_INGRESS_CLUSTER_IP, Constants.KROXYLICIOUS_PROXY_SIMPLE_NAME), KroxyliciousKafkaClusterRefTemplates.defaultKafkaClusterRefCR(deploymentNamespace, clusterName), + KroxyliciousConfigMapTemplates.getClusterCaConfigMap(deploymentNamespace, Constants.KROXYLICIOUS_TLS_CLIENT_CA_CERT, tls), KroxyliciousVirtualKafkaClusterTemplates.defaultVirtualKafkaClusterWithTlsCR(deploymentNamespace, clusterName, Constants.KROXYLICIOUS_PROXY_SIMPLE_NAME, clusterName, Constants.KROXYLICIOUS_INGRESS_CLUSTER_IP, tls)); } @@ -138,27 +139,27 @@ public Tls createCertificateConfigMapFromListener(String namespace) { @NonNull private static TrustAnchorRef buildTrustAnchorRef() { - //formatter:off + // formatter:off return new TrustAnchorRefBuilder() .withNewRef() - .withName(Constants.KROXYLICIOUS_TLS_CLIENT_CA_CERT) - .withKind(Constants.CONFIG_MAP) + .withName(Constants.KROXYLICIOUS_TLS_CLIENT_CA_CERT) + .withKind(Constants.CONFIG_MAP) .endRef() .withKey(Constants.KROXYLICIOUS_TLS_CA_NAME) .build(); - //formatter:on + // formatter:on } public Tls tlsConfigFromCert(String certNane) { TlsBuilder tlsBuilder = new TlsBuilder(); if (certNane != null) { - //formatter:off + // formatter:off tlsBuilder - .withNewCertificateRef() + .withNewCertificateRef() .withName(certNane) .withKind("Secret") - .endCertificateRef(); - //formatter:on + .endCertificateRef(); + // formatter:on } tlsBuilder.withTrustAnchorRef(buildTrustAnchorRef()); return tlsBuilder.build(); diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/manager/ResourceManager.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/manager/ResourceManager.java index e14848dcec..347fde5b5f 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/manager/ResourceManager.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/manager/ResourceManager.java @@ -7,6 +7,7 @@ package io.kroxylicious.systemtests.resources.manager; import java.util.Arrays; +import java.util.Objects; import java.util.function.Consumer; import org.junit.jupiter.api.extension.ExtensionContext; @@ -123,12 +124,13 @@ public final void createResourceFromBuilderWithWait(Builder... resources) { - KubeResourceManager.get().createResourceWithWait(Arrays.stream(resources).map(Builder::build).toList().toArray(new HasMetadata[0])); + KubeResourceManager.get().createResourceWithWait(Arrays.stream(resources).filter(Objects::nonNull).map(Builder::build).toList().toArray(new HasMetadata[0])); } @SafeVarargs public final void createOrUpdateResourceWithWait(Builder... resources) { - KubeResourceManager.get().createOrUpdateResourceWithWait(Arrays.stream(resources).map(Builder::build).toList().toArray(new HasMetadata[0])); + KubeResourceManager.get() + .createOrUpdateResourceWithWait(Arrays.stream(resources).filter(Objects::nonNull).map(Builder::build).toList().toArray(new HasMetadata[0])); } public void replaceResourceWithRetries(T resource, Consumer editor) { diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousConfigMapTemplates.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousConfigMapTemplates.java index 11666d8790..18e4a5f153 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousConfigMapTemplates.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousConfigMapTemplates.java @@ -8,8 +8,11 @@ import java.util.Map; +import io.fabric8.kubernetes.api.builder.Builder; import io.fabric8.kubernetes.api.model.ConfigMapBuilder; +import io.fabric8.kubernetes.api.model.HasMetadata; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaservicespec.Tls; import io.kroxylicious.systemtests.Constants; /** @@ -55,4 +58,20 @@ public static ConfigMapBuilder getClusterCaConfigMap(String namespace, String na .withData(Map.of(Constants.KROXYLICIOUS_TLS_CA_NAME, certificate)); // @formatter:on } + + public static Builder getClusterCaConfigMap(String namespace, String name, Tls tls) { + if (tls.getTrustAnchorRef() != null) { + // @formatter:off + return new ConfigMapBuilder() + .withNewMetadata() + .withName(name) + .withNamespace(namespace) + .endMetadata() + .withData(Map.of(Constants.KROXYLICIOUS_TLS_CA_NAME, tls.getTrustAnchorRef().getRef().getName())); + // @formatter:on + } + else { + return null; + } + } } diff --git a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java index d1fa74b118..d39a7e56e3 100644 --- a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java +++ b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java @@ -167,11 +167,10 @@ void shouldUpdateDeploymentWhenDownstreamTrustUpdated(String namespace) throws I String originalChecksum = getInitialChecksum(namespace, kubeClient); ConfigMap trustAnchorConfig = new ConfigMapBuilder().withNewMetadata().withName(Constants.KROXYLICIOUS_TLS_CLIENT_CA_CERT).withNamespace(namespace).endMetadata() .build(); + // When resourceManager.replaceResourceWithRetries(trustAnchorConfig, - trustConfigMap -> { - trustAnchorConfig.setData(Map.of(Constants.KROXYLICIOUS_TLS_CA_NAME, "server-certificate")); - }); + trustConfigMap -> trustAnchorConfig.setData(Map.of(Constants.KROXYLICIOUS_TLS_CA_NAME, "server-certificate1"))); LOGGER.info("Downstream trust updated"); // Then From 4656dddce35479ae6fa201fc52fe4cb6d7d51e44 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Mon, 2 Jun 2025 11:08:31 +1200 Subject: [PATCH 49/83] skip null trust anchors Co-authored-by: Keith Wall Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../kubernetes/operator/VirtualKafkaClusterReconciler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java index 378d702f26..4e12d717b1 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java @@ -159,6 +159,7 @@ private static void appendSecretsFromCertificateRefs(Context context.getSecondaryResourcesAsStream(ConfigMap.class) .filter(cm -> KubernetesResourceUtil.getName(cm).equals(trustAnchorRef.getRef().getName()))) .forEach(checksumGenerator::appendMetadata); From 1acfc35c878ff1f6b077299bbdccbabe4def4de2 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Tue, 3 Jun 2025 10:49:09 +1200 Subject: [PATCH 50/83] Sonar: build slug only if required Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../operator/checksum/MetadataChecksumGenerator.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/checksum/MetadataChecksumGenerator.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/checksum/MetadataChecksumGenerator.java index b9079bd42d..10b8a58e89 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/checksum/MetadataChecksumGenerator.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/checksum/MetadataChecksumGenerator.java @@ -36,7 +36,9 @@ default void appendMetadata(ObjectMeta objectMeta) { } default void appendMetadata(HasMetadata entity) { - LOGGER.debug("appendMetadata for: {}", ResourcesUtil.namespacedSlug(entity)); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("appendMetadata for: {}", ResourcesUtil.namespacedSlug(entity)); + } appendMetadata(entity.getMetadata()); } From 5f947cef06962fd3266143865ea43975fb7b407d Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Tue, 3 Jun 2025 10:50:56 +1200 Subject: [PATCH 51/83] Sonar: use more specific return type Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../templates/kroxylicious/KroxyliciousConfigMapTemplates.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousConfigMapTemplates.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousConfigMapTemplates.java index 18e4a5f153..abf9ecc995 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousConfigMapTemplates.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousConfigMapTemplates.java @@ -59,7 +59,7 @@ public static ConfigMapBuilder getClusterCaConfigMap(String namespace, String na // @formatter:on } - public static Builder getClusterCaConfigMap(String namespace, String name, Tls tls) { + public static ConfigMapBuilder getClusterCaConfigMap(String namespace, String name, Tls tls) { if (tls.getTrustAnchorRef() != null) { // @formatter:off return new ConfigMapBuilder() From 506119c1a03b61aa0b0c4218d56c4b995bf63e4c Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Tue, 3 Jun 2025 14:52:06 +1200 Subject: [PATCH 52/83] Sonar: Reduce complexity Add unit tests for `checkTrustAnchorRef` as a bonus Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../operator/assertj/ConditionAssert.java | 14 ++- .../kubernetes/operator/ResourcesUtil.java | 16 ++- .../operator/ResourcesUtilTest.java | 115 ++++++++++++++++++ .../KroxyliciousConfigMapTemplates.java | 2 - 4 files changed, 139 insertions(+), 8 deletions(-) diff --git a/kroxylicious-operator-test-support/src/main/java/io/kroxylicious/kubernetes/operator/assertj/ConditionAssert.java b/kroxylicious-operator-test-support/src/main/java/io/kroxylicious/kubernetes/operator/assertj/ConditionAssert.java index 85d355b65b..04502af7fb 100644 --- a/kroxylicious-operator-test-support/src/main/java/io/kroxylicious/kubernetes/operator/assertj/ConditionAssert.java +++ b/kroxylicious-operator-test-support/src/main/java/io/kroxylicious/kubernetes/operator/assertj/ConditionAssert.java @@ -16,6 +16,7 @@ import org.assertj.core.api.Assertions; import org.assertj.core.api.InstanceOfAssertFactories; import org.assertj.core.api.ObjectAssert; +import org.assertj.core.api.ThrowingConsumer; import io.fabric8.kubernetes.api.model.HasMetadata; @@ -69,6 +70,11 @@ public ConditionAssert hasMessage(String expected) { return this; } + public ConditionAssert hasMessage(ThrowingConsumer expected) { + message().describedAs(actual.toString()).satisfies(expected); + return this; + } + public AbstractStringAssert reason() { return Assertions.assertThat(actual.getReason()); } @@ -126,11 +132,15 @@ public ConditionAssert isResolvedRefsUnknown(String reason, String message) { return this; } - public ConditionAssert isResolvedRefsFalse(String reason, String message) { + public ConditionAssert isResolvedRefsFalse(String reason, String expectedMessage) { + return isResolvedRefsFalse(reason, (String actualMessage) -> message().describedAs(actualMessage).isEqualTo(expectedMessage)); + } + + public ConditionAssert isResolvedRefsFalse(String reason, ThrowingConsumer messageAssertion) { hasType(Condition.Type.ResolvedRefs); hasStatus(Condition.Status.FALSE); hasReason(reason); - hasMessage(message); + hasMessage(messageAssertion); return this; } diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ResourcesUtil.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ResourcesUtil.java index 4a31af0c3d..c911c7b06a 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ResourcesUtil.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ResourcesUtil.java @@ -491,16 +491,14 @@ private static boolean isStatusFresh(T resource, Functio Condition.REASON_INVALID, path + " must specify 'key'"), List.of()); } - if (!key.endsWith(".pem") - && !key.endsWith(".p12") - && !key.endsWith(".jks")) { + if (isSupportedFileType(key)) { return new ResourceCheckResult<>(statusFactory.newFalseConditionStatusPatch(resource, ResolvedRefs, Condition.REASON_INVALID, path + ".key should end with .pem, .p12 or .jks"), List.of()); } else { ConfigMap configMap = configMapOpt.get(); - if (!configMap.getData().containsKey(trustAnchorRef.getKey())) { + if (keyIsMissingFromConfigMap(trustAnchorRef, configMap)) { return new ResourceCheckResult<>(statusFactory.newFalseConditionStatusPatch(resource, ResolvedRefs, Condition.REASON_INVALID_REFERENCED_RESOURCE, path + ": referenced resource does not contain key " + trustAnchorRef.getKey()), List.of()); @@ -517,4 +515,14 @@ private static boolean isStatusFresh(T resource, Functio path + " supports referents: configmaps"), List.of()); } } + + private static boolean keyIsMissingFromConfigMap(TrustAnchorRef trustAnchorRef, ConfigMap configMap) { + return !configMap.getData().containsKey(trustAnchorRef.getKey()); + } + + private static boolean isSupportedFileType(String key) { + return !key.endsWith(".pem") + && !key.endsWith(".p12") + && !key.endsWith(".jks"); + } } diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/ResourcesUtilTest.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/ResourcesUtilTest.java index fb7a0fd0fa..6146d7694d 100644 --- a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/ResourcesUtilTest.java +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/ResourcesUtilTest.java @@ -6,7 +6,9 @@ package io.kroxylicious.kubernetes.operator; +import java.time.Clock; import java.time.Instant; +import java.time.ZoneId; import java.util.List; import java.util.Map; import java.util.Optional; @@ -14,9 +16,11 @@ import java.util.stream.Collector; import java.util.stream.Stream; +import org.assertj.core.api.ThrowingConsumer; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; import io.fabric8.kubernetes.api.model.ConfigMap; @@ -30,6 +34,7 @@ import io.fabric8.kubernetes.client.dsl.MixedOperation; import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation; import io.fabric8.kubernetes.client.dsl.Resource; +import io.javaoperatorsdk.operator.api.reconciler.Context; import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; import io.javaoperatorsdk.operator.processing.event.ResourceID; @@ -40,6 +45,8 @@ import io.kroxylicious.kubernetes.api.common.IngressRefBuilder; import io.kroxylicious.kubernetes.api.common.KafkaServiceRefBuilder; import io.kroxylicious.kubernetes.api.common.ProxyRefBuilder; +import io.kroxylicious.kubernetes.api.common.TrustAnchorRef; +import io.kroxylicious.kubernetes.api.common.TrustAnchorRefBuilder; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxy; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyBuilder; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngress; @@ -48,10 +55,14 @@ import io.kroxylicious.kubernetes.api.v1alpha1.KafkaServiceBuilder; import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaCluster; import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaClusterBuilder; +import io.kroxylicious.kubernetes.api.v1alpha1.virtualkafkaclusterspec.Ingresses; +import io.kroxylicious.kubernetes.api.v1alpha1.virtualkafkaclusterspec.IngressesBuilder; import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilter; import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilterBuilder; +import io.kroxylicious.kubernetes.operator.assertj.VirtualKafkaClusterStatusAssert; import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; import static io.kroxylicious.kubernetes.operator.ResourcesUtil.findOnlyResourceNamed; import static io.kroxylicious.kubernetes.operator.ResourcesUtil.toByNameMap; @@ -65,6 +76,8 @@ class ResourcesUtilTest { public static final String RESOURCE_NAME = "name"; + public static final Clock TEST_CLOCK = Clock.fixed(Instant.EPOCH, ZoneId.of("Z")); + protected static final ConfigMap EMPTY_CONFIG_NMAP = new ConfigMapBuilder().withData(Map.of()).build(); @Test void rfc1035DnsLabel() { @@ -659,4 +672,106 @@ private EventSourceContext prepareMockContextToProduc when(eventSourceContext.getClient()).thenReturn(client); return eventSourceContext; } + + @ParameterizedTest + @MethodSource("invalidTrustAnchorRefs") + void shouldReturnResolvedRefsFalseStatusCondition(TrustAnchorRef trustAnchorRef, + @Nullable ConfigMap targetedConfigMap, + String expectedCondition, + ThrowingConsumer stringThrowingConsumer) { + // Given + Ingresses ingress = new IngressesBuilder().withNewTls().withTrustAnchorRef(trustAnchorRef).endTls().build(); + VirtualKafkaCluster vkc = new VirtualKafkaClusterBuilder().withNewSpec().withIngresses(List.of(ingress)).endSpec().build(); + @SuppressWarnings("unchecked") + Context reconcilerContext = mock(Context.class); + when(reconcilerContext.getSecondaryResource(ConfigMap.class, VirtualKafkaClusterReconciler.CONFIGMAPS_EVENT_SOURCE_NAME)) + .thenReturn(Optional.ofNullable(targetedConfigMap)); + + // When + ResourceCheckResult actual = ResourcesUtil.checkTrustAnchorRef(vkc, reconcilerContext, + VirtualKafkaClusterReconciler.CONFIGMAPS_EVENT_SOURCE_NAME, trustAnchorRef, + "spec.ingresses[].tls.trustAnchor", new VirtualKafkaClusterStatusFactory(TEST_CLOCK)); + + // Then + assertThat(actual) + .isNotNull() + .satisfies(virtualKafkaClusterResourceCheckResult -> assertThat(virtualKafkaClusterResourceCheckResult.resource()) + .isNotNull() + .satisfies(actualVkc -> VirtualKafkaClusterStatusAssert.assertThat(actualVkc.getStatus()) + .singleCondition() + .isResolvedRefsFalse(expectedCondition, stringThrowingConsumer))); + + } + + @ParameterizedTest + @CsvSource({ "key.pem", "key.p12", "key.jks" }) + void shouldAcceptSupportedKeyfileExtensions(String key) { + // Given + TrustAnchorRef trustAnchorRef = new TrustAnchorRefBuilder().withKey(key).withNewRef().withName("configmap").endRef().build(); + Ingresses ingress = new IngressesBuilder().withNewTls().withTrustAnchorRef(trustAnchorRef).endTls().build(); + VirtualKafkaCluster vkc = new VirtualKafkaClusterBuilder().withNewSpec().withIngresses(List.of(ingress)).endSpec().build(); + @SuppressWarnings("unchecked") + Context reconcilerContext = mock(Context.class); + when(reconcilerContext.getSecondaryResource(ConfigMap.class, VirtualKafkaClusterReconciler.CONFIGMAPS_EVENT_SOURCE_NAME)) + .thenReturn(Optional.ofNullable( + new ConfigMapBuilder().withNewMetadata().withName("configmap").endMetadata().withData(Map.of(key, "I'm a key honnest")).build())); + + // When + ResourceCheckResult actual = ResourcesUtil.checkTrustAnchorRef(vkc, reconcilerContext, + VirtualKafkaClusterReconciler.CONFIGMAPS_EVENT_SOURCE_NAME, trustAnchorRef, + "spec.ingresses[].tls.trustAnchor", new VirtualKafkaClusterStatusFactory(TEST_CLOCK)); + + // Then + assertThat(actual).isNotNull() + .satisfies(virtualKafkaClusterResourceCheckResult -> assertThat(virtualKafkaClusterResourceCheckResult.resource()).isNull()); + } + + static Stream invalidTrustAnchorRefs() { + return Stream.of( + argumentSet("Invalid reference to config map invalid kind.", + new TrustAnchorRefBuilder().withKey("key.pem").withNewRef().withKind("custom").withName("configmap").endRef().build(), + null, + Condition.REASON_REF_GROUP_KIND_NOT_SUPPORTED, + (ThrowingConsumer) message -> assertThat(message).endsWith("supports referents: configmaps")), + argumentSet("Invalid reference to config map invalid group.", + new TrustAnchorRefBuilder().withKey("key.pem").withNewRef().withGroup("custom").withName("configmap").endRef().build(), + null, + Condition.REASON_REF_GROUP_KIND_NOT_SUPPORTED, + (ThrowingConsumer) message -> assertThat(message).endsWith("supports referents: configmaps")), + argumentSet("Empty target config map.", + new TrustAnchorRefBuilder().withKey("key.pem").withNewRef().withName("configmap").endRef().build(), + null, + Condition.REASON_REFS_NOT_FOUND, + (ThrowingConsumer) message -> assertThat(message).endsWith("referenced resource not found")), + argumentSet("Empty target config map.", + new TrustAnchorRefBuilder().withKey("key.pem").withNewRef().withName("configmap").endRef().build(), + null, + Condition.REASON_REFS_NOT_FOUND, + (ThrowingConsumer) message -> assertThat(message).endsWith("referenced resource not found")), + argumentSet("ConfigMap not found", + new TrustAnchorRefBuilder().withNewRef().withName("unknown_config_map").endRef().build(), + null, + Condition.REASON_REFS_NOT_FOUND, + (ThrowingConsumer) message -> assertThat(message).endsWith("referenced resource not found")), + argumentSet("null key", + new TrustAnchorRefBuilder().withKey(null).withNewRef().withName("configmap").endRef().build(), + EMPTY_CONFIG_NMAP, + Condition.REASON_INVALID, + (ThrowingConsumer) message -> assertThat(message).endsWith("must specify 'key'")), + argumentSet("blank key", + new TrustAnchorRefBuilder().withKey("").withNewRef().withName("configmap").endRef().build(), + EMPTY_CONFIG_NMAP, + Condition.REASON_INVALID, + (ThrowingConsumer) message -> assertThat(message).endsWith(".key should end with .pem, .p12 or .jks")), + argumentSet("unsupported key file extension", + new TrustAnchorRefBuilder().withKey("/path/to/random.key").withNewRef().withName("configmap").endRef().build(), + EMPTY_CONFIG_NMAP, + Condition.REASON_INVALID, + (ThrowingConsumer) message -> assertThat(message).endsWith(".key should end with .pem, .p12 or .jks")), + argumentSet("Config map does not contain key", + new TrustAnchorRefBuilder().withKey("key.p12").withNewRef().withName("configmap").endRef().build(), + EMPTY_CONFIG_NMAP, + Condition.REASON_INVALID_REFERENCED_RESOURCE, + (ThrowingConsumer) message -> assertThat(message).contains("referenced resource does not contain key"))); + } } diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousConfigMapTemplates.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousConfigMapTemplates.java index abf9ecc995..3ffecc1d83 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousConfigMapTemplates.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousConfigMapTemplates.java @@ -8,9 +8,7 @@ import java.util.Map; -import io.fabric8.kubernetes.api.builder.Builder; import io.fabric8.kubernetes.api.model.ConfigMapBuilder; -import io.fabric8.kubernetes.api.model.HasMetadata; import io.kroxylicious.kubernetes.api.v1alpha1.kafkaservicespec.Tls; import io.kroxylicious.systemtests.Constants; From 4c7f486e93372327d8a64aa7b474ae445d36e5f6 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Tue, 3 Jun 2025 14:58:29 +1200 Subject: [PATCH 53/83] PR Review: drop log level Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../kubernetes/operator/VirtualKafkaClusterReconciler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java index 4e12d717b1..c22ab2c02a 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java @@ -141,7 +141,7 @@ public UpdateControl reconcile(VirtualKafkaCluster cluster, private static void appendSecretsFromCertificateRefs(Context context, VirtualKafkaCluster updatedCluster, @NonNull MetadataChecksumGenerator checksumGenerator) { - LOGGER.info("Including secrets from ingress TLS in checksum"); + LOGGER.debug("Including secrets from ingress TLS in checksum"); updatedCluster.getSpec() .getIngresses() .stream() @@ -152,7 +152,7 @@ private static void appendSecretsFromCertificateRefs(Context KubernetesResourceUtil.getName(secret).equals(certificateRef.getName()))) .forEach(checksumGenerator::appendMetadata); - LOGGER.info("Including TrustAnchors from ingress TLS in checksum"); + LOGGER.debug("Including TrustAnchors from ingress TLS in checksum"); updatedCluster.getSpec() .getIngresses() .stream() From fc45394b9ebc1efed734c2128d205a8b6925615a Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Tue, 3 Jun 2025 15:10:21 +1200 Subject: [PATCH 54/83] PR Review: add test to generate checksum generator Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../VirtualKafkaClusterReconcilerTest.java | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java index fb2e6e7161..c23651c957 100644 --- a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java @@ -63,6 +63,7 @@ import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilter; import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilterBuilder; import io.kroxylicious.kubernetes.operator.assertj.ConditionListAssert; +import io.kroxylicious.kubernetes.operator.assertj.MetadataAssert; import io.kroxylicious.kubernetes.operator.assertj.VirtualKafkaClusterStatusAssert; import io.kroxylicious.kubernetes.operator.checksum.MetadataChecksumGenerator; import io.kroxylicious.kubernetes.operator.resolver.DependencyResolver; @@ -71,6 +72,7 @@ import static io.kroxylicious.kubernetes.operator.ResourcesUtil.name; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.InstanceOfAssertFactories.OPTIONAL; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -221,6 +223,7 @@ class VirtualKafkaClusterReconcilerTest { .withNewMetadata() .withName(SERVER_CERT_SECRET_NAME) .withNamespace(NAMESPACE) + .withUid(UUID.randomUUID().toString()) .withGeneration(42L) .endMetadata() .withType("kubernetes.io/tls") @@ -231,6 +234,7 @@ class VirtualKafkaClusterReconcilerTest { .withNewMetadata() .withName(SERVER_CERT_SECRET_NAME) .withNamespace(NAMESPACE) + .withUid(UUID.randomUUID().toString()) .withGeneration(42L) .endMetadata() .withType("example.com/nottls") // unexpected type value @@ -243,6 +247,7 @@ class VirtualKafkaClusterReconcilerTest { .withName(TRUST_ANCHOR_CERT_CONFIGMAP_NAME) .withNamespace(NAMESPACE) .withGeneration(42L) + .withUid(UUID.randomUUID().toString()) .endMetadata() .addToData("ca-bundle.pem", "value") .build(); @@ -1034,6 +1039,37 @@ void shouldIncludeDownstreamTlsTrustAnchorInChecksum() { verify(checksumGenerator).appendMetadata(PEM_CONFIG_MAP); } + @Test + void shouldCreateChecksumGeneratorIfNotPresentInReconcilerContext() { + // Given + Context reconcilerContext = mock(); + when(workflowContext.get(MetadataChecksumGenerator.CHECKSUM_CONTEXT_KEY, MetadataChecksumGenerator.class)).thenReturn(Optional.empty()); + when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(reconcilerContext.getSecondaryResource(ConfigMap.class)).thenReturn(Optional.of(buildProxyConfigMapWithPatch(CLUSTER_TLS_NO_FILTERS_WITH_TRUST_ANCHOR))); + when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); + when(reconcilerContext.getSecondaryResources(KafkaProtocolFilter.class)).thenReturn(Set.of()); + when(reconcilerContext.getSecondaryResources(Service.class)).thenReturn(Set.of(KUBERNETES_INGRESS_SERVICES)); + when(reconcilerContext.getSecondaryResource(Secret.class, "secrets")).thenReturn(Optional.of(KUBE_TLS_CERT_SECRET)); + when(reconcilerContext.getSecondaryResource(ConfigMap.class, "configmaps")).thenReturn(Optional.of(PEM_CONFIG_MAP)); + when(reconcilerContext.getSecondaryResourcesAsStream(Secret.class)).thenReturn(Stream.of(KUBE_TLS_CERT_SECRET)); + when(reconcilerContext.getSecondaryResourcesAsStream(ConfigMap.class)).thenReturn(Stream.of(PEM_CONFIG_MAP)); + when(reconcilerContext.managedWorkflowAndDependentResourceContext()).thenReturn(workflowContext); + + // When + var actualUpdate = virtualKafkaClusterReconciler.reconcile(CLUSTER_TLS_NO_FILTERS_WITH_TRUST_ANCHOR, reconcilerContext); + + // Then + assertThat(actualUpdate) + .isNotNull() + .extracting(UpdateControl::getResource).asInstanceOf(OPTIONAL) + .isPresent() + .get(InstanceOfAssertFactories.type(VirtualKafkaCluster.class)) + .satisfies(virtualKafkaCluster -> MetadataAssert.assertThat(virtualKafkaCluster) + .hasAnnotationSatisfying(MetadataChecksumGenerator.REFERENT_CHECKSUM_ANNOTATION, + (value) -> assertThat(value).isBase64())); + } + private static EventSourceContext mockContextContaining(VirtualKafkaCluster cluster) { EventSourceContext eventSourceContext = mock(); KubernetesClient client = mock(); From d7480794d06760c58d7b66d818249cafd8469aa7 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Tue, 3 Jun 2025 16:50:35 +1200 Subject: [PATCH 55/83] Edit the right freaking resource Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../io/kroxylicious/systemtests/OperatorChangeDetectionST.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java index d39a7e56e3..8750bc167e 100644 --- a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java +++ b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java @@ -170,7 +170,7 @@ void shouldUpdateDeploymentWhenDownstreamTrustUpdated(String namespace) throws I // When resourceManager.replaceResourceWithRetries(trustAnchorConfig, - trustConfigMap -> trustAnchorConfig.setData(Map.of(Constants.KROXYLICIOUS_TLS_CA_NAME, "server-certificate1"))); + trustConfigMap -> trustConfigMap.getData().put(Constants.KROXYLICIOUS_TLS_CA_NAME, "server-certificate1")); LOGGER.info("Downstream trust updated"); // Then From abedcf7d10cb5a240221229fe416182896289bca Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Tue, 3 Jun 2025 16:53:18 +1200 Subject: [PATCH 56/83] Sonar nits Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../io/kroxylicious/kubernetes/operator/ResourcesUtilTest.java | 3 ++- .../kubernetes/operator/VirtualKafkaClusterReconcilerTest.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/ResourcesUtilTest.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/ResourcesUtilTest.java index 6146d7694d..88142b0f77 100644 --- a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/ResourcesUtilTest.java +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/ResourcesUtilTest.java @@ -512,7 +512,8 @@ void hasResolvedRefsFalse(HasMetadata cluster, boolean hasResolvedRefsFalse) { @Test void hasResolvedRefsFalseThrowsWhenResourceDoesntUseResolveRefs() { - assertThatThrownBy(() -> ResourcesUtil.hasFreshResolvedRefsFalseCondition(new KafkaProxy())).isInstanceOf(IllegalArgumentException.class) + KafkaProxy kafkaProxy = new KafkaProxy(); + assertThatThrownBy(() -> ResourcesUtil.hasFreshResolvedRefsFalseCondition(kafkaProxy)).isInstanceOf(IllegalArgumentException.class) .hasMessage("Resource kind 'KafkaProxy' does not use ResolveRefs conditions"); } diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java index c23651c957..acaf85d13f 100644 --- a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java @@ -1067,7 +1067,7 @@ void shouldCreateChecksumGeneratorIfNotPresentInReconcilerContext() { .get(InstanceOfAssertFactories.type(VirtualKafkaCluster.class)) .satisfies(virtualKafkaCluster -> MetadataAssert.assertThat(virtualKafkaCluster) .hasAnnotationSatisfying(MetadataChecksumGenerator.REFERENT_CHECKSUM_ANNOTATION, - (value) -> assertThat(value).isBase64())); + value -> assertThat(value).isBase64())); } private static EventSourceContext mockContextContaining(VirtualKafkaCluster cluster) { From 34d668a30a0c4b113e6d372376832077af19f9c6 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Tue, 3 Jun 2025 16:59:10 +1200 Subject: [PATCH 57/83] use the return value to make code analysis STFU Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../kubernetes/operator/ResourcesUtil.java | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ResourcesUtil.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ResourcesUtil.java index c911c7b06a..d8e8edc17c 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ResourcesUtil.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ResourcesUtil.java @@ -46,6 +46,8 @@ import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilter; import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilterStatus; +import edu.umd.cs.findbugs.annotations.NonNull; + import static io.kroxylicious.kubernetes.api.common.Condition.Type.ResolvedRefs; public class ResourcesUtil { @@ -96,9 +98,8 @@ static String requireIsDnsLabel(String string, boolean rfc1035, String message) static String volumeName(String group, String plural, String resourceName) { String volumeNamePrefix = group.isEmpty() ? plural : group + "." + plural; String volumeName = volumeNamePrefix + "-" + resourceName; - ResourcesUtil.requireIsDnsLabel(volumeName, true, + return ResourcesUtil.requireIsDnsLabel(volumeName, true, "volume name would not be a DNS label: " + volumeName); - return volumeName; } static boolean isSecret(LocalRef ref) { @@ -497,15 +498,7 @@ private static boolean isStatusFresh(T resource, Functio path + ".key should end with .pem, .p12 or .jks"), List.of()); } else { - ConfigMap configMap = configMapOpt.get(); - if (keyIsMissingFromConfigMap(trustAnchorRef, configMap)) { - return new ResourceCheckResult<>(statusFactory.newFalseConditionStatusPatch(resource, ResolvedRefs, - Condition.REASON_INVALID_REFERENCED_RESOURCE, - path + ": referenced resource does not contain key " + trustAnchorRef.getKey()), List.of()); - } - else { - return new ResourceCheckResult<>(null, List.of(configMap)); - } + return handleSupportedFileExtension(resource, trustAnchorRef, path, statusFactory, configMapOpt.get()); } } } @@ -516,6 +509,18 @@ private static boolean isStatusFresh(T resource, Functio } } + private static > @NonNull ResourceCheckResult handleSupportedFileExtension(T resource, TrustAnchorRef trustAnchorRef, String path, + StatusFactory statusFactory, ConfigMap configMap) { + if (keyIsMissingFromConfigMap(trustAnchorRef, configMap)) { + return new ResourceCheckResult<>(statusFactory.newFalseConditionStatusPatch(resource, ResolvedRefs, + Condition.REASON_INVALID_REFERENCED_RESOURCE, + path + ": referenced resource does not contain key " + trustAnchorRef.getKey()), List.of()); + } + else { + return new ResourceCheckResult<>(null, List.of(configMap)); + } + } + private static boolean keyIsMissingFromConfigMap(TrustAnchorRef trustAnchorRef, ConfigMap configMap) { return !configMap.getData().containsKey(trustAnchorRef.getKey()); } From 1446612929f53f8bd12c1d4f606b103d999e6c16 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Tue, 3 Jun 2025 17:07:49 +1200 Subject: [PATCH 58/83] stop things moaning about possible NPEs Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../kroxylicious/kubernetes/operator/ResourcesUtil.java | 5 +++-- .../kubernetes/operator/ResourcesUtilTest.java | 9 +++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ResourcesUtil.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ResourcesUtil.java index d8e8edc17c..a3213eb0eb 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ResourcesUtil.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ResourcesUtil.java @@ -312,8 +312,9 @@ public static String namespacedSlug(HasMetadata resource) { private static String slug(LocalRef ref) { String group = ref.getGroup(); String name = ref.getName(); - String groupString = group.isEmpty() ? "" : "." + group; - return ref.getKind().toLowerCase(Locale.ROOT) + groupString + "/" + name; + String kind = ref.getKind() == null || ref.getKind().isEmpty() ? "" : ref.getKind().toLowerCase(Locale.ROOT); + String groupString = group == null || group.isEmpty() ? "" : "." + group; + return kind + groupString + "/" + name; } private static String slug(HasMetadata ref) { diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/ResourcesUtilTest.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/ResourcesUtilTest.java index 88142b0f77..cde389ab6d 100644 --- a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/ResourcesUtilTest.java +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/ResourcesUtilTest.java @@ -38,6 +38,7 @@ import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; import io.javaoperatorsdk.operator.processing.event.ResourceID; +import io.kroxylicious.kubernetes.api.common.AnyLocalRef; import io.kroxylicious.kubernetes.api.common.AnyLocalRefBuilder; import io.kroxylicious.kubernetes.api.common.Condition; import io.kroxylicious.kubernetes.api.common.ConditionBuilder; @@ -359,6 +360,14 @@ void slugWithNamespaceNonEmptyGroup() { assertThat(ResourcesUtil.namespacedSlug(ResourcesUtil.toLocalRef(secret), secret)).isEqualTo("kafkaproxy.kroxylicious.io/secreto in namespace 'my-ns'"); } + @SuppressWarnings("DataFlowIssue") + @Test + void slugWithNamespaceEmptyKind() { + Secret secret = new SecretBuilder().withNewMetadata().withNamespace("my-ns").withName("secreto").endMetadata().build(); + AnyLocalRef ref = new AnyLocalRefBuilder().withName("secreto").withKind(null).withName(ResourcesUtil.name(secret)).build(); + assertThat(ResourcesUtil.namespacedSlug(ref, secret)).isEqualTo("/secreto in namespace 'my-ns'"); + } + @Test void toLocalRef() { assertThat(ResourcesUtil.toLocalRef(new KafkaProxyBuilder().withNewMetadata().withName("foo").endMetadata().build())) From 5d62ff83f6d2bd0cf23528b97cdbb9b2e91fa5e7 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Wed, 4 Jun 2025 09:33:48 +1200 Subject: [PATCH 59/83] PR Review: initialise workflowContext once Did some other renaming to try and make things easier to grok. Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../VirtualKafkaClusterReconcilerTest.java | 221 +++++++++--------- 1 file changed, 110 insertions(+), 111 deletions(-) diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java index acaf85d13f..a21a7633e9 100644 --- a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java @@ -23,7 +23,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.mockito.Mock; import io.fabric8.kubernetes.api.model.ConfigMap; import io.fabric8.kubernetes.api.model.ConfigMapBuilder; @@ -252,43 +251,43 @@ class VirtualKafkaClusterReconcilerTest { .addToData("ca-bundle.pem", "value") .build(); // @formatter:on - @Mock - private static ManagedWorkflowAndDependentResourceContext workflowContext; + + private static final ManagedWorkflowAndDependentResourceContext workflowContext = mock(ManagedWorkflowAndDependentResourceContext.class); private VirtualKafkaClusterReconciler virtualKafkaClusterReconciler; @BeforeEach void setUp() { - virtualKafkaClusterReconciler = new VirtualKafkaClusterReconciler(Clock.systemUTC(), DependencyResolver.create()); - workflowContext = mock(ManagedWorkflowAndDependentResourceContext.class); + virtualKafkaClusterReconciler = new VirtualKafkaClusterReconciler(TEST_CLOCK, DependencyResolver.create()); + } static List shouldSetResolvedRefs() { List result = new ArrayList<>(); { - Context> context = mockContext(); + Context> reconcilerContext = mockReconcilerContext(); - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); + when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); result.add(Arguments.argumentSet("no filter", CLUSTER_NO_FILTERS, - context, + reconcilerContext, (Consumer) conditionList -> conditionList .singleElement() .isResolvedRefsTrue())); } { - Context> context = mockContext(); - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); - when(context.getSecondaryResource(ConfigMap.class)).thenReturn(Optional.of(buildProxyConfigMapWithPatch(CLUSTER_ONE_FILTER))); - when(context.getSecondaryResources(KafkaProtocolFilter.class)).thenReturn(Set.of(FILTER_MY_FILTER)); + Context> reconcilerContext = mockReconcilerContext(); + when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); + when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + when(reconcilerContext.getSecondaryResource(ConfigMap.class)).thenReturn(Optional.of(buildProxyConfigMapWithPatch(CLUSTER_ONE_FILTER))); + when(reconcilerContext.getSecondaryResources(KafkaProtocolFilter.class)).thenReturn(Set.of(FILTER_MY_FILTER)); result.add(Arguments.argumentSet("one filter", CLUSTER_ONE_FILTER, - context, + reconcilerContext, (Consumer) conditionList -> conditionList .singleElement() .isResolvedRefsTrue())); @@ -296,29 +295,29 @@ static List shouldSetResolvedRefs() { { - Context> context = mockContext(); - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); - when(context.getSecondaryResource(ConfigMap.class)).thenReturn(Optional.of(buildProxyConfigMapWithPatch( + Context> reconcilerContext = mockReconcilerContext(); + when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); + when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + when(reconcilerContext.getSecondaryResource(ConfigMap.class)).thenReturn(Optional.of(buildProxyConfigMapWithPatch( new VirtualKafkaClusterBuilder(CLUSTER_ONE_FILTER).editMetadata().withGeneration(40L).endMetadata().build()))); - when(context.getSecondaryResources(KafkaProtocolFilter.class)).thenReturn(Set.of(FILTER_MY_FILTER)); + when(reconcilerContext.getSecondaryResources(KafkaProtocolFilter.class)).thenReturn(Set.of(FILTER_MY_FILTER)); result.add(Arguments.argumentSet("one filter with stale configmap", new VirtualKafkaClusterBuilder(CLUSTER_ONE_FILTER).editOrNewStatus().withObservedGeneration(ResourcesUtil.generation(CLUSTER_NO_FILTERS)) .endStatus().build(), - context, + reconcilerContext, (Consumer) conditionList -> conditionList .singleElement() .isResolvedRefsTrue())); } { - Context> context = mockContext(); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + Context> reconcilerContext = mockReconcilerContext(); + when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); + when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); result.add(Arguments.argumentSet("proxy not found", CLUSTER_NO_FILTERS, - context, + reconcilerContext, (Consumer) conditionList -> conditionList .singleElement() .isResolvedRefsFalse( @@ -327,12 +326,12 @@ static List shouldSetResolvedRefs() { } { - Context> context = mockContext(); - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); + Context> reconcilerContext = mockReconcilerContext(); + when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); result.add(Arguments.argumentSet("service not found", CLUSTER_NO_FILTERS, - context, + reconcilerContext, (Consumer) conditionList -> conditionList .singleElement() .isResolvedRefsFalse( @@ -341,14 +340,14 @@ static List shouldSetResolvedRefs() { } { - Context> context = mockContext(); - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn( + Context> reconcilerContext = mockReconcilerContext(); + when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn( Set.of(INGRESS.edit().editSpec().withNewProxyRef().withName("not-my-proxy").endProxyRef().endSpec().build())); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); result.add(Arguments.argumentSet("ingress refers to a different proxy than virtual cluster", CLUSTER_NO_FILTERS, - context, + reconcilerContext, (Consumer) conditionList -> conditionList .singleElement() .isResolvedRefsFalse( @@ -357,10 +356,10 @@ static List shouldSetResolvedRefs() { } { - Context> context = mockContext(); - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(new KafkaServiceBuilder(SERVICE).withNewStatus().addNewCondition() + Context> reconcilerContext = mockReconcilerContext(); + when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); + when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(new KafkaServiceBuilder(SERVICE).withNewStatus().addNewCondition() .withType(Condition.Type.ResolvedRefs) .withStatus(Condition.Status.FALSE) .withLastTransitionTime(Instant.now()) @@ -370,7 +369,7 @@ static List shouldSetResolvedRefs() { .endCondition().endStatus().build())); result.add(Arguments.argumentSet("service has ResolvedRefs=False condition", CLUSTER_NO_FILTERS, - context, + reconcilerContext, (Consumer) conditionList -> conditionList .singleElement() .isResolvedRefsFalse( @@ -379,12 +378,12 @@ static List shouldSetResolvedRefs() { } { - Context> context = mockContext(); - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + Context> reconcilerContext = mockReconcilerContext(); + when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); result.add(Arguments.argumentSet("ingress not found", CLUSTER_NO_FILTERS, - context, + reconcilerContext, (Consumer) conditionList -> conditionList .singleElement() .isResolvedRefsFalse( @@ -393,10 +392,10 @@ static List shouldSetResolvedRefs() { } { - Context> context = mockContext(); - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn( + Context> reconcilerContext = mockReconcilerContext(); + when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn( Set.of(new KafkaProxyIngressBuilder(INGRESS).withNewStatus().addNewCondition().withType(Condition.Type.ResolvedRefs) .withStatus(Condition.Status.FALSE) .withLastTransitionTime(Instant.now()) @@ -407,7 +406,7 @@ static List shouldSetResolvedRefs() { result.add(Arguments.argumentSet("ingress has ResolvedRefs=False condition", CLUSTER_NO_FILTERS, - context, + reconcilerContext, (Consumer) conditionList -> conditionList .singleElement() .isResolvedRefsFalse( @@ -416,13 +415,13 @@ static List shouldSetResolvedRefs() { } { - Context> context = mockContext(); - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + Context> reconcilerContext = mockReconcilerContext(); + when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); + when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); result.add(Arguments.argumentSet("filter not found", CLUSTER_ONE_FILTER, - context, + reconcilerContext, (Consumer) conditionList -> conditionList .singleElement() .isResolvedRefsFalse( @@ -431,11 +430,11 @@ static List shouldSetResolvedRefs() { } { - Context> context = mockContext(); - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); - when(context.getSecondaryResources(KafkaProtocolFilter.class)).thenReturn(Set.of(new KafkaProtocolFilterBuilder(FILTER_MY_FILTER).withNewStatus() + Context> reconcilerContext = mockReconcilerContext(); + when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); + when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + when(reconcilerContext.getSecondaryResources(KafkaProtocolFilter.class)).thenReturn(Set.of(new KafkaProtocolFilterBuilder(FILTER_MY_FILTER).withNewStatus() .addNewCondition() .withType(Condition.Type.ResolvedRefs) .withStatus(Condition.Status.FALSE) @@ -447,7 +446,7 @@ static List shouldSetResolvedRefs() { result.add(Arguments.argumentSet("filter has ResolvedRefs=False condition", CLUSTER_ONE_FILTER, - context, + reconcilerContext, (Consumer) conditionList -> conditionList .singleElement() .isResolvedRefsFalse( @@ -456,29 +455,29 @@ static List shouldSetResolvedRefs() { } { - Context> context = mockContext(); - mockGetSecret(context, Optional.of(KUBE_TLS_CERT_SECRET)); - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + Context> reconcilerContext = mockReconcilerContext(); + mockGetSecret(reconcilerContext, Optional.of(KUBE_TLS_CERT_SECRET)); + when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); + when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); result.add(Arguments.argumentSet("cluster with tls", CLUSTER_TLS_NO_FILTERS, - context, + reconcilerContext, (Consumer) conditionList -> conditionList .singleElement() .isResolvedRefsTrue())); } { - Context> context = mockContext(); + Context> reconcilerContext = mockReconcilerContext(); - mockGetSecret(context, Optional.empty()); - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + mockGetSecret(reconcilerContext, Optional.empty()); + when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); + when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); result.add(Arguments.argumentSet("cluster with tls - server cert secret not found", CLUSTER_TLS_NO_FILTERS, - context, + reconcilerContext, (Consumer) conditionList -> conditionList .singleElement() .isResolvedRefsFalse( @@ -487,15 +486,15 @@ static List shouldSetResolvedRefs() { } { - Context> context = mockContext(); + Context> reconcilerContext = mockReconcilerContext(); - mockGetSecret(context, Optional.of(NON_KUBE_TLS_CERT_SECRET)); - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + mockGetSecret(reconcilerContext, Optional.of(NON_KUBE_TLS_CERT_SECRET)); + when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); + when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); result.add(Arguments.argumentSet("cluster with tls - server cert secret wrong type", CLUSTER_TLS_NO_FILTERS, - context, + reconcilerContext, (Consumer) conditionList -> conditionList .singleElement() .isResolvedRefsFalse( @@ -504,15 +503,15 @@ static List shouldSetResolvedRefs() { } { - Context> context = mockContext(); + Context> reconcilerContext = mockReconcilerContext(); - mockGetSecret(context, Optional.empty()); - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + mockGetSecret(reconcilerContext, Optional.empty()); + when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); + when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); result.add(Arguments.argumentSet("cluster with tls - server cert unsupported resource type", CLUSTER_TLS_NO_FILTERS_WITH_SECRET_WRONG_RESOURCE_TYPE, - context, + reconcilerContext, (Consumer) conditionList -> conditionList .singleElement() .isResolvedRefsFalse( @@ -521,31 +520,31 @@ static List shouldSetResolvedRefs() { } { - Context> context = mockContext(); - mockGetSecret(context, Optional.of(KUBE_TLS_CERT_SECRET)); - mockGetConfigMap(context, Optional.of(PEM_CONFIG_MAP)); - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + Context> reconcilerContext = mockReconcilerContext(); + mockGetSecret(reconcilerContext, Optional.of(KUBE_TLS_CERT_SECRET)); + mockGetConfigMap(reconcilerContext, Optional.of(PEM_CONFIG_MAP)); + when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); + when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); result.add(Arguments.argumentSet("cluster with tls with trust anchor", CLUSTER_TLS_NO_FILTERS_WITH_TRUST_ANCHOR, - context, + reconcilerContext, (Consumer) conditionList -> conditionList .singleElement() .isResolvedRefsTrue())); } { - Context> context = mockContext(); + Context> reconcilerContext = mockReconcilerContext(); - mockGetSecret(context, Optional.of(KUBE_TLS_CERT_SECRET)); - mockGetConfigMap(context, Optional.empty()); - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + mockGetSecret(reconcilerContext, Optional.of(KUBE_TLS_CERT_SECRET)); + mockGetConfigMap(reconcilerContext, Optional.empty()); + when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); + when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); result.add(Arguments.argumentSet("cluster with tls - trust anchor cert configmap not found", CLUSTER_TLS_NO_FILTERS_WITH_TRUST_ANCHOR, - context, + reconcilerContext, (Consumer) conditionList -> conditionList .singleElement() .isResolvedRefsFalse( @@ -554,14 +553,14 @@ static List shouldSetResolvedRefs() { } { - Context> context = mockContext(); + Context> reconcilerContext = mockReconcilerContext(); - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); + when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); result.add(Arguments.argumentSet("cluster defines tls, ingress does not", CLUSTER_TLS_NO_FILTERS, - context, + reconcilerContext, (Consumer) conditionList -> conditionList .singleElement() .isResolvedRefsFalse( @@ -570,14 +569,14 @@ static List shouldSetResolvedRefs() { } { - Context> context = mockContext(); + Context> reconcilerContext = mockReconcilerContext(); - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); + when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); + when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); result.add(Arguments.argumentSet("cluster does not define tls, ingress does", CLUSTER_NO_FILTERS, - context, + reconcilerContext, (Consumer) conditionList -> conditionList .singleElement() .isResolvedRefsFalse( @@ -589,7 +588,7 @@ static List shouldSetResolvedRefs() { } @NonNull - private static Context> mockContext() { + private static Context> mockReconcilerContext() { Context> context = mock(); when(context.managedWorkflowAndDependentResourceContext()).thenReturn(workflowContext); return context; From c69c058cee7bef134428f7cc31f0bb61e13de508 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Wed, 4 Jun 2025 09:43:16 +1200 Subject: [PATCH 60/83] Remove dead code Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../KroxyliciousKafkaClusterRefTemplates.java | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaClusterRefTemplates.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaClusterRefTemplates.java index 6b73bb2aaa..47a1259bd2 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaClusterRefTemplates.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/templates/kroxylicious/KroxyliciousKafkaClusterRefTemplates.java @@ -8,11 +8,8 @@ import io.strimzi.api.kafka.model.kafka.listener.ListenerStatus; -import io.kroxylicious.kubernetes.api.common.TrustAnchorRef; -import io.kroxylicious.kubernetes.api.common.TrustAnchorRefBuilder; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaServiceBuilder; import io.kroxylicious.kubernetes.api.v1alpha1.kafkaservicespec.Tls; -import io.kroxylicious.kubernetes.api.v1alpha1.kafkaservicespec.TlsBuilder; import io.kroxylicious.systemtests.Constants; import io.kroxylicious.systemtests.utils.KafkaUtils; @@ -41,26 +38,6 @@ public static KafkaServiceBuilder defaultKafkaClusterRefCR(String namespaceName, // @formatter:on } - /** - * Kafka cluster ref CR with tls. - * - * @param namespaceName the namespace name - * @param clusterRefName the cluster ref name - * @return the kafka service builder - */ - public static KafkaServiceBuilder kafkaClusterRefCRWithTls(String namespaceName, String clusterRefName) { - final TrustAnchorRef trustAnchorRef = new TrustAnchorRefBuilder() - .withNewRef() - .withName(Constants.KROXYLICIOUS_TLS_CLIENT_CA_CERT) - .withKind(Constants.CONFIG_MAP) - .endRef() - .withKey(Constants.KROXYLICIOUS_TLS_CA_NAME) - .build(); - return kafkaClusterRefCRWithTls(namespaceName, clusterRefName, new TlsBuilder() - .withTrustAnchorRef(trustAnchorRef) - .build()); - } - public static KafkaServiceBuilder kafkaClusterRefCRWithTls(String namespaceName, String clusterRefName, Tls tls) { // @formatter:off return defaultKafkaClusterRefCR(namespaceName, clusterRefName) From 09dc790398fc30f1c3f282a9a3ccf5dcbd0d87ac Mon Sep 17 00:00:00 2001 From: Robert Young Date: Wed, 21 May 2025 12:05:40 +1200 Subject: [PATCH 61/83] Implement LoadBalancer ingress If any referenced KafkaProxyIngress contains a loadBalancer property, we: 1. create a single Service with type LoadBalancer for the proxy. This is to be shared by all LoadBalancer ingresses. 2. add a shared-sni container port to the proxy, 9291. This is the port all of the shared Service ports will target. Then, for each loadbalancer KafkaProxyIngress referred to by a VKC we add a gateway to that VKC's proxy config. The gateway uses SNI and we pass through: - the bootstrapAddress from the KPI custom resource, with the shared-sni container port appended, as this is the address the proxy will listen on. - the advertisedBrokerAddressPattern from the KPI custom resource, with the client-facint Service port appended (currently defaulted to 9083) as this is the port clients will have to connect to. In future the port exposed by the shared Service for a KafkaProxyIngress will be customizable, the intent is that the shared Service will accumulate all the client-facing ports across all ingresses and map them all to the shared-sni port on the proxy container. We have not implemented VKC ingress status for loadbalancer ingresses yet. Signed-off-by: Robert Young Co-authored-by: Keith Wall Signed-off-by: Ujjwal --- .../operator/assertj/ProxyConfigAssert.java | 24 ++ .../00.Namespace.my-proxy.yaml | 15 + .../01.KafkaProxy.simple.yaml | 13 + .../02.KafkaProxyIngress-loadbalancer.yaml | 18 + .../03.KafkaService.my-cluster.yaml | 17 + .../04.VirtualKafkaCluster.my-cluster.yaml | 24 ++ .../05.Issuer.self-signed-issuer.yaml | 14 + .../06.Certificate.server-certificate.yaml | 31 ++ .../examples/loadbalancer-ingress/README.md | 32 ++ .../ClusterServiceDependentResource.java | 60 ++- .../operator/KafkaProxyReconciler.java | 15 +- .../ProxyDeploymentDependentResource.java | 9 +- .../VirtualKafkaClusterReconciler.java | 20 +- ...lusterIPClusterIngressNetworkingModel.java | 2 +- .../ClusterIngressNetworkingModel.java | 20 +- ...BalancerClusterIngressNetworkingModel.java | 70 ++++ .../networking/NetworkPlanningException.java | 15 + .../model/networking/NetworkingPlanner.java | 107 ++++- .../networking/ProxyNetworkingModel.java | 12 +- .../resolver/ProxyResolutionResult.java | 2 +- .../operator/resolver/ResolutionResult.java | 5 + .../operator/KafkaProxyReconcilerIT.java | 226 ++++++++++- .../VirtualKafkaClusterReconcilerTest.java | 1 + ...erIPClusterIngressNetworkingModelTest.java | 8 +- ...ncerClusterIngressNetworkingModelTest.java | 167 ++++++++ .../networking/NetworkingPlannerTest.java | 382 ++++++++++++++++++ .../in-KafkaProxy.yaml | 13 + .../in-KafkaProxyIngress-cluster-ip.yaml | 20 + .../in-KafkaProxyIngress-load-balancer.yaml | 21 + .../in-KafkaService-kafka.yaml | 17 + .../in-Secret-downstream-tls-cert.yaml | 14 + .../in-VirtualKafkaCluster-one.yaml | 31 ++ .../out-ConfigMap-minimal-config-state.yaml | 39 ++ .../out-ConfigMap-minimal-proxy-config.yaml | 53 +++ .../out-Deployment-minimal.yaml | 97 +++++ .../out-Service-minimal-sni.yaml | 36 ++ .../out-Service-one-cluster-ip.yaml | 53 +++ .../in-KafkaProxy.yaml | 13 + .../in-KafkaProxyIngress-load-balancer.yaml | 21 + .../in-KafkaService-barref.yaml | 17 + .../in-KafkaService-fooref.yaml | 17 + .../in-Secret-downstream-tls-cert.yaml | 14 + .../in-VirtualKafkaCluster-bar.yaml | 26 ++ .../in-VirtualKafkaCluster-foo.yaml | 33 ++ ...out-ConfigMap-twocluster-config-state.yaml | 57 +++ ...out-ConfigMap-twocluster-proxy-config.yaml | 41 ++ .../out-Deployment-twocluster.yaml | 89 ++++ .../out-Service-twocluster-sni.yaml | 36 ++ .../loadbalancer-ingress/in-KafkaProxy.yaml | 13 + .../in-KafkaProxyIngress-load-balancer.yaml | 21 + .../in-KafkaService-kafka.yaml | 17 + .../in-Secret-downstream-tls-cert.yaml | 14 + .../in-VirtualKafkaCluster-one.yaml | 26 ++ .../out-ConfigMap-minimal-config-state.yaml | 39 ++ .../out-ConfigMap-minimal-proxy-config.yaml | 41 ++ .../out-Deployment-minimal.yaml | 89 ++++ .../out-Service-minimal-sni.yaml | 36 ++ .../in-KafkaProxy.yaml | 13 + .../in-KafkaProxyIngress-load-balancer.yaml | 21 + .../in-KafkaService-barref.yaml | 17 + .../in-KafkaService-fooref.yaml | 17 + .../in-Secret-downstream-tls-cert.yaml | 14 + .../in-VirtualKafkaCluster-bar.yaml | 26 ++ .../in-VirtualKafkaCluster-foo.yaml | 26 ++ ...out-ConfigMap-twocluster-config-state.yaml | 56 +++ ...out-ConfigMap-twocluster-proxy-config.yaml | 53 +++ .../out-Deployment-twocluster.yaml | 89 ++++ .../out-Service-twocluster-sni.yaml | 36 ++ 68 files changed, 2682 insertions(+), 49 deletions(-) create mode 100644 kroxylicious-operator/examples/loadbalancer-ingress/00.Namespace.my-proxy.yaml create mode 100644 kroxylicious-operator/examples/loadbalancer-ingress/01.KafkaProxy.simple.yaml create mode 100644 kroxylicious-operator/examples/loadbalancer-ingress/02.KafkaProxyIngress-loadbalancer.yaml create mode 100644 kroxylicious-operator/examples/loadbalancer-ingress/03.KafkaService.my-cluster.yaml create mode 100644 kroxylicious-operator/examples/loadbalancer-ingress/04.VirtualKafkaCluster.my-cluster.yaml create mode 100644 kroxylicious-operator/examples/loadbalancer-ingress/05.Issuer.self-signed-issuer.yaml create mode 100644 kroxylicious-operator/examples/loadbalancer-ingress/06.Certificate.server-certificate.yaml create mode 100644 kroxylicious-operator/examples/loadbalancer-ingress/README.md create mode 100644 kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/LoadBalancerClusterIngressNetworkingModel.java create mode 100644 kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/NetworkPlanningException.java create mode 100644 kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/LoadBalancerClusterIngressNetworkingModelTest.java create mode 100644 kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/NetworkingPlannerTest.java create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-KafkaProxy.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-KafkaProxyIngress-cluster-ip.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-KafkaProxyIngress-load-balancer.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-KafkaService-kafka.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-Secret-downstream-tls-cert.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-VirtualKafkaCluster-one.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-ConfigMap-minimal-config-state.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-ConfigMap-minimal-proxy-config.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Deployment-minimal.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-minimal-sni.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-one-cluster-ip.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-KafkaProxy.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-KafkaProxyIngress-load-balancer.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-KafkaService-barref.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-KafkaService-fooref.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-Secret-downstream-tls-cert.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-VirtualKafkaCluster-bar.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-VirtualKafkaCluster-foo.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/out-ConfigMap-twocluster-config-state.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/out-ConfigMap-twocluster-proxy-config.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/out-Deployment-twocluster.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/out-Service-twocluster-sni.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/in-KafkaProxy.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/in-KafkaProxyIngress-load-balancer.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/in-KafkaService-kafka.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/in-Secret-downstream-tls-cert.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/in-VirtualKafkaCluster-one.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/out-ConfigMap-minimal-config-state.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/out-ConfigMap-minimal-proxy-config.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/out-Deployment-minimal.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/out-Service-minimal-sni.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-KafkaProxy.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-KafkaProxyIngress-load-balancer.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-KafkaService-barref.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-KafkaService-fooref.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-Secret-downstream-tls-cert.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-VirtualKafkaCluster-bar.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-VirtualKafkaCluster-foo.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/out-ConfigMap-twocluster-config-state.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/out-ConfigMap-twocluster-proxy-config.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/out-Deployment-twocluster.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/out-Service-twocluster-sni.yaml diff --git a/kroxylicious-operator-test-support/src/main/java/io/kroxylicious/kubernetes/operator/assertj/ProxyConfigAssert.java b/kroxylicious-operator-test-support/src/main/java/io/kroxylicious/kubernetes/operator/assertj/ProxyConfigAssert.java index 503559eac3..22be8f85da 100644 --- a/kroxylicious-operator-test-support/src/main/java/io/kroxylicious/kubernetes/operator/assertj/ProxyConfigAssert.java +++ b/kroxylicious-operator-test-support/src/main/java/io/kroxylicious/kubernetes/operator/assertj/ProxyConfigAssert.java @@ -16,6 +16,7 @@ import io.kroxylicious.proxy.config.Configuration; import io.kroxylicious.proxy.config.NamedRange; import io.kroxylicious.proxy.config.PortIdentifiesNodeIdentificationStrategy; +import io.kroxylicious.proxy.config.SniHostIdentifiesNodeIdentificationStrategy; import io.kroxylicious.proxy.config.VirtualCluster; import io.kroxylicious.proxy.config.VirtualClusterGateway; import io.kroxylicious.proxy.service.HostPort; @@ -69,6 +70,29 @@ public ProxyConfigPortIdentifiesNodeGatewayAssert portIdentifiesNode() { return new ProxyConfigPortIdentifiesNodeGatewayAssert(actual.portIdentifiesNode()); } + public ProxyConfigSniHostIdentifiesNodeGatewayAssert sniHostIdentifiesNode() { + Assertions.assertThat(actual.sniHostIdentifiesNode()).isNotNull(); + return new ProxyConfigSniHostIdentifiesNodeGatewayAssert(actual.sniHostIdentifiesNode()); + } + + } + + public static class ProxyConfigSniHostIdentifiesNodeGatewayAssert + extends AbstractObjectAssert { + + public ProxyConfigSniHostIdentifiesNodeGatewayAssert(SniHostIdentifiesNodeIdentificationStrategy sniHostIdentifiesNodeIdentificationStrategy) { + super(sniHostIdentifiesNodeIdentificationStrategy, ProxyConfigSniHostIdentifiesNodeGatewayAssert.class); + } + + public ProxyConfigSniHostIdentifiesNodeGatewayAssert hasBootstrapAddress(String bootstrapAddress) { + Assertions.assertThat(actual.bootstrapAddress()).isEqualTo(bootstrapAddress); + return this; + } + + public ProxyConfigSniHostIdentifiesNodeGatewayAssert hasAdvertisedBrokerAddressPattern(String advertisedBrokerAddressPattern) { + Assertions.assertThat(actual.advertisedBrokerAddressPattern()).isEqualTo(advertisedBrokerAddressPattern); + return this; + } } public static class ProxyConfigPortIdentifiesNodeGatewayAssert diff --git a/kroxylicious-operator/examples/loadbalancer-ingress/00.Namespace.my-proxy.yaml b/kroxylicious-operator/examples/loadbalancer-ingress/00.Namespace.my-proxy.yaml new file mode 100644 index 0000000000..3acfe85dfa --- /dev/null +++ b/kroxylicious-operator/examples/loadbalancer-ingress/00.Namespace.my-proxy.yaml @@ -0,0 +1,15 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +# A Namespace for a single KafkaProxy instance +apiVersion: v1 +kind: Namespace +metadata: + name: my-proxy + labels: + app.kubernetes.io/name: kroxylicious + app.kubernetes.io/component: proxy \ No newline at end of file diff --git a/kroxylicious-operator/examples/loadbalancer-ingress/01.KafkaProxy.simple.yaml b/kroxylicious-operator/examples/loadbalancer-ingress/01.KafkaProxy.simple.yaml new file mode 100644 index 0000000000..a913d39363 --- /dev/null +++ b/kroxylicious-operator/examples/loadbalancer-ingress/01.KafkaProxy.simple.yaml @@ -0,0 +1,13 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: KafkaProxy +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: simple + namespace: my-proxy +spec: {} diff --git a/kroxylicious-operator/examples/loadbalancer-ingress/02.KafkaProxyIngress-loadbalancer.yaml b/kroxylicious-operator/examples/loadbalancer-ingress/02.KafkaProxyIngress-loadbalancer.yaml new file mode 100644 index 0000000000..eed5583584 --- /dev/null +++ b/kroxylicious-operator/examples/loadbalancer-ingress/02.KafkaProxyIngress-loadbalancer.yaml @@ -0,0 +1,18 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: KafkaProxyIngress +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: loadbalancer + namespace: my-proxy +spec: + proxyRef: + name: simple + loadBalancer: + bootstrapAddress: '$(virtualClusterName).kafkaproxy' + advertisedBrokerAddressPattern: '$(virtualClusterName)-$(nodeId).kafkaproxy' \ No newline at end of file diff --git a/kroxylicious-operator/examples/loadbalancer-ingress/03.KafkaService.my-cluster.yaml b/kroxylicious-operator/examples/loadbalancer-ingress/03.KafkaService.my-cluster.yaml new file mode 100644 index 0000000000..8b08413352 --- /dev/null +++ b/kroxylicious-operator/examples/loadbalancer-ingress/03.KafkaService.my-cluster.yaml @@ -0,0 +1,17 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: KafkaService +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: my-cluster + namespace: my-proxy +spec: + bootstrapServers: my-cluster-kafka-bootstrap.kafka.svc.cluster.local:9092 + nodeIdRanges: + - start: 0 + end: 2 diff --git a/kroxylicious-operator/examples/loadbalancer-ingress/04.VirtualKafkaCluster.my-cluster.yaml b/kroxylicious-operator/examples/loadbalancer-ingress/04.VirtualKafkaCluster.my-cluster.yaml new file mode 100644 index 0000000000..63dde8a8e9 --- /dev/null +++ b/kroxylicious-operator/examples/loadbalancer-ingress/04.VirtualKafkaCluster.my-cluster.yaml @@ -0,0 +1,24 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: VirtualKafkaCluster +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: my-cluster + namespace: my-proxy +spec: + proxyRef: + name: simple + targetKafkaServiceRef: + name: my-cluster + ingresses: + - ingressRef: + name: loadbalancer + tls: + certificateRef: + name: server-certificate + kind: Secret diff --git a/kroxylicious-operator/examples/loadbalancer-ingress/05.Issuer.self-signed-issuer.yaml b/kroxylicious-operator/examples/loadbalancer-ingress/05.Issuer.self-signed-issuer.yaml new file mode 100644 index 0000000000..cf3dccea21 --- /dev/null +++ b/kroxylicious-operator/examples/loadbalancer-ingress/05.Issuer.self-signed-issuer.yaml @@ -0,0 +1,14 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: my-proxy +spec: + selfSigned: {} \ No newline at end of file diff --git a/kroxylicious-operator/examples/loadbalancer-ingress/06.Certificate.server-certificate.yaml b/kroxylicious-operator/examples/loadbalancer-ingress/06.Certificate.server-certificate.yaml new file mode 100644 index 0000000000..5cda6a2901 --- /dev/null +++ b/kroxylicious-operator/examples/loadbalancer-ingress/06.Certificate.server-certificate.yaml @@ -0,0 +1,31 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: server-certificate + namespace: my-proxy +spec: + commonName: my-cluster.kafkaproxy + secretName: server-certificate + privateKey: + algorithm: RSA + encoding: PKCS8 + size: 4096 + dnsNames: + - my-cluster.kafkaproxy + - my-cluster-0.kafkaproxy + - my-cluster-1.kafkaproxy + - my-cluster-2.kafkaproxy + - my-cluster-3.kafkaproxy + usages: + - server auth + issuerRef: + name: selfsigned-issuer + kind: Issuer + group: cert-manager.io diff --git a/kroxylicious-operator/examples/loadbalancer-ingress/README.md b/kroxylicious-operator/examples/loadbalancer-ingress/README.md new file mode 100644 index 0000000000..24b3244c80 --- /dev/null +++ b/kroxylicious-operator/examples/loadbalancer-ingress/README.md @@ -0,0 +1,32 @@ +These manifests declare a minimal proxy which simply forwards all requests to a single backend Apache Kafka cluster without doing anything clever either in terms of networking of protocol filters. +In this example, the downstream connection (that is, between the Kafka client and the proxy) uses TLS. + +For the purposes of this example we use +* an in-cluster Apache Kafka provided by [Strimzi](https://strimzi.io/) +* cert manager to create a server certificate which is signed by a self-signed issuer. + +To try this example out: +1. Install kubectl +2. `cd` to this directory +3. Apply cert-manager + ```shell + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.17.2/cert-manager.yaml + kubectl wait deployment/cert-manager-webhook --for=condition=Available=True --timeout=300s -n cert-manager + ``` +4. Apply the example + ```shell + kubectl apply -f . + ``` +5. Start up a `minikube tunnel` in another terminal +6. Wait for external ip to be allocated to service `my-cluster-sni`. + ```shell + oc get service -n my-proxy simple-sni -o jsonpath='{.status.loadBalancer.ingress[0].ip}' --watch + ``` +7. Create an `/etc/hosts` entry like `10.106.236.189 my-cluster.kafkaproxy my-cluster-0.kafkaproxy my-cluster-1.kafkaproxy my-cluster-2.kafkaproxy` where 10.106.236.189 is replaced with the allocated IP. +8. Try producing and consuming some messages with commands like this (obtain a kafka distribution): + ``` + CA=$(kubectl get secret -n my-proxy server-certificate -o json | jq -r ".data.\"ca.crt\" | @base64d") + ./bin/kafka-console-producer.sh --bootstrap-server my-cluster.kafkaproxy:9083 --topic mytopic --producer-property ssl.truststore.type=PEM --producer-property security.protocol=SSL --producer-property ssl.truststore.certificates="${CA}" + ./bin/kafka-console-consumer.sh --bootstrap-server my-cluster.kafkaproxy:9083 --topic mytopic --from-beginning --consumer-property ssl.truststore.type=PEM --consumer-property security.protocol=SSL --consumer-property ssl.truststore.certificates="${CA}" + ``` + diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ClusterServiceDependentResource.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ClusterServiceDependentResource.java index a56e893dce..d9961e4958 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ClusterServiceDependentResource.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ClusterServiceDependentResource.java @@ -5,12 +5,17 @@ */ package io.kroxylicious.kubernetes.operator; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Stream; +import io.fabric8.kubernetes.api.model.IntOrString; +import io.fabric8.kubernetes.api.model.ObjectMeta; +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.ServiceBuilder; import io.javaoperatorsdk.operator.api.reconciler.Context; import io.javaoperatorsdk.operator.processing.dependent.BulkDependentResource; import io.javaoperatorsdk.operator.processing.dependent.kubernetes.CRUDKubernetesDependentResource; @@ -21,6 +26,9 @@ import io.kroxylicious.kubernetes.operator.model.networking.ProxyNetworkingModel; import io.kroxylicious.kubernetes.operator.resolver.ClusterResolutionResult; +import static io.kroxylicious.kubernetes.operator.Labels.standardLabels; +import static io.kroxylicious.kubernetes.operator.ProxyDeploymentDependentResource.SHARED_SNI_PORT; +import static io.kroxylicious.kubernetes.operator.ResourcesUtil.namespace; import static io.kroxylicious.kubernetes.operator.ResourcesUtil.toByNameMap; /** @@ -51,13 +59,55 @@ public Map desiredResources( Context context) { KafkaProxyContext kafkaProxyContext = KafkaProxyContext.proxyContext(context); var model = kafkaProxyContext.model(); - Stream serviceStream = model.clustersWithValidNetworking().stream() + var clusterNetworkingModels = model.clustersWithValidNetworking().stream() .map(ClusterResolutionResult::cluster) .filter(cluster -> !kafkaProxyContext.isBroken(cluster)) - .flatMap(cluster -> model.networkingModel().clusterIngressModel(cluster) - .map(ProxyNetworkingModel.ClusterNetworkingModel::services) - .orElse(Stream.empty())); - return serviceStream.collect(toByNameMap()); + .flatMap(cluster -> model.networkingModel().clusterIngressModel(cluster).stream()) + .toList(); + + var serviceStream = clusterNetworkingModels.stream() + .flatMap(ProxyNetworkingModel.ClusterNetworkingModel::services); + + var sharedSniLoadbalancerPorts = clusterNetworkingModels.stream() + .flatMap(ProxyNetworkingModel.ClusterNetworkingModel::requiredSniLoadbalancerPorts) + .distinct().sorted().toList(); + + var sniServiceStream = sniLoadbalancerServices(primary, sharedSniLoadbalancerPorts); + + return Stream.concat(serviceStream, sniServiceStream).collect(toByNameMap()); + } + + private ObjectMeta serviceMetadata(KafkaProxy primary, String name) { + return new ObjectMetaBuilder() + .withName(name) + .withNamespace(namespace(primary)) + .addToLabels(standardLabels(primary)) + .addNewOwnerReferenceLike(ResourcesUtil.newOwnerReferenceTo(primary)).endOwnerReference() + .build(); + } + + private Stream sniLoadbalancerServices(KafkaProxy primary, List loadBalancerPorts) { + if (loadBalancerPorts.isEmpty()) { + return Stream.empty(); + } + else { + String serviceName = ResourcesUtil.name(primary) + "-sni"; + var serviceSpecBuilder = new ServiceBuilder() + .withMetadata(serviceMetadata(primary, serviceName)) + .withNewSpec() + .withType("LoadBalancer") + .withSelector(ProxyDeploymentDependentResource.podLabels(primary)); + for (Integer loadBalancerPort : loadBalancerPorts) { + serviceSpecBuilder = serviceSpecBuilder + .addNewPort() + .withName("sni-" + loadBalancerPort) + .withPort(loadBalancerPort) + .withTargetPort(new IntOrString(SHARED_SNI_PORT)) + .withProtocol("TCP") + .endPort(); + } + return Stream.of(serviceSpecBuilder.endSpec().build()); + } } @Override diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/KafkaProxyReconciler.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/KafkaProxyReconciler.java index 1082d4dd84..8e84c4302b 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/KafkaProxyReconciler.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/KafkaProxyReconciler.java @@ -71,6 +71,7 @@ import io.kroxylicious.proxy.config.NamedFilterDefinition; import io.kroxylicious.proxy.config.NodeIdentificationStrategy; import io.kroxylicious.proxy.config.PortIdentifiesNodeIdentificationStrategy; +import io.kroxylicious.proxy.config.SniHostIdentifiesNodeIdentificationStrategy; import io.kroxylicious.proxy.config.TargetCluster; import io.kroxylicious.proxy.config.VirtualCluster; import io.kroxylicious.proxy.config.VirtualClusterGateway; @@ -287,16 +288,22 @@ public static ConfigurationFragment buildVirtualClusterGa var volumes = tlsConfigFragment.volumes(); var mounts = tlsConfigFragment.mounts(); + PortIdentifiesNodeIdentificationStrategy portIdentifiesNode = null; + SniHostIdentifiesNodeIdentificationStrategy sniHostIdentifiesNode = null; NodeIdentificationStrategy nodeIdentificationStrategy = gateway.nodeIdentificationStrategy(); if (nodeIdentificationStrategy instanceof PortIdentifiesNodeIdentificationStrategy port) { - return new ConfigurationFragment<>(new VirtualClusterGateway(name(gateway.ingress()), - port, - null, - tlsConfigFragment.fragment()), volumes, mounts); + portIdentifiesNode = port; + } + else if (nodeIdentificationStrategy instanceof SniHostIdentifiesNodeIdentificationStrategy sniHost) { + sniHostIdentifiesNode = sniHost; } else { throw new IllegalStateException("Unsupported node identification strategy: " + nodeIdentificationStrategy); } + return new ConfigurationFragment<>(new VirtualClusterGateway(name(gateway.ingress()), + portIdentifiesNode, + sniHostIdentifiesNode, + tlsConfigFragment.fragment()), volumes, mounts); } private static ConfigurationFragment> buildTlsFragment(io.kroxylicious.kubernetes.api.v1alpha1.virtualkafkaclusterspec.ingresses.Tls ingressTls) { diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ProxyDeploymentDependentResource.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ProxyDeploymentDependentResource.java index c24c066af4..172c812a48 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ProxyDeploymentDependentResource.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ProxyDeploymentDependentResource.java @@ -11,6 +11,7 @@ import java.util.Map; import java.util.Optional; import java.util.Properties; +import java.util.concurrent.atomic.AtomicBoolean; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,6 +56,7 @@ public class ProxyDeploymentDependentResource private static final int MANAGEMENT_PORT = 9190; private static final String MANAGEMENT_PORT_NAME = "management"; public static final int PROXY_PORT_START = 9292; + public static final int SHARED_SNI_PORT = 9291; private final String kroxyliciousImage = getOperandImage(); static final String KROXYLICIOUS_IMAGE_ENV_VAR = "KROXYLICIOUS_IMAGE"; @@ -199,15 +201,20 @@ private Container proxyContainer(KafkaProxyContext kafkaProxyContext, .withContainerPort(MANAGEMENT_PORT) .withName(MANAGEMENT_PORT_NAME) .endPort(); + // @formatter:on // broker ports + AtomicBoolean anyIngressRequiresSharedSniPort = new AtomicBoolean(false); clusterResolutionResults.forEach(resolutionResult -> { if (!kafkaProxyContext.isBroken(resolutionResult.cluster())) { ProxyNetworkingModel.ClusterNetworkingModel clusterNetworkingModel = ingressModel.clusterIngressModel(resolutionResult.cluster()).orElseThrow(); clusterNetworkingModel.registerProxyContainerPorts(containerBuilder::addToPorts); + anyIngressRequiresSharedSniPort.compareAndExchange(false, clusterNetworkingModel.anyIngressRequiresSharedSniPort()); } }); + if (anyIngressRequiresSharedSniPort.get()) { + containerBuilder.addNewPort().withContainerPort(SHARED_SNI_PORT).withName("shared-sni-port").endPort(); + } return containerBuilder.build(); - // @formatter:on } @VisibleForTesting diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java index c22ab2c02a..64dcf860eb 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java @@ -44,11 +44,11 @@ import io.kroxylicious.kubernetes.api.common.TrustAnchorRef; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxy; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngress; -import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngressSpec; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaService; import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaCluster; import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaClusterSpec; import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.LoadBalancer; import io.kroxylicious.kubernetes.api.v1alpha1.virtualkafkaclusterspec.Ingresses; import io.kroxylicious.kubernetes.api.v1alpha1.virtualkafkaclusterspec.ingresses.Tls; import io.kroxylicious.kubernetes.api.v1alpha1.virtualkafkaclusterstatus.Ingresses.Protocol; @@ -585,11 +585,7 @@ private VirtualKafkaCluster checkTlsConfigConsistency(Context proxyContainerPorts() { + public Stream identifyingProxyContainerPorts() { Stream bootstrapPort = Stream.of(new ContainerPortBuilder().withContainerPort(firstIdentifyingPort) .withName(firstIdentifyingPort + "-bootstrap").build()); int nodeCount = lastIdentifyingPort - firstIdentifyingPort; diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/ClusterIngressNetworkingModel.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/ClusterIngressNetworkingModel.java index 7b5ddf52d6..c754d61d28 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/ClusterIngressNetworkingModel.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/ClusterIngressNetworkingModel.java @@ -52,10 +52,12 @@ public interface ClusterIngressNetworkingModel { Stream services(); /** - * ContainerPorts to be added to the proxy for this model + * ContainerPorts to be added to the proxy for this model. These ports will be used to uniquely + * identify an upstream node, so they cannot be shared with other ClusterIngressNetworkingModel + * instances. * @return a stream of ContainerPorts */ - Stream proxyContainerPorts(); + Stream identifyingProxyContainerPorts(); /** * The node identification strategy to be injected into the Proxy Config for this model @@ -67,4 +69,18 @@ public interface ClusterIngressNetworkingModel { * The downstream TLS to be injected into the Proxy Config for this model, if available */ Optional downstreamTls(); + + /** + * @return true if this cluster ingress requires a shared SNI port in the proxy container to be provided + */ + default boolean requiresSharedSniContainerPort() { + return false; + } + + /** + * @return the client facing ports that should be exposed on the shared SNI loadbalancer service + */ + default Stream requiredSniLoadBalancerServicePorts() { + return Stream.empty(); + } } diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/LoadBalancerClusterIngressNetworkingModel.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/LoadBalancerClusterIngressNetworkingModel.java new file mode 100644 index 0000000000..407693600b --- /dev/null +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/LoadBalancerClusterIngressNetworkingModel.java @@ -0,0 +1,70 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.kubernetes.operator.model.networking; + +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Stream; + +import io.fabric8.kubernetes.api.model.ContainerPort; +import io.fabric8.kubernetes.api.model.ServiceBuilder; + +import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngress; +import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaCluster; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.LoadBalancer; +import io.kroxylicious.kubernetes.api.v1alpha1.virtualkafkaclusterspec.ingresses.Tls; +import io.kroxylicious.proxy.config.NodeIdentificationStrategy; +import io.kroxylicious.proxy.config.SniHostIdentifiesNodeIdentificationStrategy; +import io.kroxylicious.proxy.service.HostPort; + +public record LoadBalancerClusterIngressNetworkingModel(VirtualKafkaCluster cluster, + KafkaProxyIngress ingress, + LoadBalancer loadBalancer, + Tls tls, + int sharedSniPort) + implements ClusterIngressNetworkingModel { + + public LoadBalancerClusterIngressNetworkingModel { + Objects.requireNonNull(cluster); + Objects.requireNonNull(ingress); + Objects.requireNonNull(loadBalancer); + Objects.requireNonNull(tls); + } + + public static final int DEFAULT_LOADBALANCER_PORT = 9083; + + @Override + public Stream services() { + return Stream.empty(); + } + + @Override + public Stream identifyingProxyContainerPorts() { + return Stream.empty(); + } + + @Override + public NodeIdentificationStrategy nodeIdentificationStrategy() { + return new SniHostIdentifiesNodeIdentificationStrategy(new HostPort(loadBalancer.getBootstrapAddress(), sharedSniPort).toString(), + new HostPort(loadBalancer.getAdvertisedBrokerAddressPattern(), DEFAULT_LOADBALANCER_PORT).toString()); + } + + @Override + public Optional downstreamTls() { + return Optional.of(tls); + } + + @Override + public boolean requiresSharedSniContainerPort() { + return true; + } + + @Override + public Stream requiredSniLoadBalancerServicePorts() { + return Stream.of(DEFAULT_LOADBALANCER_PORT); + } +} diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/NetworkPlanningException.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/NetworkPlanningException.java new file mode 100644 index 0000000000..1cef20fd65 --- /dev/null +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/NetworkPlanningException.java @@ -0,0 +1,15 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.kubernetes.operator.model.networking; + +public class NetworkPlanningException extends RuntimeException { + + public NetworkPlanningException(String message) { + super(message); + } + +} diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/NetworkingPlanner.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/NetworkingPlanner.java index 2a1302f6a2..1459f5ed9b 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/NetworkingPlanner.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/NetworkingPlanner.java @@ -12,22 +12,29 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxy; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngress; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaService; import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaCluster; import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.LoadBalancer; import io.kroxylicious.kubernetes.api.v1alpha1.kafkaservicespec.NodeIdRanges; import io.kroxylicious.kubernetes.api.v1alpha1.kafkaservicespec.NodeIdRangesBuilder; import io.kroxylicious.kubernetes.api.v1alpha1.virtualkafkaclusterspec.ingresses.Tls; import io.kroxylicious.kubernetes.operator.model.networking.ProxyNetworkingModel.ClusterIngressNetworkingModelResult; import io.kroxylicious.kubernetes.operator.model.networking.ProxyNetworkingModel.ClusterNetworkingModel; import io.kroxylicious.kubernetes.operator.resolver.ClusterResolutionResult; +import io.kroxylicious.kubernetes.operator.resolver.IngressResolutionResult; import io.kroxylicious.kubernetes.operator.resolver.ProxyResolutionResult; +import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import static io.kroxylicious.kubernetes.operator.ProxyDeploymentDependentResource.PROXY_PORT_START; +import static io.kroxylicious.kubernetes.operator.ProxyDeploymentDependentResource.SHARED_SNI_PORT; import static io.kroxylicious.kubernetes.operator.ResourcesUtil.name; /** @@ -46,6 +53,8 @@ private NetworkingPlanner() { private static final List DEFAULT_NODE_ID_RANGES = List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()); + private static final Logger LOGGER = LoggerFactory.getLogger(NetworkingPlanner.class); + /** * Allocates a ProxyNetworkingModel. The aim is to deterministically produce a model of the ports that will * be used by the proxy container. We want this to be as stable as possible, so we will allocate ports @@ -74,6 +83,7 @@ private static ClusterNetworkingModel planClusterNetworking(KafkaProxy primary, int toAllocate = networkingDefinition.numIdentifyingPortsRequired(); Integer firstIdentifyingPort = null; Integer lastIdentifyingPort = null; + Integer sharedSniPort = null; IngressConflictException exception = null; if (toAllocate != 0) { if (identifyingPorts.get() != PROXY_PORT_START) { @@ -84,7 +94,10 @@ private static ClusterNetworkingModel planClusterNetworking(KafkaProxy primary, firstIdentifyingPort = identifyingPorts.get(); lastIdentifyingPort = identifyingPorts.addAndGet(toAllocate) - 1; } - ClusterIngressNetworkingModel networkingModel = networkingDefinition.createNetworkingModel(firstIdentifyingPort, lastIdentifyingPort); + if (networkingDefinition.requiresSharedSniPort()) { + sharedSniPort = SHARED_SNI_PORT; + } + ClusterIngressNetworkingModel networkingModel = networkingDefinition.createNetworkingModel(firstIdentifyingPort, lastIdentifyingPort, sharedSniPort); return new ClusterIngressNetworkingModelResult(networkingModel, exception); }).toList(); return new ClusterNetworkingModel(clusterResolutionResult.cluster(), ingressResults); @@ -97,32 +110,48 @@ static Stream planClusterIngressNetworkingDe .flatMap( ingressResolutionResult -> { Optional maybeIngress = ingressResolutionResult.ingressResolutionResult().maybeReferentResource(); - if (maybeIngress.isPresent()) { - Optional maybeService = clusterResolutionResult.serviceResolutionResult().maybeReferentResource(); - // todo, maybe we should not include the case where the service does not resolve, it's optimistic to assume its using the default node id range - List nodeIdRanges = maybeService.map(s -> s.getSpec().getNodeIdRanges()).orElse(DEFAULT_NODE_ID_RANGES); - return Stream.of( - clusterIngressNetworkingDefinition(primary, cluster, maybeIngress.get(), nodeIdRanges, - ingressResolutionResult.ingress().getTls())); - } - else { - // skip unresolved ingresses - return Stream.empty(); - } + Optional maybeService = clusterResolutionResult.serviceResolutionResult().maybeReferentResource(); + // skip if ingress or service unresolved + return maybeIngress.map(ingress -> planClusterIngressNetworkingDefinition(primary, ingressResolutionResult, ingress, maybeService, cluster)) + .orElseGet(Stream::empty); }); } + @NonNull + private static Stream planClusterIngressNetworkingDefinition(KafkaProxy primary, + IngressResolutionResult ingressResolutionResult, + KafkaProxyIngress ingress, + Optional service, + VirtualKafkaCluster cluster) { + // todo we should change this so that we skip if KafkaService is not resolved + List nodeIdRanges = service.map(s -> s.getSpec().getNodeIdRanges()).orElse(DEFAULT_NODE_ID_RANGES); + try { + ClusterIngressNetworkingDefinition definition = clusterIngressNetworkingDefinition(primary, cluster, ingress, nodeIdRanges, + ingressResolutionResult.ingress().getTls()); + return Stream.of( + definition); + } + catch (NetworkPlanningException e) { + LOGGER.warn("skipping ingress {} for cluster {} due to planning exception", name(ingress), name(cluster), e); + return Stream.empty(); + } + } + private static ClusterIngressNetworkingDefinition clusterIngressNetworkingDefinition(KafkaProxy primary, VirtualKafkaCluster cluster, KafkaProxyIngress ingress, List nodeIdRanges, @Nullable Tls tls) { ClusterIP clusterIP = ingress.getSpec().getClusterIP(); + LoadBalancer loadBalancer = ingress.getSpec().getLoadBalancer(); if (clusterIP != null) { return new ClusterIPClusterIngressNetworkingDefinition(ingress, cluster, primary, nodeIdRanges, tls); } + else if (loadBalancer != null) { + return new LoadBalancerClusterIngressNetworkingDefinition(ingress, cluster, loadBalancer, tls); + } else { - throw new IllegalStateException("ingress must have clusterIP specified"); + throw new NetworkPlanningException("ingress must have clusterIP or loadBalancer specified"); } } @@ -149,9 +178,11 @@ private interface ClusterIngressNetworkingDefinition { * * @param firstIdentifyingPort the first identifying port allocated to this Ingress * @param lastIdentifyingPort the last identifying port (inclusive) allocated to this Ingress + * @param sharedSniPort the shared SNI port (if definition required SNI port) * @return a non-null ClusterIngressNetworkingModel */ - ClusterIngressNetworkingModel createNetworkingModel(@Nullable Integer firstIdentifyingPort, @Nullable Integer lastIdentifyingPort); + ClusterIngressNetworkingModel createNetworkingModel(@Nullable Integer firstIdentifyingPort, @Nullable Integer lastIdentifyingPort, + @Nullable Integer sharedSniPort); /** * Some Ingress strategies require a set of ports in the proxy pod to be unique and exclusive so that the Proxy @@ -159,7 +190,13 @@ private interface ClusterIngressNetworkingDefinition { * * @return the number of identifying ports this ingress requires */ - int numIdentifyingPortsRequired(); + default int numIdentifyingPortsRequired() { + return 0; + } + + default boolean requiresSharedSniPort() { + return false; + } } private record ClusterIPClusterIngressNetworkingDefinition( @@ -171,9 +208,10 @@ private record ClusterIPClusterIngressNetworkingDefinition( implements ClusterIngressNetworkingDefinition { @Override - public ClusterIngressNetworkingModel createNetworkingModel(@Nullable Integer firstIdentifyingPort, @Nullable Integer lastIdentifyingPort) { - Objects.requireNonNull(firstIdentifyingPort); - Objects.requireNonNull(lastIdentifyingPort); + public ClusterIngressNetworkingModel createNetworkingModel(@Nullable Integer firstIdentifyingPort, @Nullable Integer lastIdentifyingPort, + @Nullable Integer sharedSniPort) { + validateNotNull(firstIdentifyingPort, "firstIdentifyingPort must be non null for ClusterIP ingress"); + validateNotNull(lastIdentifyingPort, "lastIdentifyingPort must be non null for ClusterIP ingress"); return new ClusterIPClusterIngressNetworkingModel(primary, cluster, ingress, nodeIdRanges, tls, firstIdentifyingPort, lastIdentifyingPort); } @@ -183,4 +221,35 @@ public int numIdentifyingPortsRequired() { } } + + private record LoadBalancerClusterIngressNetworkingDefinition( + KafkaProxyIngress ingress, + VirtualKafkaCluster cluster, + LoadBalancer loadBalancer, + Tls tls) + implements ClusterIngressNetworkingDefinition { + + private LoadBalancerClusterIngressNetworkingDefinition { + validateNotNull(tls, "LoadBalancer requires TLS to be provided by the virtualkafkacluster"); + } + + @Override + public ClusterIngressNetworkingModel createNetworkingModel(@Nullable Integer firstIdentifyingPort, @Nullable Integer lastIdentifyingPort, + @Nullable Integer sharedSniPort) { + Objects.requireNonNull(sharedSniPort); + return new LoadBalancerClusterIngressNetworkingModel(cluster, ingress, loadBalancer, tls, sharedSniPort); + } + + @Override + public boolean requiresSharedSniPort() { + return true; + } + } + + private static void validateNotNull(@Nullable Object object, String message) { + if (object == null) { + throw new NetworkPlanningException(message); + } + } + } diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/ProxyNetworkingModel.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/ProxyNetworkingModel.java index d98056d629..388bfb4ead 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/ProxyNetworkingModel.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/ProxyNetworkingModel.java @@ -69,6 +69,16 @@ public void registerProxyContainerPorts(Consumer portConsumer) { result.proxyContainerPorts().forEach(portConsumer); }); } + + public boolean anyIngressRequiresSharedSniPort() { + return clusterIngressNetworkingModelResults.stream() + .anyMatch(ingressModelResult -> ingressModelResult.clusterIngressNetworkingModel().requiresSharedSniContainerPort()); + } + + public Stream requiredSniLoadbalancerPorts() { + return clusterIngressNetworkingModelResults.stream() + .flatMap(ingressModelResult -> ingressModelResult.clusterIngressNetworkingModel().requiredSniLoadBalancerServicePorts()); + } } /** @@ -79,7 +89,7 @@ public void registerProxyContainerPorts(Consumer portConsumer) { public record ClusterIngressNetworkingModelResult(ClusterIngressNetworkingModel clusterIngressNetworkingModel, @Nullable IngressConflictException exception) { public Stream proxyContainerPorts() { - return clusterIngressNetworkingModel.proxyContainerPorts(); + return clusterIngressNetworkingModel.identifyingProxyContainerPorts(); } } diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/resolver/ProxyResolutionResult.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/resolver/ProxyResolutionResult.java index 0ab5086d96..6aebf097b7 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/resolver/ProxyResolutionResult.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/resolver/ProxyResolutionResult.java @@ -26,7 +26,7 @@ public class ProxyResolutionResult { private final Set clusterResolutionResults; - ProxyResolutionResult(Set clusterResolutionResults) { + public ProxyResolutionResult(Set clusterResolutionResults) { Objects.requireNonNull(clusterResolutionResults); this.clusterResolutionResults = clusterResolutionResults; } diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/resolver/ResolutionResult.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/resolver/ResolutionResult.java index bac281a008..d8fb956686 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/resolver/ResolutionResult.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/resolver/ResolutionResult.java @@ -11,6 +11,7 @@ import io.fabric8.kubernetes.api.model.HasMetadata; import io.kroxylicious.kubernetes.api.common.LocalRef; +import io.kroxylicious.kubernetes.operator.ResourcesUtil; import edu.umd.cs.findbugs.annotations.Nullable; @@ -37,4 +38,8 @@ public T referentResource() { public Optional maybeReferentResource() { return Optional.ofNullable(referentNew); } + + public static ResolutionResult resolved(HasMetadata referrer, T referent) { + return new ResolutionResult<>(ResourcesUtil.toLocalRef(referrer), ResourcesUtil.toLocalRef(referent), referent); + } } diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/KafkaProxyReconcilerIT.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/KafkaProxyReconcilerIT.java index dc640c523a..e5ca5bff08 100644 --- a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/KafkaProxyReconcilerIT.java +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/KafkaProxyReconcilerIT.java @@ -75,8 +75,11 @@ import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilterStatusBuilder; import io.kroxylicious.kubernetes.operator.assertj.OperatorAssertions; import io.kroxylicious.kubernetes.operator.assertj.ProxyConfigAssert; +import io.kroxylicious.kubernetes.operator.model.networking.LoadBalancerClusterIngressNetworkingModel; import io.kroxylicious.proxy.config.ConfigParser; import io.kroxylicious.proxy.config.Configuration; +import io.kroxylicious.proxy.config.VirtualCluster; +import io.kroxylicious.proxy.config.VirtualClusterGateway; import io.kroxylicious.proxy.service.HostPort; import edu.umd.cs.findbugs.annotations.NonNull; @@ -87,6 +90,7 @@ import static io.kroxylicious.kubernetes.operator.ResourcesUtil.generation; import static io.kroxylicious.kubernetes.operator.ResourcesUtil.name; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.InstanceOfAssertFactories.list; import static org.awaitility.Awaitility.await; @EnabledIf(value = "io.kroxylicious.kubernetes.operator.OperatorTestUtils#isKubeClientAvailable", disabledReason = "no viable kube client available") @@ -100,10 +104,12 @@ class KafkaProxyReconcilerIT { private static final String FILTER_NAME = "validation"; private static final String CLUSTER_FOO = "foo"; private static final String CLUSTER_FOO_CLUSTERIP_INGRESS = "foo-cluster-ip"; + private static final String CLUSTER_FOO_LOADBALANCER_INGRESS = "foo-load-balancer"; private static final String CLUSTER_FOO_BOOTSTRAP = "my-cluster-kafka-bootstrap.foo.svc.cluster.local:9092"; private static final String CLUSTER_BAR_REF = "barref"; private static final String CLUSTER_BAR = "bar"; private static final String CLUSTER_BAR_CLUSTERIP_INGRESS = "bar-cluster-ip"; + private static final String CLUSTER_BAR_LOADBALANCER_INGRESS = "bar-load-balancer"; private static final String CLUSTER_BAR_BOOTSTRAP = "my-cluster-kafka-bootstrap.bar.svc.cluster.local:9092"; private static final String NEW_BOOTSTRAP = "new-bootstrap:9092"; @@ -176,7 +182,7 @@ void testCreateWithKafkaServiceTls() { } @Test - void createVirtualClusterUsingTls() { + void virtualClusterWithClusterIpIngress() { // Given KafkaProxy proxy = testActor.create(kafkaProxy(PROXY_A)); @@ -200,7 +206,7 @@ void createVirtualClusterUsingTls() { } @Test - void createVirtualClusterUsingTlsWithTrustAnchors() { + void virtualClusterWithClusterIpIngressWithTrustAnchor() { // Given KafkaProxy proxy = testActor.create(kafkaProxy(PROXY_A)); @@ -230,6 +236,194 @@ void createVirtualClusterUsingTlsWithTrustAnchors() { assertDeploymentMountsConfigMap(proxy, downstreamTrustAnchorName); } + @Test + void virtualClusterWithLoadBalancerIngressWithTrustAnchor() { + KafkaProxy proxy = testActor.create(kafkaProxy(PROXY_A)); + KafkaService kafkaService = updateStatusObservedGeneration(testActor.create(kafkaService(CLUSTER_BAR_REF, CLUSTER_BAR_BOOTSTRAP))); + + String loadbalancerBootstrap = "bootstrap.kafka"; + String loadbalancerBrokerAddressPattern = "broker-$(nodeId).kafka"; + KafkaProxyIngress loadBalancerIngress = updateStatusObservedGeneration( + testActor.create(loadBalancerIngress(CLUSTER_BAR_LOADBALANCER_INGRESS, proxy, loadbalancerBootstrap, + loadbalancerBrokerAddressPattern))); + + Secret tlsServerCert = testActor.create(tlsKeyAndCertSecret("downstream-tls-certificate")); + String downstreamTrustAnchorName = "downstream-tls-trust-anchor"; + ConfigMap trustAnchor = testActor.create(trustAnchorConfigMap(downstreamTrustAnchorName, "tls.pem")); + + Ingresses clusterIngress = new IngressesBuilder() + .withIngressRef(toIngressRef(loadBalancerIngress)) + .withNewTls() + .withCertificateRef(toCertificateRef(tlsServerCert)) + .withTrustAnchorRef(toTrustAnchorRef(trustAnchor)) + .endTls() + .build(); + VirtualKafkaCluster cluster = virtualKafkaCluster(CLUSTER_BAR, proxy, kafkaService, List.of(clusterIngress), Optional.empty()); + + // when + updateStatusObservedGeneration(testActor.create(cluster)); + + int clientFacingPort = LoadBalancerClusterIngressNetworkingModel.DEFAULT_LOADBALANCER_PORT; + int proxyListenPort = ProxyDeploymentDependentResource.SHARED_SNI_PORT; + + AWAIT.alias("shared sni service manifested").untilAsserted(() -> { + String serviceName = name(proxy) + "-sni"; + var service = testActor.get(Service.class, serviceName); + assertThat(service).isNotNull() + .describedAs( + "Expect shared SNI Service for proxy '" + name(proxy) + " to exist") + .extracting(svc -> svc.getSpec().getSelector()) + .describedAs("Service's selector should select proxy pods") + .isEqualTo(ProxyDeploymentDependentResource.podLabels(proxy)); + assertThat(service.getSpec().getType()).isEqualTo("LoadBalancer"); + // cannot use equality because the ServicePort has a random nodePort assigned to it + assertThat(service.getSpec().getPorts()).singleElement().satisfies(onlyPort -> { + String expectedName = "sni-" + clientFacingPort; + assertThat(onlyPort.getName()).isEqualTo(expectedName); + assertThat(onlyPort.getProtocol()).isEqualTo("TCP"); + assertThat(onlyPort.getPort()).isEqualTo(clientFacingPort); + assertThat(onlyPort.getTargetPort()).isEqualTo(new IntOrString(proxyListenPort)); + }); + }); + + AWAIT.alias("proxy deployment exposes shared sni port").untilAsserted(() -> { + var deployment = testActor.get(Deployment.class, ProxyDeploymentDependentResource.deploymentName(proxy)); + assertThat(deployment).isNotNull(); + List containers = deployment.getSpec().getTemplate().getSpec().getContainers(); + assertThat(containers).hasSize(1).singleElement().satisfies(container -> { + assertThat(container.getName()).isEqualTo("proxy"); + ContainerPort metricsPort = new ContainerPortBuilder().withContainerPort(9190).withName("management").withProtocol("TCP").build(); + ContainerPort bootstrapContainerPort = createContainerPort(proxyListenPort, "shared-sni-port"); + assertThat(container.getPorts()) + .containsExactlyInAnyOrder( + metricsPort, + bootstrapContainerPort); + }); + }); + + AWAIT.alias("proxy config - gateway configured for SNI loadbalancer ingress").untilAsserted(() -> { + assertProxyConfigInConfigMap(proxy) + .cluster(name(cluster)) + .gateway(name(loadBalancerIngress)) + .sniHostIdentifiesNode() + .hasBootstrapAddress(new HostPort(loadbalancerBootstrap, proxyListenPort).toString()) + .hasAdvertisedBrokerAddressPattern(new HostPort(loadbalancerBrokerAddressPattern, clientFacingPort).toString()); + }); + } + + @Test + void virtualClusterWithLoadBalancerAndClusterIpIngress() { + KafkaProxy proxy = testActor.create(kafkaProxy(PROXY_A)); + KafkaService kafkaService = updateStatusObservedGeneration(testActor.create(kafkaService(CLUSTER_BAR_REF, CLUSTER_BAR_BOOTSTRAP))); + + KafkaProxyIngress loadBalancerIngress = updateStatusObservedGeneration(testActor.create(loadBalancerIngress(CLUSTER_BAR_LOADBALANCER_INGRESS, proxy, + "bootstrap.kafka", + "broker-$(nodeId).kafka"))); + + Secret loadBalancerTlsServerCert = testActor.create(tlsKeyAndCertSecret("loadbalancer-tls-certificate")); + + Ingresses lbIngress = new IngressesBuilder() + .withIngressRef(toIngressRef(loadBalancerIngress)) + .withNewTls() + .withCertificateRef(toCertificateRef(loadBalancerTlsServerCert)) + .endTls() + .build(); + + Secret clusterIpTlsServerCert = testActor.create(tlsKeyAndCertSecret("clusterip-tls-certificate")); + + KafkaProxyIngress clusterIpIngress = updateStatusObservedGeneration(testActor.create(clusterIpIngress(CLUSTER_BAR_CLUSTERIP_INGRESS, proxy, TLS))); + + Ingresses cipIngress = new IngressesBuilder() + .withIngressRef(toIngressRef(clusterIpIngress)) + .withNewTls() + .withCertificateRef(toCertificateRef(clusterIpTlsServerCert)) + .endTls() + .build(); + + VirtualKafkaCluster cluster = virtualKafkaCluster(CLUSTER_BAR, proxy, kafkaService, List.of(lbIngress, cipIngress), Optional.empty()); + + // when + updateStatusObservedGeneration(testActor.create(cluster)); + + // then + AWAIT.alias("services manifested").untilAsserted(() -> { + String sharedSniServiceName = name(proxy) + "-sni"; + String clusterIpServiceName = CLUSTER_BAR + "-" + clusterIpIngress.getMetadata().getName(); + var services = testActor.resources(Service.class).list().getItems(); + assertThat(services) + .extracting(service -> service.getMetadata().getName()) + .contains(sharedSniServiceName, clusterIpServiceName); + }); + + AWAIT.alias("proxy config - gateway configured for both ingress types").untilAsserted(() -> { + assertProxyConfigInConfigMap(proxy) + .cluster(name(cluster)) + .extracting(VirtualCluster::gateways, list(VirtualClusterGateway.class)) + .extracting(VirtualClusterGateway::name) + .containsExactly(name(loadBalancerIngress), name(clusterIpIngress)); + }); + } + + @Test + void twoVirtualClusterUsingLoadBalancer() { + KafkaProxy proxy = testActor.create(kafkaProxy(PROXY_A)); + KafkaService kafkaService = updateStatusObservedGeneration(testActor.create(kafkaService(CLUSTER_BAR_REF, CLUSTER_BAR_BOOTSTRAP))); + + KafkaProxyIngress loadBalancerIngressFoo = updateStatusObservedGeneration(testActor.create(loadBalancerIngress(CLUSTER_FOO_LOADBALANCER_INGRESS, proxy, + "bootstrap.foo.kafka", + "broker-$(nodeId).foo.kafka"))); + + KafkaProxyIngress loadBalancerIngressBar = updateStatusObservedGeneration(testActor.create(loadBalancerIngress(CLUSTER_BAR_LOADBALANCER_INGRESS, proxy, + "bootstrap.bar.kafka", + "broker-$(nodeId).bar.kafka"))); + + Secret loadBalancerTlsServerCertFoo = testActor.create(tlsKeyAndCertSecret("loadbalancer-tls-certificate-foo")); + Secret loadBalancerTlsServerCertBar = testActor.create(tlsKeyAndCertSecret("loadbalancer-tls-certificate-bar")); + + Ingresses lbIngressFoo = new IngressesBuilder() + .withIngressRef(toIngressRef(loadBalancerIngressFoo)) + .withNewTls() + .withCertificateRef(toCertificateRef(loadBalancerTlsServerCertFoo)) + .endTls() + .build(); + + Ingresses lbIngressBar = new IngressesBuilder() + .withIngressRef(toIngressRef(loadBalancerIngressBar)) + .withNewTls() + .withCertificateRef(toCertificateRef(loadBalancerTlsServerCertBar)) + .endTls() + .build(); + + VirtualKafkaCluster fooCluster = testActor.create(virtualKafkaCluster(CLUSTER_FOO, proxy, kafkaService, List.of(lbIngressFoo), Optional.empty())); + VirtualKafkaCluster barCluster = testActor.create(virtualKafkaCluster(CLUSTER_BAR, proxy, kafkaService, List.of(lbIngressBar), Optional.empty())); + + // when + List.of(fooCluster, barCluster).forEach(this::updateStatusObservedGeneration); + + // then + AWAIT.alias("shared sni service manifested").untilAsserted(() -> { + String sharedSniServiceName = name(proxy) + "-sni"; + var services = testActor.resources(Service.class).list().getItems(); + assertThat(services) + .extracting(service -> service.getMetadata().getName()) + .containsExactly(sharedSniServiceName); + }); + + AWAIT.alias("proxy config - each virtual cluster configured with correct ingress").untilAsserted(() -> { + assertProxyConfigInConfigMap(proxy) + .cluster(name(fooCluster)) + .gateway(name(loadBalancerIngressFoo)) + .sniHostIdentifiesNode() + .hasBootstrapAddress("bootstrap.foo.kafka:" + ProxyDeploymentDependentResource.SHARED_SNI_PORT); + + assertProxyConfigInConfigMap(proxy) + .cluster(name(barCluster)) + .gateway(name(loadBalancerIngressBar)) + .sniHostIdentifiesNode() + .hasBootstrapAddress("bootstrap.bar.kafka:" + ProxyDeploymentDependentResource.SHARED_SNI_PORT); + }); + } + @Test void clusterIpIngressUsesDeclaredNodeIdsOfService() { KafkaProxy proxy = testActor.create(kafkaProxy(PROXY_A)); @@ -314,7 +508,7 @@ void clusterIpIngressUsesDeclaredNodeIdsOfService() { } private static @NonNull ServicePort clusterIpServicePort(String clusterName, int port) { - return createServicePort(clusterName + "-" + port, port); + return createServicePort(clusterName + "-" + port, port, port); } private static ContainerPort createContainerPort(int port, String name) { @@ -325,8 +519,8 @@ private static NodeIdRanges createNodeIdRanges(String brokers, long start, long return new NodeIdRangesBuilder().withName(brokers).withStart(start).withEnd(end).build(); } - private static ServicePort createServicePort(String name, int port) { - return new ServicePortBuilder().withName(name).withPort(port).withProtocol("TCP").withTargetPort(new IntOrString(port)).build(); + private static ServicePort createServicePort(String name, int port, int targetPort) { + return new ServicePortBuilder().withName(name).withPort(port).withProtocol("TCP").withTargetPort(new IntOrString(targetPort)).build(); } private record CreatedResources(KafkaProxy proxy, Set clusters, Set services, Set ingresses) { @@ -421,6 +615,28 @@ private KafkaProxyIngress clusterIpIngress(String ingressName, KafkaProxy proxy, // @formatter:on } + private KafkaProxyIngress loadBalancerIngress(String ingressName, + KafkaProxy proxy, + String bootstrapAddress, + String advertisedBrokerAddressPattern) { + // @formatter:off + return new KafkaProxyIngressBuilder() + .withNewMetadata() + .withName(ingressName) + .endMetadata() + .withNewSpec() + .withNewLoadBalancer() + .withBootstrapAddress(bootstrapAddress) + .withAdvertisedBrokerAddressPattern(advertisedBrokerAddressPattern) + .endLoadBalancer() + .withNewProxyRef() + .withName(name(proxy)) + .endProxyRef() + .endSpec() + .build(); + // @formatter:on + } + private void assertDeploymentBecomesReady(KafkaProxy proxy) { // wait longer for initial operator image download AWAIT.alias("Deployment as expected").untilAsserted(() -> { diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java index a21a7633e9..8f5601bf2e 100644 --- a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java @@ -187,6 +187,7 @@ class VirtualKafkaClusterReconcilerTest { .endMetadata() .withNewSpec() .withNewProxyRef().withName(PROXY_NAME).endProxyRef() + .withNewClusterIP().withProtocol(ClusterIP.Protocol.TCP).endClusterIP() .endSpec() .build(); public static final KafkaProxyIngress INGRESS_WITH_TLS = new KafkaProxyIngressBuilder(INGRESS) diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/ClusterIPClusterIngressNetworkingModelTest.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/ClusterIPClusterIngressNetworkingModelTest.java index 1c4b8d2cbd..fdf338d791 100644 --- a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/ClusterIPClusterIngressNetworkingModelTest.java +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/ClusterIPClusterIngressNetworkingModelTest.java @@ -360,12 +360,12 @@ void rangesMustBeNonEmpty() { } @Test - void proxyContainerPortsSingleRange() { + void identifyingProxyContainerPortsSingleRange() { ClusterIngressNetworkingModel instance = new ClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, List.of(createNodeIdRange("a", 1, 3)), null, 1, 4); assertThat(instance).isNotNull(); - assertThat(instance.proxyContainerPorts()) + assertThat(instance.identifyingProxyContainerPorts()) .containsExactly(createContainerPort(1 + "-bootstrap", 1), createContainerPort(2 + "-node", 2), createContainerPort(3 + "-node", 3), @@ -373,12 +373,12 @@ void proxyContainerPortsSingleRange() { } @Test - void proxyContainerPortsMultipleRanges() { + void identifyingProxyContainerPortsMultipleRanges() { ClusterIngressNetworkingModel instance = new ClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, List.of(createNodeIdRange("a", 1L, 1L), createNodeIdRange("b", 3L, 4L)), null, 1, 4); assertThat(instance).isNotNull(); - assertThat(instance.proxyContainerPorts()) + assertThat(instance.identifyingProxyContainerPorts()) .containsExactly(createContainerPort(1 + "-bootstrap", 1), createContainerPort(2 + "-node", 2), createContainerPort(3 + "-node", 3), diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/LoadBalancerClusterIngressNetworkingModelTest.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/LoadBalancerClusterIngressNetworkingModelTest.java new file mode 100644 index 0000000000..046685bb04 --- /dev/null +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/LoadBalancerClusterIngressNetworkingModelTest.java @@ -0,0 +1,167 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.kubernetes.operator.model.networking; + +import java.util.stream.Stream; + +import org.assertj.core.api.ThrowableAssert; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import io.fabric8.kubernetes.api.model.ServiceBuilder; + +import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngress; +import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngressBuilder; +import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaCluster; +import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaClusterBuilder; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.LoadBalancer; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.LoadBalancerBuilder; +import io.kroxylicious.kubernetes.api.v1alpha1.virtualkafkaclusterspec.ingresses.Tls; +import io.kroxylicious.kubernetes.api.v1alpha1.virtualkafkaclusterspec.ingresses.TlsBuilder; +import io.kroxylicious.kubernetes.operator.KafkaProxyReconciler; +import io.kroxylicious.proxy.config.VirtualClusterGateway; +import io.kroxylicious.proxy.service.HostPort; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.params.provider.Arguments.argumentSet; + +class LoadBalancerClusterIngressNetworkingModelTest { + private static final String INGRESS_NAME = "my-ingress"; + private static final String CLUSTER_NAME = "my-cluster"; + private static final String NAMESPACE = "my-namespace"; + + private static final String BOOTSTRAP_ADDRESS = "$(virtualClusterName).kafkaproxy"; + private static final String ADVERTISED_BROKER_ADDRESS_PATTERN = "$(virtualClusterName)-$(nodeId).kafkaproxy"; + + private static final LoadBalancer LOAD_BALANCER = new LoadBalancerBuilder() + .withBootstrapAddress(BOOTSTRAP_ADDRESS) + .withAdvertisedBrokerAddressPattern(ADVERTISED_BROKER_ADDRESS_PATTERN) + .build(); + + private static final KafkaProxyIngress INGRESS = new KafkaProxyIngressBuilder() + .withNewMetadata() + .withName(INGRESS_NAME) + .withNamespace(NAMESPACE) + .endMetadata() + .withNewSpec() + .withLoadBalancer(LOAD_BALANCER) + .endSpec() + .build(); + // @formatter:on + + private static final Tls TLS = new TlsBuilder().withNewCertificateRef().withName("server-cert").endCertificateRef().build(); + + private static final VirtualKafkaCluster VIRTUAL_KAFKA_CLUSTER = new VirtualKafkaClusterBuilder() + .withNewMetadata() + .withName(CLUSTER_NAME) + .withNamespace(NAMESPACE) + .endMetadata() + .withNewSpec() + .addNewIngress() + .withNewIngressRef() + .withName(INGRESS_NAME) + .endIngressRef() + .withTls(TLS) + .endIngress() + .endSpec() + .build(); + // @formatter:on + private static final String PROXY_UID = "my-proxy-uid"; + + @Test + void createInstancesWithExpectedSniPort() { + // given + // when + LoadBalancerClusterIngressNetworkingModel model = new LoadBalancerClusterIngressNetworkingModel(VIRTUAL_KAFKA_CLUSTER, INGRESS, LOAD_BALANCER, TLS, 1); + // then + // expect to be allocated an SNI port + assertThat(model).isNotNull(); + } + + @Test + void gatewayConfig() { + // given + int sharedSniPort = 1; + LoadBalancerClusterIngressNetworkingModel model = new LoadBalancerClusterIngressNetworkingModel(VIRTUAL_KAFKA_CLUSTER, INGRESS, LOAD_BALANCER, TLS, + sharedSniPort); + + // when + VirtualClusterGateway gateway = KafkaProxyReconciler.buildVirtualClusterGateway(model).fragment(); + + // then + assertThat(gateway).isNotNull(); + assertThat(gateway.name()).isEqualTo(INGRESS_NAME); + assertThat(gateway.tls()).isNotNull(); + assertThat(gateway.portIdentifiesNode()).isNull(); + assertThat(gateway.sniHostIdentifiesNode()).isNotNull().satisfies(sniStrategy -> { + assertThat(sniStrategy.bootstrapAddress()).isEqualTo(new HostPort(BOOTSTRAP_ADDRESS, sharedSniPort).toString()); + assertThat(sniStrategy.advertisedBrokerAddressPattern()).isEqualTo(new HostPort(ADVERTISED_BROKER_ADDRESS_PATTERN, 9083).toString()); + }); + } + + @Test + void doesNotSupplyAnyServices() { + // given + LoadBalancerClusterIngressNetworkingModel model = new LoadBalancerClusterIngressNetworkingModel(VIRTUAL_KAFKA_CLUSTER, INGRESS, LOAD_BALANCER, TLS, + 9093); + assertThat(model).isNotNull(); + + // when + Stream services = model.services(); + + // then + assertThat(services).isEmpty(); + } + + @Test + void requestsLoadBalancerServicePorts() { + // given + LoadBalancerClusterIngressNetworkingModel model = new LoadBalancerClusterIngressNetworkingModel(VIRTUAL_KAFKA_CLUSTER, INGRESS, LOAD_BALANCER, TLS, + 9093); + assertThat(model).isNotNull(); + + // when + Stream services = model.requiredSniLoadBalancerServicePorts(); + + // then + assertThat(services).containsExactly(9083); + } + + @Test + void requestsSharedSniPort() { + // given + LoadBalancerClusterIngressNetworkingModel model = new LoadBalancerClusterIngressNetworkingModel(VIRTUAL_KAFKA_CLUSTER, INGRESS, LOAD_BALANCER, TLS, + 9093); + + // when + boolean requiresSharedSniPort = model.requiresSharedSniContainerPort(); + + // then + assertThat(requiresSharedSniPort).isTrue(); + } + + public static Stream constructorArgsMustBeNonNull() { + ThrowableAssert.ThrowingCallable nullIngress = () -> new LoadBalancerClusterIngressNetworkingModel(VIRTUAL_KAFKA_CLUSTER, null, LOAD_BALANCER, TLS, 1); + ThrowableAssert.ThrowingCallable nullCluster = () -> new LoadBalancerClusterIngressNetworkingModel(null, INGRESS, LOAD_BALANCER, TLS, 1); + ThrowableAssert.ThrowingCallable nullLoadBalancer = () -> new LoadBalancerClusterIngressNetworkingModel(VIRTUAL_KAFKA_CLUSTER, INGRESS, null, TLS, 1); + ThrowableAssert.ThrowingCallable nullTls = () -> new LoadBalancerClusterIngressNetworkingModel(VIRTUAL_KAFKA_CLUSTER, INGRESS, LOAD_BALANCER, null, 1); + return Stream.of(argumentSet("ingress", nullIngress), + argumentSet("cluster", nullCluster), + argumentSet("loadBalancer", nullLoadBalancer), + argumentSet("tls", nullTls)); + } + + @MethodSource + @ParameterizedTest + void constructorArgsMustBeNonNull(ThrowableAssert.ThrowingCallable callable) { + assertThatThrownBy(callable).isInstanceOf(NullPointerException.class); + } + +} diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/NetworkingPlannerTest.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/NetworkingPlannerTest.java new file mode 100644 index 0000000000..c88a541d37 --- /dev/null +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/NetworkingPlannerTest.java @@ -0,0 +1,382 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.kubernetes.operator.model.networking; + +import java.util.List; +import java.util.Set; + +import org.assertj.core.api.ListAssert; +import org.junit.jupiter.api.Test; + +import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxy; +import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyBuilder; +import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngress; +import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngressBuilder; +import io.kroxylicious.kubernetes.api.v1alpha1.KafkaService; +import io.kroxylicious.kubernetes.api.v1alpha1.KafkaServiceBuilder; +import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaCluster; +import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaClusterBuilder; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.LoadBalancer; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.LoadBalancerBuilder; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaservicespec.NodeIdRangesBuilder; +import io.kroxylicious.kubernetes.api.v1alpha1.virtualkafkaclusterspec.Ingresses; +import io.kroxylicious.kubernetes.api.v1alpha1.virtualkafkaclusterspec.IngressesBuilder; +import io.kroxylicious.kubernetes.api.v1alpha1.virtualkafkaclusterspec.ingresses.Tls; +import io.kroxylicious.kubernetes.api.v1alpha1.virtualkafkaclusterspec.ingresses.TlsBuilder; +import io.kroxylicious.kubernetes.operator.resolver.ClusterResolutionResult; +import io.kroxylicious.kubernetes.operator.resolver.IngressResolutionResult; +import io.kroxylicious.kubernetes.operator.resolver.ProxyResolutionResult; +import io.kroxylicious.kubernetes.operator.resolver.ResolutionResult; + +import static org.assertj.core.api.Assertions.assertThat; + +class NetworkingPlannerTest { + private static final String PROXY_NAME = "my-proxy"; + private static final String CLUSTER_IP_INGRESS_NAME = "my-cluster-ip-ingress"; + private static final String CLUSTER_IP_2_INGRESS_NAME = "my-cluster-ip-ingress-2"; + private static final String LOAD_BALANCER_INGRESS_NAME = "my-load-balancer-ingress"; + private static final String LOAD_BALANCER_INGRESS_2_NAME = "my-load-balancer-ingress-2"; + private static final String CLUSTER_NAME = "my-cluster"; + private static final String KAFKA_SERVICE_NAME = "my-kafka-service"; + private static final String NAMESPACE = "ns"; + // @formatter:off + private static final KafkaProxy PROXY = new KafkaProxyBuilder() + .withNewMetadata() + .withName(PROXY_NAME) + .withNamespace(NAMESPACE) + .endMetadata() + .build(); + // @formatter:on + + // @formatter:off + private static final KafkaProxyIngress CLUSTER_IP_INGRESS = new KafkaProxyIngressBuilder() + .withNewMetadata() + .withName(CLUSTER_IP_INGRESS_NAME) + .withNamespace(NAMESPACE) + .endMetadata() + .withNewSpec() + .withNewClusterIP() + .withProtocol(ClusterIP.Protocol.TCP) + .endClusterIP() + .endSpec() + .build(); + // @formatter:on + + // @formatter:off + private static final KafkaProxyIngress CLUSTER_IP_INGRESS_2 = new KafkaProxyIngressBuilder() + .withNewMetadata() + .withName(CLUSTER_IP_2_INGRESS_NAME) + .withNamespace(NAMESPACE) + .endMetadata() + .withNewSpec() + .withNewClusterIP() + .withProtocol(ClusterIP.Protocol.TCP) + .endClusterIP() + .endSpec() + .build(); + // @formatter:on + + private static final Tls TLS = new TlsBuilder().withNewCertificateRef().withName("server-cert").endCertificateRef().build(); + private static final Tls TLS_2 = new TlsBuilder().withNewCertificateRef().withName("server-cert2").endCertificateRef().build(); + + public static final LoadBalancer LOAD_BALANCER = new LoadBalancerBuilder() + .withBootstrapAddress("bootstrap.kafka") + .withAdvertisedBrokerAddressPattern("$(nodeId)-broker.kafka") + .build(); + private static final KafkaProxyIngress LOAD_BALANCER_INGRESS = createLoadBalancerIngress(LOAD_BALANCER_INGRESS_NAME, LOAD_BALANCER); + public static final LoadBalancer LOAD_BALANCER_2 = new LoadBalancerBuilder() + .withBootstrapAddress("bootstrap.other") + .withAdvertisedBrokerAddressPattern("$(nodeId)-broker.other") + .build(); + private static final KafkaProxyIngress LOAD_BALANCER_INGRESS_2 = createLoadBalancerIngress(LOAD_BALANCER_INGRESS_2_NAME, LOAD_BALANCER_2); + + private static KafkaProxyIngress createLoadBalancerIngress(String name, LoadBalancer loadBalancer) { + return new KafkaProxyIngressBuilder() + .withNewMetadata() + .withName(name) + .withNamespace(NAMESPACE) + .endMetadata() + .withNewSpec() + .withLoadBalancer(loadBalancer) + .endSpec() + .build(); + } + // @formatter:on + + public static final Ingresses CLUSTER_IP_CLUSTER_INGRESSES = new IngressesBuilder().withNewIngressRef().withName(CLUSTER_IP_INGRESS_NAME).endIngressRef().build(); + + public static final Ingresses CLUSTER_IP_2_CLUSTER_INGRESSES = new IngressesBuilder().withNewIngressRef().withName(CLUSTER_IP_2_INGRESS_NAME).endIngressRef().build(); + public static final Ingresses LOAD_BALANCER_INGRESSES = new IngressesBuilder().withTls(TLS).withNewIngressRef().withName(LOAD_BALANCER_INGRESS_NAME).endIngressRef() + .build(); + public static final Ingresses LOAD_BALANCER_2_INGRESSES = new IngressesBuilder().withTls(TLS_2).withNewIngressRef().withName(LOAD_BALANCER_INGRESS_2_NAME) + .endIngressRef().build(); + + public static final int FIRST_IDENTIFYING_PORT = 9292; + public static final String CLUSTER_NAME_2 = "my-cluster-2"; + + // @formatter:off + private static VirtualKafkaCluster clusterWithIngress(String clusterName, Ingresses... ingresses) { + return new VirtualKafkaClusterBuilder() + .withNewMetadata() + .withName(clusterName) + .withNamespace(NAMESPACE) + .endMetadata() + .withNewSpec() + .withIngresses(ingresses) + .endSpec() + .build(); + } + // @formatter:on + + // @formatter:off + private static final KafkaService KAFKA_SERVICE = new KafkaServiceBuilder() + .withNewMetadata() + .withName(KAFKA_SERVICE_NAME) + .withNamespace(NAMESPACE) + .endMetadata() + .withNewSpec() + .withBootstrapServers("localhost:9092") + .endSpec() + .build(); + // @formatter:on + + @Test + public void noClustersResolved() { + ProxyNetworkingModel networkingModel = NetworkingPlanner.planNetworking(PROXY, new ProxyResolutionResult(Set.of())); + assertThat(networkingModel.clusterNetworkingModels()).isEmpty(); + } + + @Test + public void clusterIpIngress() { + VirtualKafkaCluster virtualKafkaCluster = clusterWithIngress(CLUSTER_NAME, CLUSTER_IP_CLUSTER_INGRESSES); + ClusterResolutionResult clusterResolutionResult = new ClusterResolutionResult(virtualKafkaCluster, ResolutionResult.resolved(virtualKafkaCluster, PROXY), + List.of(), ResolutionResult.resolved(virtualKafkaCluster, KAFKA_SERVICE), + List.of(new IngressResolutionResult(ResolutionResult.resolved(virtualKafkaCluster, CLUSTER_IP_INGRESS), + ResolutionResult.resolved(CLUSTER_IP_INGRESS, PROXY), CLUSTER_IP_CLUSTER_INGRESSES))); + ProxyNetworkingModel networkingModel = NetworkingPlanner.planNetworking(PROXY, new ProxyResolutionResult(Set.of(clusterResolutionResult))); + assertThat(networkingModel.clusterIngressModel(virtualKafkaCluster)).isNotNull(); + + assertThat(networkingModel.clusterNetworkingModels()).singleElement().satisfies(clusterNetworkingModel -> { + assertThat(clusterNetworkingModel.cluster()).isEqualTo(virtualKafkaCluster); + assertThat(clusterNetworkingModel.ingressExceptions()).isEmpty(); + assertThat(clusterNetworkingModel.clusterIngressNetworkingModelResults()).singleElement().satisfies(model -> { + assertThat(model.exception()).isNull(); + assertThat(model.clusterIngressNetworkingModel()).isEqualTo(new ClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster, CLUSTER_IP_INGRESS, + List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), null, FIRST_IDENTIFYING_PORT, + FIRST_IDENTIFYING_PORT + 3)); + }); + }); + } + + @Test + public void loadBalancerIngress() { + VirtualKafkaCluster virtualKafkaCluster = clusterWithIngress(CLUSTER_NAME, LOAD_BALANCER_INGRESSES); + ClusterResolutionResult clusterResolutionResult = new ClusterResolutionResult(virtualKafkaCluster, ResolutionResult.resolved(virtualKafkaCluster, PROXY), + List.of(), ResolutionResult.resolved(virtualKafkaCluster, KAFKA_SERVICE), + List.of(new IngressResolutionResult(ResolutionResult.resolved(virtualKafkaCluster, LOAD_BALANCER_INGRESS), + ResolutionResult.resolved(LOAD_BALANCER_INGRESS, PROXY), LOAD_BALANCER_INGRESSES))); + ProxyNetworkingModel networkingModel = NetworkingPlanner.planNetworking(PROXY, new ProxyResolutionResult(Set.of(clusterResolutionResult))); + assertThat(networkingModel.clusterIngressModel(virtualKafkaCluster)).isNotNull(); + + assertThat(networkingModel.clusterNetworkingModels()).singleElement().satisfies(clusterNetworkingModel -> { + assertThat(clusterNetworkingModel.cluster()).isEqualTo(virtualKafkaCluster); + assertThat(clusterNetworkingModel.ingressExceptions()).isEmpty(); + assertThat(clusterNetworkingModel.clusterIngressNetworkingModelResults()).singleElement().satisfies(model -> { + assertThat(model.exception()).isNull(); + assertThat(model.clusterIngressNetworkingModel()).isEqualTo( + new LoadBalancerClusterIngressNetworkingModel(virtualKafkaCluster, LOAD_BALANCER_INGRESS, LOAD_BALANCER, TLS, 9291)); + }); + }); + } + + @Test + public void clusterIpAndLoadBalancerIngress() { + VirtualKafkaCluster virtualKafkaCluster = clusterWithIngress(CLUSTER_NAME, CLUSTER_IP_CLUSTER_INGRESSES, LOAD_BALANCER_INGRESSES); + ClusterResolutionResult clusterResolutionResult = new ClusterResolutionResult(virtualKafkaCluster, ResolutionResult.resolved(virtualKafkaCluster, PROXY), + List.of(), ResolutionResult.resolved(virtualKafkaCluster, KAFKA_SERVICE), + List.of(new IngressResolutionResult(ResolutionResult.resolved(virtualKafkaCluster, CLUSTER_IP_INGRESS), + ResolutionResult.resolved(CLUSTER_IP_INGRESS, PROXY), CLUSTER_IP_CLUSTER_INGRESSES), + new IngressResolutionResult(ResolutionResult.resolved(virtualKafkaCluster, LOAD_BALANCER_INGRESS), + ResolutionResult.resolved(LOAD_BALANCER_INGRESS, PROXY), LOAD_BALANCER_INGRESSES))); + ProxyNetworkingModel networkingModel = NetworkingPlanner.planNetworking(PROXY, new ProxyResolutionResult(Set.of(clusterResolutionResult))); + assertThat(networkingModel.clusterIngressModel(virtualKafkaCluster)).isNotNull(); + + assertThat(networkingModel.clusterNetworkingModels()).singleElement().satisfies(clusterNetworkingModel -> { + assertThat(clusterNetworkingModel.cluster()).isEqualTo(virtualKafkaCluster); + assertThat(clusterNetworkingModel.ingressExceptions()).isEmpty(); + var listAssert = assertThat(clusterNetworkingModel.clusterIngressNetworkingModelResults()).hasSize(2); + listAssert.element(0).satisfies(model -> { + assertThat(model.exception()).isNull(); + assertThat(model.clusterIngressNetworkingModel()).isEqualTo(new ClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster, CLUSTER_IP_INGRESS, + List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), null, FIRST_IDENTIFYING_PORT, + FIRST_IDENTIFYING_PORT + 3)); + }); + listAssert.element(1).satisfies(model -> { + assertThat(model.exception()).isNull(); + assertThat(model.clusterIngressNetworkingModel()).isEqualTo( + new LoadBalancerClusterIngressNetworkingModel(virtualKafkaCluster, LOAD_BALANCER_INGRESS, LOAD_BALANCER, TLS, 9291)); + }); + + }); + } + + @Test + public void multipleLoadBalancerIngress() { + VirtualKafkaCluster virtualKafkaCluster = clusterWithIngress(CLUSTER_NAME, LOAD_BALANCER_INGRESSES, LOAD_BALANCER_2_INGRESSES); + ClusterResolutionResult clusterResolutionResult = new ClusterResolutionResult(virtualKafkaCluster, ResolutionResult.resolved(virtualKafkaCluster, PROXY), + List.of(), ResolutionResult.resolved(virtualKafkaCluster, KAFKA_SERVICE), + List.of(new IngressResolutionResult(ResolutionResult.resolved(virtualKafkaCluster, LOAD_BALANCER_INGRESS), + ResolutionResult.resolved(LOAD_BALANCER_INGRESS, PROXY), LOAD_BALANCER_INGRESSES), + new IngressResolutionResult(ResolutionResult.resolved(virtualKafkaCluster, LOAD_BALANCER_INGRESS_2), + ResolutionResult.resolved(LOAD_BALANCER_INGRESS_2, PROXY), LOAD_BALANCER_2_INGRESSES))); + ProxyNetworkingModel networkingModel = NetworkingPlanner.planNetworking(PROXY, new ProxyResolutionResult(Set.of(clusterResolutionResult))); + assertThat(networkingModel.clusterIngressModel(virtualKafkaCluster)).isNotNull(); + + assertThat(networkingModel.clusterNetworkingModels()).singleElement().satisfies(clusterNetworkingModel -> { + assertThat(clusterNetworkingModel.cluster()).isEqualTo(virtualKafkaCluster); + assertThat(clusterNetworkingModel.ingressExceptions()).isEmpty(); + var listAssert = assertThat(clusterNetworkingModel.clusterIngressNetworkingModelResults()).hasSize(2); + listAssert.element(0).satisfies(model -> { + assertThat(model.exception()).isNull(); + assertThat(model.clusterIngressNetworkingModel()).isEqualTo( + new LoadBalancerClusterIngressNetworkingModel(virtualKafkaCluster, LOAD_BALANCER_INGRESS, LOAD_BALANCER, TLS, 9291)); + }); + listAssert.element(1).satisfies(model -> { + assertThat(model.exception()).isNull(); + assertThat(model.clusterIngressNetworkingModel()).isEqualTo( + new LoadBalancerClusterIngressNetworkingModel(virtualKafkaCluster, LOAD_BALANCER_INGRESS_2, LOAD_BALANCER_2, TLS_2, 9291)); + }); + }); + } + + @Test + public void multipleClustersWithLoadBalancerIngress() { + VirtualKafkaCluster virtualKafkaCluster = clusterWithIngress(CLUSTER_NAME, LOAD_BALANCER_INGRESSES); + VirtualKafkaCluster virtualKafkaCluster2 = clusterWithIngress(CLUSTER_NAME_2, LOAD_BALANCER_2_INGRESSES); + ClusterResolutionResult clusterResolutionResult = new ClusterResolutionResult(virtualKafkaCluster, ResolutionResult.resolved(virtualKafkaCluster, PROXY), + List.of(), ResolutionResult.resolved(virtualKafkaCluster, KAFKA_SERVICE), + List.of(new IngressResolutionResult(ResolutionResult.resolved(virtualKafkaCluster, LOAD_BALANCER_INGRESS), + ResolutionResult.resolved(LOAD_BALANCER_INGRESS, PROXY), LOAD_BALANCER_INGRESSES))); + ClusterResolutionResult clusterResolutionResult2 = new ClusterResolutionResult(virtualKafkaCluster2, ResolutionResult.resolved(virtualKafkaCluster, PROXY), + List.of(), ResolutionResult.resolved(virtualKafkaCluster2, KAFKA_SERVICE), + List.of(new IngressResolutionResult(ResolutionResult.resolved(virtualKafkaCluster2, LOAD_BALANCER_INGRESS_2), + ResolutionResult.resolved(LOAD_BALANCER_INGRESS_2, PROXY), LOAD_BALANCER_2_INGRESSES))); + ProxyNetworkingModel networkingModel = NetworkingPlanner.planNetworking(PROXY, + new ProxyResolutionResult(Set.of(clusterResolutionResult, clusterResolutionResult2))); + assertThat(networkingModel.clusterIngressModel(virtualKafkaCluster)).isNotNull(); + + ListAssert listAssert = assertThat(networkingModel.clusterNetworkingModels()).hasSize(2); + listAssert.element(0).satisfies(clusterNetworkingModel -> { + assertThat(clusterNetworkingModel.cluster()).isEqualTo(virtualKafkaCluster); + assertThat(clusterNetworkingModel.ingressExceptions()).isEmpty(); + assertThat(clusterNetworkingModel.clusterIngressNetworkingModelResults()).singleElement().satisfies(model -> { + assertThat(model.exception()).isNull(); + assertThat(model.clusterIngressNetworkingModel()).isEqualTo( + new LoadBalancerClusterIngressNetworkingModel(virtualKafkaCluster, LOAD_BALANCER_INGRESS, LOAD_BALANCER, TLS, 9291)); + + }); + }); + listAssert.element(1).satisfies(clusterNetworkingModel -> { + assertThat(clusterNetworkingModel.cluster()).isEqualTo(virtualKafkaCluster2); + assertThat(clusterNetworkingModel.ingressExceptions()).isEmpty(); + assertThat(clusterNetworkingModel.clusterIngressNetworkingModelResults()).singleElement().satisfies(model -> { + assertThat(model.exception()).isNull(); + assertThat(model.clusterIngressNetworkingModel()).isEqualTo( + new LoadBalancerClusterIngressNetworkingModel(virtualKafkaCluster2, LOAD_BALANCER_INGRESS_2, LOAD_BALANCER_2, TLS_2, 9291)); + + }); + }); + } + + @Test + public void multipleClusterIpIngresses() { + VirtualKafkaCluster virtualKafkaCluster = clusterWithIngress(CLUSTER_NAME, CLUSTER_IP_CLUSTER_INGRESSES, CLUSTER_IP_2_CLUSTER_INGRESSES); + ClusterResolutionResult clusterResolutionResult = new ClusterResolutionResult(virtualKafkaCluster, ResolutionResult.resolved(virtualKafkaCluster, PROXY), + List.of(), ResolutionResult.resolved(virtualKafkaCluster, KAFKA_SERVICE), + List.of(new IngressResolutionResult(ResolutionResult.resolved(virtualKafkaCluster, CLUSTER_IP_INGRESS), + ResolutionResult.resolved(CLUSTER_IP_INGRESS, PROXY), CLUSTER_IP_CLUSTER_INGRESSES), + new IngressResolutionResult(ResolutionResult.resolved(virtualKafkaCluster, CLUSTER_IP_INGRESS_2), + ResolutionResult.resolved(CLUSTER_IP_INGRESS_2, PROXY), CLUSTER_IP_2_CLUSTER_INGRESSES))); + ProxyNetworkingModel networkingModel = NetworkingPlanner.planNetworking(PROXY, new ProxyResolutionResult(Set.of(clusterResolutionResult))); + assertThat(networkingModel.clusterIngressModel(virtualKafkaCluster)).isNotNull(); + + assertThat(networkingModel.clusterNetworkingModels()).singleElement().satisfies(clusterNetworkingModel -> { + assertThat(clusterNetworkingModel.cluster()).isEqualTo(virtualKafkaCluster); + assertThat(clusterNetworkingModel.ingressExceptions()).hasSize(1).singleElement().satisfies(model -> { + assertThat(model).hasMessageContaining("Currently we do not support a virtual cluster with multiple ingresses that need unique ports to" + + " identify which node the client is connecting to"); + }); + var listAssert = assertThat(clusterNetworkingModel.clusterIngressNetworkingModelResults()).hasSize(2); + listAssert.element(0).satisfies(model -> { + assertThat(model.exception()).isNull(); + assertThat(model.clusterIngressNetworkingModel()).isEqualTo(new ClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster, CLUSTER_IP_INGRESS, + List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), null, FIRST_IDENTIFYING_PORT, + FIRST_IDENTIFYING_PORT + 3)); + }); + listAssert.element(1).satisfies(model -> { + assertThat(model.exception()).isNotNull().hasMessageContaining( + "Currently we do not support a virtual cluster with multiple ingresses that need unique ports to" + + " identify which node the client is connecting to"); + // ports are still allocated to try and keep the model stable + assertThat(model.clusterIngressNetworkingModel()).isEqualTo(new ClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster, CLUSTER_IP_INGRESS_2, + List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), null, FIRST_IDENTIFYING_PORT + 4, + FIRST_IDENTIFYING_PORT + 7)); + }); + }); + } + + @Test + public void multipleClustersWithClusterIpIngress() { + VirtualKafkaCluster virtualKafkaCluster = clusterWithIngress(CLUSTER_NAME, CLUSTER_IP_CLUSTER_INGRESSES); + ClusterResolutionResult clusterResolutionResult = new ClusterResolutionResult(virtualKafkaCluster, ResolutionResult.resolved(virtualKafkaCluster, PROXY), + List.of(), ResolutionResult.resolved(virtualKafkaCluster, KAFKA_SERVICE), + List.of(new IngressResolutionResult(ResolutionResult.resolved(virtualKafkaCluster, CLUSTER_IP_INGRESS), + ResolutionResult.resolved(CLUSTER_IP_INGRESS, PROXY), CLUSTER_IP_CLUSTER_INGRESSES))); + + VirtualKafkaCluster virtualKafkaCluster2 = clusterWithIngress(CLUSTER_NAME_2, CLUSTER_IP_2_CLUSTER_INGRESSES); + ClusterResolutionResult clusterResolutionResult2 = new ClusterResolutionResult(virtualKafkaCluster2, ResolutionResult.resolved(virtualKafkaCluster, PROXY), + List.of(), ResolutionResult.resolved(virtualKafkaCluster2, KAFKA_SERVICE), + List.of( + new IngressResolutionResult(ResolutionResult.resolved(virtualKafkaCluster2, CLUSTER_IP_INGRESS_2), + ResolutionResult.resolved(CLUSTER_IP_INGRESS_2, PROXY), CLUSTER_IP_2_CLUSTER_INGRESSES))); + ProxyNetworkingModel networkingModel = NetworkingPlanner.planNetworking(PROXY, + new ProxyResolutionResult(Set.of(clusterResolutionResult, clusterResolutionResult2))); + assertThat(networkingModel.clusterIngressModel(virtualKafkaCluster2)).isNotNull(); + + var assertList = assertThat(networkingModel.clusterNetworkingModels()).hasSize(2); + assertList.element(0).satisfies(clusterNetworkingModel -> { + assertThat(clusterNetworkingModel.cluster()).isEqualTo(virtualKafkaCluster); + assertThat(clusterNetworkingModel.ingressExceptions()).isEmpty(); + assertThat(clusterNetworkingModel.clusterIngressNetworkingModelResults()).singleElement().satisfies(model -> { + assertThat(model.exception()).isNull(); + assertThat(model.clusterIngressNetworkingModel()).isEqualTo(new ClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster, CLUSTER_IP_INGRESS, + List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), null, FIRST_IDENTIFYING_PORT, + FIRST_IDENTIFYING_PORT + 3)); + + }); + }); + assertList.element(1).satisfies(clusterNetworkingModel -> { + assertThat(clusterNetworkingModel.cluster()).isEqualTo(virtualKafkaCluster2); + assertThat(clusterNetworkingModel.ingressExceptions()).singleElement().satisfies(ingressConflictException -> { + assertThat(ingressConflictException).hasMessageContaining( + "Currently we do not support a virtual cluster with multiple ingresses that need unique ports to" + + " identify which node the client is connecting to"); + }); + assertThat(clusterNetworkingModel.clusterIngressNetworkingModelResults()).singleElement().satisfies(model -> { + assertThat(model.exception()).isNotNull().hasMessageContaining( + "Currently we do not support a virtual cluster with multiple ingresses that need unique ports to" + + " identify which node the client is connecting to"); + assertThat(model.clusterIngressNetworkingModel()).isEqualTo(new ClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster2, CLUSTER_IP_INGRESS_2, + List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), null, FIRST_IDENTIFYING_PORT + 4, + FIRST_IDENTIFYING_PORT + 7)); + + }); + }); + } + +} \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-KafkaProxy.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-KafkaProxy.yaml new file mode 100644 index 0000000000..db68c9e761 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-KafkaProxy.yaml @@ -0,0 +1,13 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: KafkaProxy +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: minimal + namespace: proxy-ns + generation: 1 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-KafkaProxyIngress-cluster-ip.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-KafkaProxyIngress-cluster-ip.yaml new file mode 100644 index 0000000000..acff4c190d --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-KafkaProxyIngress-cluster-ip.yaml @@ -0,0 +1,20 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: KafkaProxyIngress +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: cluster-ip + namespace: proxy-ns + generation: 2 +spec: + proxyRef: + name: minimal + clusterIP: + protocol: TLS +status: + observedGeneration: 2 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-KafkaProxyIngress-load-balancer.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-KafkaProxyIngress-load-balancer.yaml new file mode 100644 index 0000000000..d61db291e7 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-KafkaProxyIngress-load-balancer.yaml @@ -0,0 +1,21 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: KafkaProxyIngress +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: load-balancer + namespace: proxy-ns + generation: 2 +spec: + proxyRef: + name: minimal + loadBalancer: + bootstrapAddress: '$(virtualClusterName).kafkaproxy' + advertisedBrokerAddressPattern: '$(virtualClusterName)-$(nodeId).kafkaproxy' +status: + observedGeneration: 2 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-KafkaService-kafka.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-KafkaService-kafka.yaml new file mode 100644 index 0000000000..54c2a1e580 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-KafkaService-kafka.yaml @@ -0,0 +1,17 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: KafkaService +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: kafka + namespace: proxy-ns + generation: 3 +spec: + bootstrapServers: my-cluster-kafka-bootstrap.kafka.svc.cluster.local:9092 +status: + observedGeneration: 3 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-Secret-downstream-tls-cert.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-Secret-downstream-tls-cert.yaml new file mode 100644 index 0000000000..bff0dccf38 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-Secret-downstream-tls-cert.yaml @@ -0,0 +1,14 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +kind: Secret +apiVersion: v1 +metadata: + name: downstream-tls-cert +type: kubernetes.io/tls +data: + tls.crt: whatever + tls.key: whatever diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-VirtualKafkaCluster-one.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-VirtualKafkaCluster-one.yaml new file mode 100644 index 0000000000..36de2b68cb --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/in-VirtualKafkaCluster-one.yaml @@ -0,0 +1,31 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: VirtualKafkaCluster +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: one + namespace: proxy-ns + generation: 4 +spec: + proxyRef: + name: minimal + targetKafkaServiceRef: + name: kafka + ingresses: + - ingressRef: + name: load-balancer + tls: + certificateRef: + name: downstream-tls-cert + - ingressRef: + name: cluster-ip + tls: + certificateRef: + name: downstream-tls-cert +status: + observedGeneration: 4 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-ConfigMap-minimal-config-state.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-ConfigMap-minimal-config-state.yaml new file mode 100644 index 0000000000..c3c1d9fc76 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-ConfigMap-minimal-config-state.yaml @@ -0,0 +1,39 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "ConfigMap" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "minimal-config-state" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" +data: + cluster-one: |- + --- + metadata: + generation: 4 + name: "one" + namespace: "proxy-ns" + status: + conditions: + - observedGeneration: 4 + type: "Accepted" + status: "True" + lastTransitionTime: "1970-01-01T00:00:00Z" + reason: "Accepted" + message: "" + ingresses: [] + observedGeneration: 4 diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-ConfigMap-minimal-proxy-config.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-ConfigMap-minimal-proxy-config.yaml new file mode 100644 index 0000000000..51ee0b2c73 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-ConfigMap-minimal-proxy-config.yaml @@ -0,0 +1,53 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "ConfigMap" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "minimal-proxy-config" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" +data: + proxy-config.yaml: |- + --- + management: + endpoints: + prometheus: {} + virtualClusters: + - name: "one" + targetCluster: + bootstrapServers: "my-cluster-kafka-bootstrap.kafka.svc.cluster.local:9092" + gateways: + - name: "load-balancer" + sniHostIdentifiesNode: + bootstrapAddress: "$(virtualClusterName).kafkaproxy:9291" + advertisedBrokerAddressPattern: "$(virtualClusterName)-$(nodeId).kafkaproxy:9083" + tls: + key: + privateKeyFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.key" + certificateFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.crt" + - name: "cluster-ip" + portIdentifiesNode: + bootstrapAddress: "localhost:9292" + advertisedBrokerAddressPattern: "one-cluster-ip.proxy-ns.svc.cluster.local" + nodeIdRanges: + - name: "default" + start: 0 + end: 2 + tls: + key: + privateKeyFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.key" + certificateFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.crt" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Deployment-minimal.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Deployment-minimal.yaml new file mode 100644 index 0000000000..9c5ecd896f --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Deployment-minimal.yaml @@ -0,0 +1,97 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + labels: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "minimal" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" +spec: + replicas: 1 + selector: + matchLabels: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + template: + metadata: + labels: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + annotations: + kroxylicious.io/referent-checksum: "AAAAAAAB4wY" + spec: + containers: + - name: "proxy" + image: "quay.io/kroxylicious/kroxylicious:test" + args: + - "--config" + - "/opt/kroxylicious/config/proxy-config.yaml" + livenessProbe: + failureThreshold: 3 + httpGet: + path: "/livez" + port: "management" + initialDelaySeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - containerPort: 9190 + name: "management" + - containerPort: 9292 + name: "9292-bootstrap" + - containerPort: 9293 + name: "9293-node" + - containerPort: 9294 + name: "9294-node" + - containerPort: 9295 + name: "9295-node" + - containerPort: 9291 + name: "shared-sni-port" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + readOnlyRootFilesystem: true + terminationMessagePolicy: "FallbackToLogsOnError" + volumeMounts: + - mountPath: "/opt/kroxylicious/config/proxy-config.yaml" + name: "config-volume" + subPath: "proxy-config.yaml" + - mountPath: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert" + name: "secrets-downstream-tls-cert" + readOnly: true + securityContext: + runAsNonRoot: true + seccompProfile: + type: "RuntimeDefault" + volumes: + - configMap: + name: "minimal-proxy-config" + name: "config-volume" + - name: "secrets-downstream-tls-cert" + secret: + secretName: "downstream-tls-cert" diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-minimal-sni.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-minimal-sni.yaml new file mode 100644 index 0000000000..dd4cf0e958 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-minimal-sni.yaml @@ -0,0 +1,36 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "minimal-sni" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" +spec: + ports: + - name: "sni-9083" + port: 9083 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + type: "LoadBalancer" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-one-cluster-ip.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-one-cluster-ip.yaml new file mode 100644 index 0000000000..f6f71092c7 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-one-cluster-ip.yaml @@ -0,0 +1,53 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - name: "one-9292" + port: 9292 + protocol: "TCP" + targetPort: 9292 + - name: "one-9293" + port: 9293 + protocol: "TCP" + targetPort: 9293 + - name: "one-9294" + port: 9294 + protocol: "TCP" + targetPort: 9294 + - name: "one-9295" + port: 9295 + protocol: "TCP" + targetPort: 9295 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-KafkaProxy.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-KafkaProxy.yaml new file mode 100644 index 0000000000..ae43ec77b6 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-KafkaProxy.yaml @@ -0,0 +1,13 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: KafkaProxy +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: twocluster + namespace: proxy-ns + generation: 1 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-KafkaProxyIngress-load-balancer.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-KafkaProxyIngress-load-balancer.yaml new file mode 100644 index 0000000000..0a55ca62d7 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-KafkaProxyIngress-load-balancer.yaml @@ -0,0 +1,21 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: KafkaProxyIngress +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: load-balancer + namespace: proxy-ns + generation: 2 +spec: + proxyRef: + name: twocluster + loadBalancer: + bootstrapAddress: '$(virtualClusterName).kafkaproxy' + advertisedBrokerAddressPattern: '$(virtualClusterName)-$(nodeId).kafkaproxy' +status: + observedGeneration: 2 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-KafkaService-barref.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-KafkaService-barref.yaml new file mode 100644 index 0000000000..3f05ae0214 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-KafkaService-barref.yaml @@ -0,0 +1,17 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: KafkaService +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: barref + namespace: proxy-ns + generation: 4 +spec: + bootstrapServers: first-kafka.kafka1.svc.cluster.local:9092 +status: + observedGeneration: 4 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-KafkaService-fooref.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-KafkaService-fooref.yaml new file mode 100644 index 0000000000..e7db39dc2d --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-KafkaService-fooref.yaml @@ -0,0 +1,17 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: KafkaService +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: fooref + namespace: proxy-ns + generation: 5 +spec: + bootstrapServers: second-kafka.kafka2.svc.cluster.local:9092 +status: + observedGeneration: 5 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-Secret-downstream-tls-cert.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-Secret-downstream-tls-cert.yaml new file mode 100644 index 0000000000..bff0dccf38 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-Secret-downstream-tls-cert.yaml @@ -0,0 +1,14 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +kind: Secret +apiVersion: v1 +metadata: + name: downstream-tls-cert +type: kubernetes.io/tls +data: + tls.crt: whatever + tls.key: whatever diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-VirtualKafkaCluster-bar.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-VirtualKafkaCluster-bar.yaml new file mode 100644 index 0000000000..681a7f786b --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-VirtualKafkaCluster-bar.yaml @@ -0,0 +1,26 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: VirtualKafkaCluster +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: bar + namespace: proxy-ns + generation: 6 +spec: + proxyRef: + name: twocluster + targetKafkaServiceRef: + name: barref + ingresses: + - ingressRef: + name: load-balancer + tls: + certificateRef: + name: downstream-tls-cert +status: + observedGeneration: 6 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-VirtualKafkaCluster-foo.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-VirtualKafkaCluster-foo.yaml new file mode 100644 index 0000000000..0be3b5d227 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/in-VirtualKafkaCluster-foo.yaml @@ -0,0 +1,33 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: VirtualKafkaCluster +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: foo + namespace: proxy-ns + generation: 7 +spec: + proxyRef: + name: twocluster + targetKafkaServiceRef: + name: fooref + ingresses: + - ingressRef: + name: load-balancer +status: + conditions: + - lastTransitionTime: "2025-05-27T01:28:04.162595871Z" + message: 'spec.ingresses[].tls: Inconsistent TLS configuration. kafkaproxyingress.kroxylicious.io/loadbalancer + in namespace ''my-proxy'' requires the use of TLS but the cluster ingress + (loadbalancer) does not define a tls object.' + observedGeneration: 7 + reason: InvalidReferencedResource + status: "False" + type: ResolvedRefs + ingresses: [] + observedGeneration: 7 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/out-ConfigMap-twocluster-config-state.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/out-ConfigMap-twocluster-config-state.yaml new file mode 100644 index 0000000000..7dee0c7cc8 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/out-ConfigMap-twocluster-config-state.yaml @@ -0,0 +1,57 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "ConfigMap" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "twocluster" + app.kubernetes.io/component: "proxy" + name: "twocluster-config-state" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "twocluster" +data: + cluster-foo: |- + --- + metadata: + generation: 7 + name: "foo" + namespace: "proxy-ns" + status: + conditions: + - observedGeneration: 7 + type: "ResolvedRefs" + status: "False" + lastTransitionTime: "1970-01-01T00:00:00Z" + reason: "InvalidReferencedResource" + message: "spec.ingresses[].tls: Inconsistent TLS configuration. kafkaproxyingress.kroxylicious.io/loadbalancer\ + \ in namespace 'my-proxy' requires the use of TLS but the cluster ingress (loadbalancer)\ + \ does not define a tls object." + ingresses: [] + observedGeneration: 7 + cluster-bar: |- + --- + metadata: + generation: 6 + name: "bar" + namespace: "proxy-ns" + status: + conditions: + - observedGeneration: 6 + type: "Accepted" + status: "True" + lastTransitionTime: "1970-01-01T00:00:00Z" + reason: "Accepted" + message: "" + ingresses: [] + observedGeneration: 6 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/out-ConfigMap-twocluster-proxy-config.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/out-ConfigMap-twocluster-proxy-config.yaml new file mode 100644 index 0000000000..05c6665db6 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/out-ConfigMap-twocluster-proxy-config.yaml @@ -0,0 +1,41 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "ConfigMap" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "twocluster" + app.kubernetes.io/component: "proxy" + name: "twocluster-proxy-config" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "twocluster" +data: + proxy-config.yaml: |- + --- + management: + endpoints: + prometheus: {} + virtualClusters: + - name: "bar" + targetCluster: + bootstrapServers: "first-kafka.kafka1.svc.cluster.local:9092" + gateways: + - name: "load-balancer" + sniHostIdentifiesNode: + bootstrapAddress: "$(virtualClusterName).kafkaproxy:9291" + advertisedBrokerAddressPattern: "$(virtualClusterName)-$(nodeId).kafkaproxy:9083" + tls: + key: + privateKeyFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.key" + certificateFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.crt" diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/out-Deployment-twocluster.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/out-Deployment-twocluster.yaml new file mode 100644 index 0000000000..1ce925841b --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/out-Deployment-twocluster.yaml @@ -0,0 +1,89 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + labels: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "twocluster" + app.kubernetes.io/component: "proxy" + name: "twocluster" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "twocluster" +spec: + replicas: 1 + selector: + matchLabels: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "twocluster" + app.kubernetes.io/component: "proxy" + template: + metadata: + labels: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "twocluster" + app.kubernetes.io/component: "proxy" + annotations: + kroxylicious.io/referent-checksum: "AAAAAAAB4wY" + spec: + containers: + - name: "proxy" + image: "quay.io/kroxylicious/kroxylicious:test" + args: + - "--config" + - "/opt/kroxylicious/config/proxy-config.yaml" + livenessProbe: + failureThreshold: 3 + httpGet: + path: "/livez" + port: "management" + initialDelaySeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - containerPort: 9190 + name: "management" + - containerPort: 9291 + name: "shared-sni-port" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + readOnlyRootFilesystem: true + terminationMessagePolicy: "FallbackToLogsOnError" + volumeMounts: + - mountPath: "/opt/kroxylicious/config/proxy-config.yaml" + name: "config-volume" + subPath: "proxy-config.yaml" + - mountPath: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert" + name: "secrets-downstream-tls-cert" + readOnly: true + securityContext: + runAsNonRoot: true + seccompProfile: + type: "RuntimeDefault" + volumes: + - configMap: + name: "twocluster-proxy-config" + name: "config-volume" + - name: "secrets-downstream-tls-cert" + secret: + secretName: "downstream-tls-cert" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/out-Service-twocluster-sni.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/out-Service-twocluster-sni.yaml new file mode 100644 index 0000000000..8488302d02 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress-with-no-downstream-tls-on-cluster/out-Service-twocluster-sni.yaml @@ -0,0 +1,36 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "twocluster" + app.kubernetes.io/component: "proxy" + name: "twocluster-sni" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "twocluster" +spec: + ports: + - name: "sni-9083" + port: 9083 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "twocluster" + app.kubernetes.io/component: "proxy" + type: "LoadBalancer" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/in-KafkaProxy.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/in-KafkaProxy.yaml new file mode 100644 index 0000000000..db68c9e761 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/in-KafkaProxy.yaml @@ -0,0 +1,13 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: KafkaProxy +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: minimal + namespace: proxy-ns + generation: 1 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/in-KafkaProxyIngress-load-balancer.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/in-KafkaProxyIngress-load-balancer.yaml new file mode 100644 index 0000000000..d61db291e7 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/in-KafkaProxyIngress-load-balancer.yaml @@ -0,0 +1,21 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: KafkaProxyIngress +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: load-balancer + namespace: proxy-ns + generation: 2 +spec: + proxyRef: + name: minimal + loadBalancer: + bootstrapAddress: '$(virtualClusterName).kafkaproxy' + advertisedBrokerAddressPattern: '$(virtualClusterName)-$(nodeId).kafkaproxy' +status: + observedGeneration: 2 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/in-KafkaService-kafka.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/in-KafkaService-kafka.yaml new file mode 100644 index 0000000000..54c2a1e580 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/in-KafkaService-kafka.yaml @@ -0,0 +1,17 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: KafkaService +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: kafka + namespace: proxy-ns + generation: 3 +spec: + bootstrapServers: my-cluster-kafka-bootstrap.kafka.svc.cluster.local:9092 +status: + observedGeneration: 3 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/in-Secret-downstream-tls-cert.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/in-Secret-downstream-tls-cert.yaml new file mode 100644 index 0000000000..bff0dccf38 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/in-Secret-downstream-tls-cert.yaml @@ -0,0 +1,14 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +kind: Secret +apiVersion: v1 +metadata: + name: downstream-tls-cert +type: kubernetes.io/tls +data: + tls.crt: whatever + tls.key: whatever diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/in-VirtualKafkaCluster-one.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/in-VirtualKafkaCluster-one.yaml new file mode 100644 index 0000000000..1dc4c98c1c --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/in-VirtualKafkaCluster-one.yaml @@ -0,0 +1,26 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: VirtualKafkaCluster +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: one + namespace: proxy-ns + generation: 4 +spec: + proxyRef: + name: minimal + targetKafkaServiceRef: + name: kafka + ingresses: + - ingressRef: + name: load-balancer + tls: + certificateRef: + name: downstream-tls-cert +status: + observedGeneration: 4 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/out-ConfigMap-minimal-config-state.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/out-ConfigMap-minimal-config-state.yaml new file mode 100644 index 0000000000..c3c1d9fc76 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/out-ConfigMap-minimal-config-state.yaml @@ -0,0 +1,39 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "ConfigMap" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "minimal-config-state" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" +data: + cluster-one: |- + --- + metadata: + generation: 4 + name: "one" + namespace: "proxy-ns" + status: + conditions: + - observedGeneration: 4 + type: "Accepted" + status: "True" + lastTransitionTime: "1970-01-01T00:00:00Z" + reason: "Accepted" + message: "" + ingresses: [] + observedGeneration: 4 diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/out-ConfigMap-minimal-proxy-config.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/out-ConfigMap-minimal-proxy-config.yaml new file mode 100644 index 0000000000..0bdaa44081 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/out-ConfigMap-minimal-proxy-config.yaml @@ -0,0 +1,41 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "ConfigMap" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "minimal-proxy-config" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" +data: + proxy-config.yaml: |- + --- + management: + endpoints: + prometheus: {} + virtualClusters: + - name: "one" + targetCluster: + bootstrapServers: "my-cluster-kafka-bootstrap.kafka.svc.cluster.local:9092" + gateways: + - name: "load-balancer" + sniHostIdentifiesNode: + bootstrapAddress: "$(virtualClusterName).kafkaproxy:9291" + advertisedBrokerAddressPattern: "$(virtualClusterName)-$(nodeId).kafkaproxy:9083" + tls: + key: + privateKeyFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.key" + certificateFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.crt" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/out-Deployment-minimal.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/out-Deployment-minimal.yaml new file mode 100644 index 0000000000..4fbb76526d --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/out-Deployment-minimal.yaml @@ -0,0 +1,89 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + labels: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "minimal" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" +spec: + replicas: 1 + selector: + matchLabels: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + template: + metadata: + labels: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + annotations: + kroxylicious.io/referent-checksum: "AAAAAAAB4wY" + spec: + containers: + - name: "proxy" + image: "quay.io/kroxylicious/kroxylicious:test" + args: + - "--config" + - "/opt/kroxylicious/config/proxy-config.yaml" + livenessProbe: + failureThreshold: 3 + httpGet: + path: "/livez" + port: "management" + initialDelaySeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - containerPort: 9190 + name: "management" + - containerPort: 9291 + name: "shared-sni-port" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + readOnlyRootFilesystem: true + terminationMessagePolicy: "FallbackToLogsOnError" + volumeMounts: + - mountPath: "/opt/kroxylicious/config/proxy-config.yaml" + name: "config-volume" + subPath: "proxy-config.yaml" + - mountPath: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert" + name: "secrets-downstream-tls-cert" + readOnly: true + securityContext: + runAsNonRoot: true + seccompProfile: + type: "RuntimeDefault" + volumes: + - configMap: + name: "minimal-proxy-config" + name: "config-volume" + - name: "secrets-downstream-tls-cert" + secret: + secretName: "downstream-tls-cert" diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/out-Service-minimal-sni.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/out-Service-minimal-sni.yaml new file mode 100644 index 0000000000..dd4cf0e958 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-ingress/out-Service-minimal-sni.yaml @@ -0,0 +1,36 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "minimal-sni" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" +spec: + ports: + - name: "sni-9083" + port: 9083 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + type: "LoadBalancer" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-KafkaProxy.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-KafkaProxy.yaml new file mode 100644 index 0000000000..ae43ec77b6 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-KafkaProxy.yaml @@ -0,0 +1,13 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: KafkaProxy +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: twocluster + namespace: proxy-ns + generation: 1 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-KafkaProxyIngress-load-balancer.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-KafkaProxyIngress-load-balancer.yaml new file mode 100644 index 0000000000..0a55ca62d7 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-KafkaProxyIngress-load-balancer.yaml @@ -0,0 +1,21 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: KafkaProxyIngress +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: load-balancer + namespace: proxy-ns + generation: 2 +spec: + proxyRef: + name: twocluster + loadBalancer: + bootstrapAddress: '$(virtualClusterName).kafkaproxy' + advertisedBrokerAddressPattern: '$(virtualClusterName)-$(nodeId).kafkaproxy' +status: + observedGeneration: 2 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-KafkaService-barref.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-KafkaService-barref.yaml new file mode 100644 index 0000000000..3f05ae0214 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-KafkaService-barref.yaml @@ -0,0 +1,17 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: KafkaService +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: barref + namespace: proxy-ns + generation: 4 +spec: + bootstrapServers: first-kafka.kafka1.svc.cluster.local:9092 +status: + observedGeneration: 4 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-KafkaService-fooref.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-KafkaService-fooref.yaml new file mode 100644 index 0000000000..e7db39dc2d --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-KafkaService-fooref.yaml @@ -0,0 +1,17 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: KafkaService +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: fooref + namespace: proxy-ns + generation: 5 +spec: + bootstrapServers: second-kafka.kafka2.svc.cluster.local:9092 +status: + observedGeneration: 5 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-Secret-downstream-tls-cert.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-Secret-downstream-tls-cert.yaml new file mode 100644 index 0000000000..bff0dccf38 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-Secret-downstream-tls-cert.yaml @@ -0,0 +1,14 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +kind: Secret +apiVersion: v1 +metadata: + name: downstream-tls-cert +type: kubernetes.io/tls +data: + tls.crt: whatever + tls.key: whatever diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-VirtualKafkaCluster-bar.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-VirtualKafkaCluster-bar.yaml new file mode 100644 index 0000000000..681a7f786b --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-VirtualKafkaCluster-bar.yaml @@ -0,0 +1,26 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: VirtualKafkaCluster +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: bar + namespace: proxy-ns + generation: 6 +spec: + proxyRef: + name: twocluster + targetKafkaServiceRef: + name: barref + ingresses: + - ingressRef: + name: load-balancer + tls: + certificateRef: + name: downstream-tls-cert +status: + observedGeneration: 6 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-VirtualKafkaCluster-foo.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-VirtualKafkaCluster-foo.yaml new file mode 100644 index 0000000000..1cc1297f87 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/in-VirtualKafkaCluster-foo.yaml @@ -0,0 +1,26 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: VirtualKafkaCluster +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: foo + namespace: proxy-ns + generation: 7 +spec: + proxyRef: + name: twocluster + targetKafkaServiceRef: + name: fooref + ingresses: + - ingressRef: + name: load-balancer + tls: + certificateRef: + name: downstream-tls-cert +status: + observedGeneration: 7 \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/out-ConfigMap-twocluster-config-state.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/out-ConfigMap-twocluster-config-state.yaml new file mode 100644 index 0000000000..4c1d76d042 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/out-ConfigMap-twocluster-config-state.yaml @@ -0,0 +1,56 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "ConfigMap" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "twocluster" + app.kubernetes.io/component: "proxy" + name: "twocluster-config-state" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "twocluster" +data: + cluster-bar: |- + --- + metadata: + generation: 6 + name: "bar" + namespace: "proxy-ns" + status: + conditions: + - observedGeneration: 6 + type: "Accepted" + status: "True" + lastTransitionTime: "1970-01-01T00:00:00Z" + reason: "Accepted" + message: "" + ingresses: [] + observedGeneration: 6 + cluster-foo: |- + --- + metadata: + generation: 7 + name: "foo" + namespace: "proxy-ns" + status: + conditions: + - observedGeneration: 7 + type: "Accepted" + status: "True" + lastTransitionTime: "1970-01-01T00:00:00Z" + reason: "Accepted" + message: "" + ingresses: [] + observedGeneration: 7 + diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/out-ConfigMap-twocluster-proxy-config.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/out-ConfigMap-twocluster-proxy-config.yaml new file mode 100644 index 0000000000..4df9e497c0 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/out-ConfigMap-twocluster-proxy-config.yaml @@ -0,0 +1,53 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "ConfigMap" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "twocluster" + app.kubernetes.io/component: "proxy" + name: "twocluster-proxy-config" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "twocluster" +data: + proxy-config.yaml: |- + --- + management: + endpoints: + prometheus: {} + virtualClusters: + - name: "bar" + targetCluster: + bootstrapServers: "first-kafka.kafka1.svc.cluster.local:9092" + gateways: + - name: "load-balancer" + sniHostIdentifiesNode: + bootstrapAddress: "$(virtualClusterName).kafkaproxy:9291" + advertisedBrokerAddressPattern: "$(virtualClusterName)-$(nodeId).kafkaproxy:9083" + tls: + key: + privateKeyFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.key" + certificateFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.crt" + - name: "foo" + targetCluster: + bootstrapServers: "second-kafka.kafka2.svc.cluster.local:9092" + gateways: + - name: "load-balancer" + sniHostIdentifiesNode: + bootstrapAddress: "$(virtualClusterName).kafkaproxy:9291" + advertisedBrokerAddressPattern: "$(virtualClusterName)-$(nodeId).kafkaproxy:9083" + tls: + key: + privateKeyFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.key" + certificateFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.crt" diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/out-Deployment-twocluster.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/out-Deployment-twocluster.yaml new file mode 100644 index 0000000000..1ce925841b --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/out-Deployment-twocluster.yaml @@ -0,0 +1,89 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + labels: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "twocluster" + app.kubernetes.io/component: "proxy" + name: "twocluster" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "twocluster" +spec: + replicas: 1 + selector: + matchLabels: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "twocluster" + app.kubernetes.io/component: "proxy" + template: + metadata: + labels: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "twocluster" + app.kubernetes.io/component: "proxy" + annotations: + kroxylicious.io/referent-checksum: "AAAAAAAB4wY" + spec: + containers: + - name: "proxy" + image: "quay.io/kroxylicious/kroxylicious:test" + args: + - "--config" + - "/opt/kroxylicious/config/proxy-config.yaml" + livenessProbe: + failureThreshold: 3 + httpGet: + path: "/livez" + port: "management" + initialDelaySeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - containerPort: 9190 + name: "management" + - containerPort: 9291 + name: "shared-sni-port" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + readOnlyRootFilesystem: true + terminationMessagePolicy: "FallbackToLogsOnError" + volumeMounts: + - mountPath: "/opt/kroxylicious/config/proxy-config.yaml" + name: "config-volume" + subPath: "proxy-config.yaml" + - mountPath: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert" + name: "secrets-downstream-tls-cert" + readOnly: true + securityContext: + runAsNonRoot: true + seccompProfile: + type: "RuntimeDefault" + volumes: + - configMap: + name: "twocluster-proxy-config" + name: "config-volume" + - name: "secrets-downstream-tls-cert" + secret: + secretName: "downstream-tls-cert" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/out-Service-twocluster-sni.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/out-Service-twocluster-sni.yaml new file mode 100644 index 0000000000..8488302d02 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/two-clusters-with-shared-loadbalancer-ingress/out-Service-twocluster-sni.yaml @@ -0,0 +1,36 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "twocluster" + app.kubernetes.io/component: "proxy" + name: "twocluster-sni" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "twocluster" +spec: + ports: + - name: "sni-9083" + port: 9083 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "twocluster" + app.kubernetes.io/component: "proxy" + type: "LoadBalancer" \ No newline at end of file From e84334b1021f877b6b3106d0f2fa367156f66aa0 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Wed, 4 Jun 2025 12:00:49 +1200 Subject: [PATCH 62/83] Simplify mocking Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../VirtualKafkaClusterReconcilerTest.java | 245 ++++++++---------- 1 file changed, 101 insertions(+), 144 deletions(-) diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java index 8f5601bf2e..d320471f70 100644 --- a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java @@ -68,6 +68,7 @@ import io.kroxylicious.kubernetes.operator.resolver.DependencyResolver; import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; import static io.kroxylicious.kubernetes.operator.ResourcesUtil.name; import static org.assertj.core.api.Assertions.assertThat; @@ -110,6 +111,8 @@ class VirtualKafkaClusterReconcilerTest { .endSpec() .build(); + private static final ConfigMap NO_FILTERS_CONFIG_MAP=buildProxyConfigMapWithPatch(CLUSTER_NO_FILTERS); + private static final VirtualKafkaCluster CLUSTER_TLS_NO_FILTERS = new VirtualKafkaClusterBuilder(CLUSTER_NO_FILTERS) .editOrNewSpec() .withIngresses(new IngressesBuilder(CLUSTER_NO_FILTERS.getSpec().getIngresses().get(0)) @@ -139,6 +142,7 @@ class VirtualKafkaClusterReconcilerTest { .build()) .endSpec() .build(); + private static final VirtualKafkaCluster CLUSTER_TLS_NO_FILTERS_WITH_SECRET_WRONG_RESOURCE_TYPE = new VirtualKafkaClusterBuilder(CLUSTER_NO_FILTERS) .editOrNewSpec() .withIngresses(new IngressesBuilder(CLUSTER_NO_FILTERS.getSpec().getIngresses().get(0)) @@ -266,11 +270,8 @@ static List shouldSetResolvedRefs() { List result = new ArrayList<>(); { - Context> reconcilerContext = mockReconcilerContext(); + Context reconcilerContext = mockReconcilerContext(PROXY, INGRESS, SERVICE, null, Set.of()); - when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); - when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); result.add(Arguments.argumentSet("no filter", CLUSTER_NO_FILTERS, reconcilerContext, @@ -280,12 +281,11 @@ static List shouldSetResolvedRefs() { } { - Context> reconcilerContext = mockReconcilerContext(); - when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); - when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); - when(reconcilerContext.getSecondaryResource(ConfigMap.class)).thenReturn(Optional.of(buildProxyConfigMapWithPatch(CLUSTER_ONE_FILTER))); - when(reconcilerContext.getSecondaryResources(KafkaProtocolFilter.class)).thenReturn(Set.of(FILTER_MY_FILTER)); + ConfigMap proxyConfigMap = buildProxyConfigMapWithPatch(CLUSTER_ONE_FILTER); + Set filters = Set.of(FILTER_MY_FILTER); + + Context reconcilerContext = mockReconcilerContext(PROXY, INGRESS, SERVICE, proxyConfigMap, filters); + result.add(Arguments.argumentSet("one filter", CLUSTER_ONE_FILTER, reconcilerContext, @@ -296,13 +296,12 @@ static List shouldSetResolvedRefs() { { - Context> reconcilerContext = mockReconcilerContext(); - when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); - when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); - when(reconcilerContext.getSecondaryResource(ConfigMap.class)).thenReturn(Optional.of(buildProxyConfigMapWithPatch( - new VirtualKafkaClusterBuilder(CLUSTER_ONE_FILTER).editMetadata().withGeneration(40L).endMetadata().build()))); - when(reconcilerContext.getSecondaryResources(KafkaProtocolFilter.class)).thenReturn(Set.of(FILTER_MY_FILTER)); + ConfigMap proxyConfigMap = buildProxyConfigMapWithPatch( + new VirtualKafkaClusterBuilder(CLUSTER_ONE_FILTER).editMetadata().withGeneration(40L).endMetadata().build()); + Set filters = Set.of(FILTER_MY_FILTER); + + Context reconcilerContext = mockReconcilerContext(PROXY, INGRESS, SERVICE, proxyConfigMap, filters); + result.add(Arguments.argumentSet("one filter with stale configmap", new VirtualKafkaClusterBuilder(CLUSTER_ONE_FILTER).editOrNewStatus().withObservedGeneration(ResourcesUtil.generation(CLUSTER_NO_FILTERS)) .endStatus().build(), @@ -313,9 +312,8 @@ static List shouldSetResolvedRefs() { } { - Context> reconcilerContext = mockReconcilerContext(); - when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); - when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + Context reconcilerContext = mockReconcilerContext(null, INGRESS, SERVICE, null, Set.of()); + result.add(Arguments.argumentSet("proxy not found", CLUSTER_NO_FILTERS, reconcilerContext, @@ -327,9 +325,7 @@ static List shouldSetResolvedRefs() { } { - Context> reconcilerContext = mockReconcilerContext(); - when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); + Context reconcilerContext = mockReconcilerContext(PROXY, INGRESS, null, null, Set.of()); result.add(Arguments.argumentSet("service not found", CLUSTER_NO_FILTERS, reconcilerContext, @@ -341,11 +337,10 @@ static List shouldSetResolvedRefs() { } { - Context> reconcilerContext = mockReconcilerContext(); - when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn( - Set.of(INGRESS.edit().editSpec().withNewProxyRef().withName("not-my-proxy").endProxyRef().endSpec().build())); - when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + + KafkaProxyIngress ingress = INGRESS.edit().editSpec().withNewProxyRef().withName("not-my-proxy").endProxyRef().endSpec().build(); + Context reconcilerContext = mockReconcilerContext(PROXY, ingress, SERVICE, null, Set.of()); + result.add(Arguments.argumentSet("ingress refers to a different proxy than virtual cluster", CLUSTER_NO_FILTERS, reconcilerContext, @@ -357,17 +352,16 @@ static List shouldSetResolvedRefs() { } { - Context> reconcilerContext = mockReconcilerContext(); - when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); - when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(new KafkaServiceBuilder(SERVICE).withNewStatus().addNewCondition() + KafkaService service = new KafkaServiceBuilder(SERVICE).withNewStatus().addNewCondition() .withType(Condition.Type.ResolvedRefs) .withStatus(Condition.Status.FALSE) .withLastTransitionTime(Instant.now()) .withObservedGeneration(SERVICE.getMetadata().getGeneration()) .withReason("NO_FILTERS") .withMessage("no filters found") - .endCondition().endStatus().build())); + .endCondition().endStatus().build(); + Context reconcilerContext = mockReconcilerContext(PROXY, INGRESS, service, null, Set.of()); + result.add(Arguments.argumentSet("service has ResolvedRefs=False condition", CLUSTER_NO_FILTERS, reconcilerContext, @@ -379,9 +373,7 @@ static List shouldSetResolvedRefs() { } { - Context> reconcilerContext = mockReconcilerContext(); - when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + Context reconcilerContext = mockReconcilerContext(PROXY, null, SERVICE, null, Set.of()); result.add(Arguments.argumentSet("ingress not found", CLUSTER_NO_FILTERS, reconcilerContext, @@ -393,17 +385,16 @@ static List shouldSetResolvedRefs() { } { - Context> reconcilerContext = mockReconcilerContext(); - when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); - when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn( - Set.of(new KafkaProxyIngressBuilder(INGRESS).withNewStatus().addNewCondition().withType(Condition.Type.ResolvedRefs) - .withStatus(Condition.Status.FALSE) - .withLastTransitionTime(Instant.now()) - .withObservedGeneration(INGRESS.getMetadata().getGeneration()) - .withReason("NO_FILTERS") - .withMessage("no filters found") - .endCondition().endStatus().build())); + + KafkaProxyIngress ingress = new KafkaProxyIngressBuilder(INGRESS).withNewStatus().addNewCondition().withType(Condition.Type.ResolvedRefs) + .withStatus(Condition.Status.FALSE) + .withLastTransitionTime(Instant.now()) + .withObservedGeneration(INGRESS.getMetadata().getGeneration()) + .withReason("NO_FILTERS") + .withMessage("no filters found") + .endCondition().endStatus().build(); + + Context reconcilerContext = mockReconcilerContext(PROXY, ingress, SERVICE, null, Set.of()); result.add(Arguments.argumentSet("ingress has ResolvedRefs=False condition", CLUSTER_NO_FILTERS, @@ -416,10 +407,8 @@ static List shouldSetResolvedRefs() { } { - Context> reconcilerContext = mockReconcilerContext(); - when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); - when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + Context reconcilerContext = mockReconcilerContext(PROXY, INGRESS, SERVICE, null, Set.of()); + result.add(Arguments.argumentSet("filter not found", CLUSTER_ONE_FILTER, reconcilerContext, @@ -431,11 +420,7 @@ static List shouldSetResolvedRefs() { } { - Context> reconcilerContext = mockReconcilerContext(); - when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); - when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); - when(reconcilerContext.getSecondaryResources(KafkaProtocolFilter.class)).thenReturn(Set.of(new KafkaProtocolFilterBuilder(FILTER_MY_FILTER).withNewStatus() + Set filters = Set.of(new KafkaProtocolFilterBuilder(FILTER_MY_FILTER).withNewStatus() .addNewCondition() .withType(Condition.Type.ResolvedRefs) .withStatus(Condition.Status.FALSE) @@ -443,7 +428,8 @@ static List shouldSetResolvedRefs() { .withObservedGeneration(FILTER_MY_FILTER.getMetadata().getGeneration()) .withReason("RESOLVE_FAILURE") .withMessage("failed to resolve") - .endCondition().endStatus().build())); + .endCondition().endStatus().build()); + Context reconcilerContext = mockReconcilerContext(PROXY, INGRESS, SERVICE, null, filters); result.add(Arguments.argumentSet("filter has ResolvedRefs=False condition", CLUSTER_ONE_FILTER, @@ -456,11 +442,9 @@ static List shouldSetResolvedRefs() { } { - Context> reconcilerContext = mockReconcilerContext(); + Context reconcilerContext = mockReconcilerContext(PROXY, INGRESS_WITH_TLS, SERVICE, null, Set.of()); mockGetSecret(reconcilerContext, Optional.of(KUBE_TLS_CERT_SECRET)); - when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); - when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + result.add(Arguments.argumentSet("cluster with tls", CLUSTER_TLS_NO_FILTERS, reconcilerContext, @@ -470,12 +454,10 @@ static List shouldSetResolvedRefs() { } { - Context> reconcilerContext = mockReconcilerContext(); + Context reconcilerContext = mockReconcilerContext(PROXY, INGRESS_WITH_TLS, SERVICE, null, Set.of()); mockGetSecret(reconcilerContext, Optional.empty()); - when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); - when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + result.add(Arguments.argumentSet("cluster with tls - server cert secret not found", CLUSTER_TLS_NO_FILTERS, reconcilerContext, @@ -487,12 +469,10 @@ static List shouldSetResolvedRefs() { } { - Context> reconcilerContext = mockReconcilerContext(); + Context reconcilerContext = mockReconcilerContext(PROXY, INGRESS_WITH_TLS, SERVICE, null, Set.of()); mockGetSecret(reconcilerContext, Optional.of(NON_KUBE_TLS_CERT_SECRET)); - when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); - when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + result.add(Arguments.argumentSet("cluster with tls - server cert secret wrong type", CLUSTER_TLS_NO_FILTERS, reconcilerContext, @@ -504,12 +484,10 @@ static List shouldSetResolvedRefs() { } { - Context> reconcilerContext = mockReconcilerContext(); + Context reconcilerContext = mockReconcilerContext(PROXY, INGRESS_WITH_TLS, SERVICE, null, Set.of()); mockGetSecret(reconcilerContext, Optional.empty()); - when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); - when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + result.add(Arguments.argumentSet("cluster with tls - server cert unsupported resource type", CLUSTER_TLS_NO_FILTERS_WITH_SECRET_WRONG_RESOURCE_TYPE, reconcilerContext, @@ -521,12 +499,11 @@ static List shouldSetResolvedRefs() { } { - Context> reconcilerContext = mockReconcilerContext(); + Context reconcilerContext = mockReconcilerContext(PROXY, INGRESS_WITH_TLS, SERVICE, null, Set.of()); + mockGetSecret(reconcilerContext, Optional.of(KUBE_TLS_CERT_SECRET)); mockGetConfigMap(reconcilerContext, Optional.of(PEM_CONFIG_MAP)); - when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); - when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + result.add(Arguments.argumentSet("cluster with tls with trust anchor", CLUSTER_TLS_NO_FILTERS_WITH_TRUST_ANCHOR, reconcilerContext, @@ -536,13 +513,11 @@ static List shouldSetResolvedRefs() { } { - Context> reconcilerContext = mockReconcilerContext(); + Context reconcilerContext = mockReconcilerContext(PROXY, INGRESS_WITH_TLS, SERVICE, null, Set.of()); mockGetSecret(reconcilerContext, Optional.of(KUBE_TLS_CERT_SECRET)); mockGetConfigMap(reconcilerContext, Optional.empty()); - when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); - when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); + result.add(Arguments.argumentSet("cluster with tls - trust anchor cert configmap not found", CLUSTER_TLS_NO_FILTERS_WITH_TRUST_ANCHOR, reconcilerContext, @@ -554,11 +529,8 @@ static List shouldSetResolvedRefs() { } { - Context> reconcilerContext = mockReconcilerContext(); + Context reconcilerContext = mockReconcilerContext(PROXY, INGRESS, SERVICE, null, Set.of()); - when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); - when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); result.add(Arguments.argumentSet("cluster defines tls, ingress does not", CLUSTER_TLS_NO_FILTERS, reconcilerContext, @@ -570,11 +542,8 @@ static List shouldSetResolvedRefs() { } { - Context> reconcilerContext = mockReconcilerContext(); + Context reconcilerContext = mockReconcilerContext(PROXY, INGRESS_WITH_TLS, SERVICE, null, Set.of()); - when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); - when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); result.add(Arguments.argumentSet("cluster does not define tls, ingress does", CLUSTER_NO_FILTERS, reconcilerContext, @@ -588,11 +557,23 @@ static List shouldSetResolvedRefs() { return result; } + private static Context mockReconcilerContext(@Nullable KafkaProxy proxy, @Nullable KafkaProxyIngress ingress, @Nullable KafkaService service, + @Nullable ConfigMap proxyConfigMap, Set filters) { + Context reconcilerContext = mock(); + when(reconcilerContext.managedWorkflowAndDependentResourceContext()).thenReturn(workflowContext); + + when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(setOfOrEmpty(proxy)); + when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(setOfOrEmpty(ingress)); + when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(setOfOrEmpty(service)); + when(reconcilerContext.getSecondaryResource(ConfigMap.class)).thenReturn(Optional.ofNullable(proxyConfigMap)); + when(reconcilerContext.getSecondaryResources(KafkaProtocolFilter.class)).thenReturn(filters); + + return reconcilerContext; + } + @NonNull - private static Context> mockReconcilerContext() { - Context> context = mock(); - when(context.managedWorkflowAndDependentResourceContext()).thenReturn(workflowContext); - return context; + private static Set setOfOrEmpty(@Nullable T item) { + return item != null ? Set.of(item) : Set.of(); } @NonNull @@ -651,19 +632,12 @@ void shouldSetResolvedRefsToUnknown() { void shouldSetIngressStatusForClusterIPIngress() { // given var reconciler = new VirtualKafkaClusterReconciler(TEST_CLOCK, DependencyResolver.create()); + Context reconcilerContext = mockReconcilerContext(PROXY, INGRESS, SERVICE, NO_FILTERS_CONFIG_MAP, Set.of()); - Context context = mock(); - when(context.managedWorkflowAndDependentResourceContext()).thenReturn(workflowContext); - - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResource(ConfigMap.class)).thenReturn(Optional.of(buildProxyConfigMapWithPatch(CLUSTER_NO_FILTERS))); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); - when(context.getSecondaryResources(KafkaProtocolFilter.class)).thenReturn(Set.of()); - when(context.getSecondaryResources(Service.class)).thenReturn(Set.of(KUBERNETES_INGRESS_SERVICES)); + when(reconcilerContext.getSecondaryResources(Service.class)).thenReturn(Set.of(KUBERNETES_INGRESS_SERVICES)); // when - var update = reconciler.reconcile(CLUSTER_NO_FILTERS, context); + var update = reconciler.reconcile(CLUSTER_NO_FILTERS, reconcilerContext); // then assertThat(update).isNotNull(); @@ -687,18 +661,11 @@ void shouldOmitIngressIfKubernetesServiceNotPresent() { // given var reconciler = new VirtualKafkaClusterReconciler(TEST_CLOCK, DependencyResolver.create()); - Context context = mock(); - when(context.managedWorkflowAndDependentResourceContext()).thenReturn(workflowContext); - - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResource(ConfigMap.class)).thenReturn(Optional.of(buildProxyConfigMapWithPatch(CLUSTER_NO_FILTERS))); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS)); - when(context.getSecondaryResources(KafkaProtocolFilter.class)).thenReturn(Set.of()); - when(context.getSecondaryResources(Service.class)).thenReturn(Set.of()); + Context reconcilerContext = mockReconcilerContext(PROXY, INGRESS, SERVICE, NO_FILTERS_CONFIG_MAP, Set.of()); + when(reconcilerContext.getSecondaryResources(Service.class)).thenReturn(Set.of()); // when - var update = reconciler.reconcile(CLUSTER_NO_FILTERS, context); + var update = reconciler.reconcile(CLUSTER_NO_FILTERS, reconcilerContext); // then assertThat(update).isNotNull(); @@ -994,21 +961,18 @@ void kafkaServiceSecondaryToPrimaryMapperIgnoresServiceWithStaleStatus() { @Test void shouldIncludeDownstreamTlsSecretInChecksum() { // Given - Context context = mock(); + Context reconcilerContext = mockReconcilerContext(PROXY, INGRESS_WITH_TLS, SERVICE, buildProxyConfigMapWithPatch(CLUSTER_TLS_NO_FILTERS), + Set.of()); + MetadataChecksumGenerator checksumGenerator = mock(MetadataChecksumGenerator.class); when(workflowContext.get(MetadataChecksumGenerator.CHECKSUM_CONTEXT_KEY, MetadataChecksumGenerator.class)).thenReturn(Optional.of(checksumGenerator)); - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResource(ConfigMap.class)).thenReturn(Optional.of(buildProxyConfigMapWithPatch(CLUSTER_TLS_NO_FILTERS))); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); - when(context.getSecondaryResources(KafkaProtocolFilter.class)).thenReturn(Set.of()); - when(context.getSecondaryResources(Service.class)).thenReturn(Set.of(KUBERNETES_INGRESS_SERVICES)); - when(context.getSecondaryResource(Secret.class, "secrets")).thenReturn(Optional.of(KUBE_TLS_CERT_SECRET)); - when(context.getSecondaryResourcesAsStream(Secret.class)).thenReturn(Stream.of(KUBE_TLS_CERT_SECRET)); - when(context.managedWorkflowAndDependentResourceContext()).thenReturn(workflowContext); + + when(reconcilerContext.getSecondaryResources(Service.class)).thenReturn(Set.of(KUBERNETES_INGRESS_SERVICES)); + when(reconcilerContext.getSecondaryResource(Secret.class, "secrets")).thenReturn(Optional.of(KUBE_TLS_CERT_SECRET)); + when(reconcilerContext.getSecondaryResourcesAsStream(Secret.class)).thenReturn(Stream.of(KUBE_TLS_CERT_SECRET)); // When - virtualKafkaClusterReconciler.reconcile(CLUSTER_TLS_NO_FILTERS, context); + virtualKafkaClusterReconciler.reconcile(CLUSTER_TLS_NO_FILTERS, reconcilerContext); // Then verify(checksumGenerator).appendMetadata(KUBE_TLS_CERT_SECRET); @@ -1017,23 +981,19 @@ void shouldIncludeDownstreamTlsSecretInChecksum() { @Test void shouldIncludeDownstreamTlsTrustAnchorInChecksum() { // Given - Context context = mock(); + ConfigMap proxyConfigMap = buildProxyConfigMapWithPatch(CLUSTER_TLS_NO_FILTERS_WITH_TRUST_ANCHOR); + Context reconcilerContext = mockReconcilerContext(PROXY, INGRESS_WITH_TLS, SERVICE, proxyConfigMap, Set.of()); MetadataChecksumGenerator checksumGenerator = mock(MetadataChecksumGenerator.class); when(workflowContext.get(MetadataChecksumGenerator.CHECKSUM_CONTEXT_KEY, MetadataChecksumGenerator.class)).thenReturn(Optional.of(checksumGenerator)); - when(context.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(context.getSecondaryResource(ConfigMap.class)).thenReturn(Optional.of(buildProxyConfigMapWithPatch(CLUSTER_TLS_NO_FILTERS_WITH_TRUST_ANCHOR))); - when(context.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); - when(context.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); - when(context.getSecondaryResources(KafkaProtocolFilter.class)).thenReturn(Set.of()); - when(context.getSecondaryResources(Service.class)).thenReturn(Set.of(KUBERNETES_INGRESS_SERVICES)); - when(context.getSecondaryResource(Secret.class, "secrets")).thenReturn(Optional.of(KUBE_TLS_CERT_SECRET)); - when(context.getSecondaryResource(ConfigMap.class, "configmaps")).thenReturn(Optional.of(PEM_CONFIG_MAP)); - when(context.getSecondaryResourcesAsStream(Secret.class)).thenReturn(Stream.of(KUBE_TLS_CERT_SECRET)); - when(context.getSecondaryResourcesAsStream(ConfigMap.class)).thenReturn(Stream.of(PEM_CONFIG_MAP)); - when(context.managedWorkflowAndDependentResourceContext()).thenReturn(workflowContext); + + when(reconcilerContext.getSecondaryResources(Service.class)).thenReturn(Set.of(KUBERNETES_INGRESS_SERVICES)); + when(reconcilerContext.getSecondaryResource(Secret.class, "secrets")).thenReturn(Optional.of(KUBE_TLS_CERT_SECRET)); + when(reconcilerContext.getSecondaryResource(ConfigMap.class, "configmaps")).thenReturn(Optional.of(PEM_CONFIG_MAP)); + when(reconcilerContext.getSecondaryResourcesAsStream(Secret.class)).thenReturn(Stream.of(KUBE_TLS_CERT_SECRET)); + when(reconcilerContext.getSecondaryResourcesAsStream(ConfigMap.class)).thenReturn(Stream.of(PEM_CONFIG_MAP)); // When - virtualKafkaClusterReconciler.reconcile(CLUSTER_TLS_NO_FILTERS_WITH_TRUST_ANCHOR, context); + virtualKafkaClusterReconciler.reconcile(CLUSTER_TLS_NO_FILTERS_WITH_TRUST_ANCHOR, reconcilerContext); // Then verify(checksumGenerator).appendMetadata(PEM_CONFIG_MAP); @@ -1042,19 +1002,16 @@ void shouldIncludeDownstreamTlsTrustAnchorInChecksum() { @Test void shouldCreateChecksumGeneratorIfNotPresentInReconcilerContext() { // Given - Context reconcilerContext = mock(); + ConfigMap proxyConfigMap = buildProxyConfigMapWithPatch(CLUSTER_TLS_NO_FILTERS_WITH_TRUST_ANCHOR); + Context reconcilerContext = mockReconcilerContext(PROXY, INGRESS_WITH_TLS, SERVICE, proxyConfigMap, Set.of()); when(workflowContext.get(MetadataChecksumGenerator.CHECKSUM_CONTEXT_KEY, MetadataChecksumGenerator.class)).thenReturn(Optional.empty()); - when(reconcilerContext.getSecondaryResources(KafkaProxy.class)).thenReturn(Set.of(PROXY)); - when(reconcilerContext.getSecondaryResource(ConfigMap.class)).thenReturn(Optional.of(buildProxyConfigMapWithPatch(CLUSTER_TLS_NO_FILTERS_WITH_TRUST_ANCHOR))); - when(reconcilerContext.getSecondaryResources(KafkaService.class)).thenReturn(Set.of(SERVICE)); - when(reconcilerContext.getSecondaryResources(KafkaProxyIngress.class)).thenReturn(Set.of(INGRESS_WITH_TLS)); - when(reconcilerContext.getSecondaryResources(KafkaProtocolFilter.class)).thenReturn(Set.of()); + + when(reconcilerContext.getSecondaryResource(ConfigMap.class)).thenReturn(Optional.of(proxyConfigMap)); when(reconcilerContext.getSecondaryResources(Service.class)).thenReturn(Set.of(KUBERNETES_INGRESS_SERVICES)); when(reconcilerContext.getSecondaryResource(Secret.class, "secrets")).thenReturn(Optional.of(KUBE_TLS_CERT_SECRET)); when(reconcilerContext.getSecondaryResource(ConfigMap.class, "configmaps")).thenReturn(Optional.of(PEM_CONFIG_MAP)); when(reconcilerContext.getSecondaryResourcesAsStream(Secret.class)).thenReturn(Stream.of(KUBE_TLS_CERT_SECRET)); when(reconcilerContext.getSecondaryResourcesAsStream(ConfigMap.class)).thenReturn(Stream.of(PEM_CONFIG_MAP)); - when(reconcilerContext.managedWorkflowAndDependentResourceContext()).thenReturn(workflowContext); // When var actualUpdate = virtualKafkaClusterReconciler.reconcile(CLUSTER_TLS_NO_FILTERS_WITH_TRUST_ANCHOR, reconcilerContext); From ce9274ce04cb0195cf7a327839f6dbf4b19341dc Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Wed, 4 Jun 2025 16:04:34 +1200 Subject: [PATCH 63/83] Add whitespace around assignment. Formatter didn't fix it as it was in a massive block of formatter off. Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../operator/VirtualKafkaClusterReconcilerTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java index d320471f70..6fc9ab8902 100644 --- a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconcilerTest.java @@ -111,8 +111,6 @@ class VirtualKafkaClusterReconcilerTest { .endSpec() .build(); - private static final ConfigMap NO_FILTERS_CONFIG_MAP=buildProxyConfigMapWithPatch(CLUSTER_NO_FILTERS); - private static final VirtualKafkaCluster CLUSTER_TLS_NO_FILTERS = new VirtualKafkaClusterBuilder(CLUSTER_NO_FILTERS) .editOrNewSpec() .withIngresses(new IngressesBuilder(CLUSTER_NO_FILTERS.getSpec().getIngresses().get(0)) @@ -257,6 +255,8 @@ class VirtualKafkaClusterReconcilerTest { .build(); // @formatter:on + private static final ConfigMap NO_FILTERS_CONFIG_MAP = buildProxyConfigMapWithPatch(CLUSTER_NO_FILTERS); + private static final ManagedWorkflowAndDependentResourceContext workflowContext = mock(ManagedWorkflowAndDependentResourceContext.class); private VirtualKafkaClusterReconciler virtualKafkaClusterReconciler; From 8442679e6601b14e0a927819024e77a4817c883b Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Wed, 4 Jun 2025 13:40:20 +1200 Subject: [PATCH 64/83] Stop making calls to the API server when we don't need to. Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .../OperatorChangeDetectionST.java | 64 ++++++++++--------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java index 8750bc167e..f339360c39 100644 --- a/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java +++ b/kroxylicious-systemtests/src/test/java/io/kroxylicious/systemtests/OperatorChangeDetectionST.java @@ -37,7 +37,9 @@ import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxy; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyBuilder; import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngress; +import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngressBuilder; import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaCluster; +import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaClusterBuilder; import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP; import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilter; import io.kroxylicious.kubernetes.filter.api.v1alpha1.KafkaProtocolFilterBuilder; @@ -70,19 +72,18 @@ class OperatorChangeDetectionST extends AbstractST { void shouldUpdateDeploymentWhenKafkaProxyIngressChanges(String namespace) { // Given kroxylicious.deployPortIdentifiesNodeWithNoFilters(kafkaClusterName); - KubeClient kubeClient = kubeClient(namespace); - String originalChecksum = getInitialChecksum(namespace, kubeClient); + String originalChecksum = getInitialChecksum(namespace); - updateIngresProtocol(ClusterIP.Protocol.TLS, kubeClient, namespace); // move to TLS but this is invalid + updateIngresProtocol(ClusterIP.Protocol.TLS, namespace); // move to TLS but this is invalid // When // So move back to TCP and check things get updated. - updateIngresProtocol(ClusterIP.Protocol.TCP, kubeClient, namespace); + updateIngresProtocol(ClusterIP.Protocol.TCP, namespace); LOGGER.info("Kafka proxy ingress edited"); // Then - assertDeploymentUpdated(namespace, kubeClient, originalChecksum); + assertDeploymentUpdated(namespace, originalChecksum); } @Test @@ -97,13 +98,15 @@ void shouldUpdateDeploymentWhenVirtualKafkaClusterChanges(String namespace) { // @formatter:on resourceManager.createOrUpdateResourceWithWait(arbitraryFilter); kroxylicious.deployPortIdentifiesNodeWithNoFilters("test-vkc"); - KubeClient kubeClient = kubeClient(namespace); - String originalChecksum = getInitialChecksum(namespace, kubeClient); + String originalChecksum = getInitialChecksum(namespace); FilterRef filterRef = new FilterRefBuilder().withName("arbitrary-filter").build(); - VirtualKafkaCluster virtualKafkaCluster = kubeClient.getClient().resources(VirtualKafkaCluster.class).inNamespace(namespace) - .withName("test-vkc").get(); + VirtualKafkaCluster virtualKafkaCluster = new VirtualKafkaClusterBuilder().withNewMetadata() + .withName("test-vkc") + .withNamespace(namespace) + .and() + .build(); // When resourceManager.replaceResourceWithRetries(virtualKafkaCluster, vkc -> { @@ -115,7 +118,7 @@ void shouldUpdateDeploymentWhenVirtualKafkaClusterChanges(String namespace) { LOGGER.info("virtual cluster edited"); // Then - assertDeploymentUpdated(namespace, kubeClient, originalChecksum); + assertDeploymentUpdated(namespace, originalChecksum); } @Test @@ -131,9 +134,8 @@ void shouldUpdateDeploymentWhenDownstreamTlsCertUpdated(String namespace) throws var tls = kroxylicious.tlsConfigFromCert("server-certificate"); kroxylicious.deployPortIdentifiesNodeWithDownstreamTlsAndNoFilters("test-vkc", tls); - KubeClient kubeClient = kubeClient(namespace); - String originalChecksum = getInitialChecksum(namespace, kubeClient); + String originalChecksum = getInitialChecksum(namespace); // When resourceManager.replaceResourceWithRetries(cert.build(), @@ -145,7 +147,7 @@ void shouldUpdateDeploymentWhenDownstreamTlsCertUpdated(String namespace) throws LOGGER.info("SAN added to downstream tls cert"); // Then - assertDeploymentUpdated(namespace, kubeClient, originalChecksum); + assertDeploymentUpdated(namespace, originalChecksum); } @Test @@ -162,9 +164,8 @@ void shouldUpdateDeploymentWhenDownstreamTrustUpdated(String namespace) throws I var tls = kroxylicious.tlsConfigFromCert("server-certificate"); kroxylicious.deployPortIdentifiesNodeWithDownstreamTlsAndNoFilters("test-vkc", tls); - KubeClient kubeClient = kubeClient(namespace); - String originalChecksum = getInitialChecksum(namespace, kubeClient); + String originalChecksum = getInitialChecksum(namespace); ConfigMap trustAnchorConfig = new ConfigMapBuilder().withNewMetadata().withName(Constants.KROXYLICIOUS_TLS_CLIENT_CA_CERT).withNamespace(namespace).endMetadata() .build(); @@ -174,7 +175,7 @@ void shouldUpdateDeploymentWhenDownstreamTrustUpdated(String namespace) throws I LOGGER.info("Downstream trust updated"); // Then - assertDeploymentUpdated(namespace, kubeClient, originalChecksum); + assertDeploymentUpdated(namespace, originalChecksum); } @Test @@ -191,7 +192,7 @@ void shouldUpdateWhenFilterConfigurationChanges(String namespace) { resourceManager.createOrUpdateResourceWithWait(arbitraryFilter); kroxylicious.deployPortIdentifiesNodeWithFilters(kafkaClusterName, List.of("arbitrary-filter")); - String originalChecksum = getInitialChecksum(namespace, kubeClient); + String originalChecksum = getInitialChecksum(namespace); // @formatter:off KafkaProtocolFilterBuilder updatedProtocolFilter = kubeClient.getClient().resources(KafkaProtocolFilter.class) @@ -209,7 +210,7 @@ void shouldUpdateWhenFilterConfigurationChanges(String namespace) { LOGGER.info("Kafka proxy filter updated"); // Then - assertDeploymentUpdated(namespace, kubeClient, originalChecksum); + assertDeploymentUpdated(namespace, originalChecksum); } @Test @@ -218,7 +219,7 @@ void shouldUpdateDeploymentWhenKafkaProxyChanges(String namespace) { KubeClient kubeClient = kubeClient(namespace); kroxylicious.deployPortIdentifiesNodeWithNoFilters(kafkaClusterName); - String originalChecksum = getInitialChecksum(namespace, kubeClient); + String originalChecksum = getInitialChecksum(namespace); KafkaProxy kafkaProxy = kubeClient.getClient().resources(KafkaProxy.class).inNamespace(namespace) .withName(Constants.KROXYLICIOUS_PROXY_SIMPLE_NAME).get(); @@ -235,7 +236,7 @@ void shouldUpdateDeploymentWhenKafkaProxyChanges(String namespace) { LOGGER.info("Kafka proxy updated"); // Then - assertDeploymentUpdated(namespace, kubeClient, originalChecksum); + assertDeploymentUpdated(namespace, originalChecksum); } @Test @@ -255,7 +256,7 @@ void shouldUpdateDeploymentWhenSecretChanges(String namespace) { kroxylicious.deployPortIdentifiesNodeWithFilters(kafkaClusterName, List.of("arbitrary-filter")); LOGGER.info("Kroxylicious deployed"); - String originalChecksum = getInitialChecksum(namespace, kubeClient); + String originalChecksum = getInitialChecksum(namespace); Secret existingSecret = kubeClient.getClient().resources(Secret.class).inNamespace(namespace) .withName("kilted-kiwi").get(); @@ -265,10 +266,11 @@ void shouldUpdateDeploymentWhenSecretChanges(String namespace) { LOGGER.info("secret: kilted-kiwi updated"); // Then - assertDeploymentUpdated(namespace, kubeClient, originalChecksum); + assertDeploymentUpdated(namespace, originalChecksum); } - private static void assertDeploymentUpdated(String namespace, KubeClient kubeClient, String originalChecksum) { + private static void assertDeploymentUpdated(String namespace, String originalChecksum) { + var kubeClient = kubeClient(namespace); AtomicReference newChecksumFromAnnotation = new AtomicReference<>(); await().atMost(Duration.ofSeconds(90)).untilAsserted(() -> { Deployment proxyDeployment = kubeClient.getDeployment(namespace, Constants.KROXYLICIOUS_PROXY_SIMPLE_NAME); @@ -303,7 +305,8 @@ void setupBefore() { kroxyliciousOperator.deploy(); } - private String getInitialChecksum(String namespace, KubeClient kubeClient) { + private String getInitialChecksum(String namespace) { + var kubeClient = kubeClient(namespace); AtomicReference checksumFromAnnotation = new AtomicReference<>(); await().atMost(Duration.ofSeconds(90)) .untilAsserted(() -> kubeClient.listPods(namespace, "app.kubernetes.io/name", "kroxylicious-proxy"), @@ -330,15 +333,16 @@ private String getChecksumFromAnnotation(HasMetadata entity) { return KubernetesResourceUtil.getOrCreateAnnotations(entity).get("kroxylicious.io/referent-checksum"); } - private static void updateIngresProtocol(ClusterIP.Protocol protocol, KubeClient kubeClient, String namespace) { - KafkaProxyIngress existingIngress = kubeClient.getClient() - .resources(KafkaProxyIngress.class) - .inNamespace(namespace) + private static void updateIngresProtocol(ClusterIP.Protocol protocol, String namespace) { + KafkaProxyIngress resolver = new KafkaProxyIngressBuilder() + .withNewMetadata() + .withNamespace(namespace) .withName(Constants.KROXYLICIOUS_INGRESS_CLUSTER_IP) - .get(); + .endMetadata() + .build(); ResourceManager.getInstance() - .replaceResourceWithRetries(existingIngress, + .replaceResourceWithRetries(resolver, ingress -> ingress.getSpec().getClusterIP().setProtocol(protocol)); } } From 1769b090327320d9ff9205b06b974994c7759b68 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Wed, 4 Jun 2025 16:37:36 +1200 Subject: [PATCH 65/83] Restore operator dockerbuild Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .github/workflows/operator-maven.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/operator-maven.yaml b/.github/workflows/operator-maven.yaml index f22b22520c..6ef1772ee6 100644 --- a/.github/workflows/operator-maven.yaml +++ b/.github/workflows/operator-maven.yaml @@ -85,6 +85,22 @@ jobs: echo "KROXYLICIOUS_IMAGE=${KROXYLICIOUS_IMAGE}" >> "$GITHUB_ENV" # make sure everything is built and compiles mvn -B install -Pci -Djapicmp.skip=${REFERENCE_RELEASE_UNPUBLISHED} -pl ':kroxylicious-operator' -am + - name: 'Set up Docker Buildx' + uses: docker/setup-buildx-action@v3 + - name: 'Build tagged Docker image' + uses: docker/build-push-action@v6 + with: + file: Dockerfile + context: . + build-args: + KROXYLICIOUS_VERSION=${{ env.KROXYLICIOUS_VERSION }} + tags: ${{ env.KROXYLICIOUS_IMAGE }} + push: false + cache-from: type=gha + cache-to: type=gha,mode=max,compression=zstd + load: 'true' + - name: 'Load Kroxylicious Image in Minikube' + run: minikube image load ${{ env.KROXYLICIOUS_IMAGE }} - name: 'Build Kroxylicious maven project on main with Sonar' if: github.event_name == 'push' && github.ref_name == 'main' && env.SONAR_TOKEN_SET == 'true' env: From aef28a68be72adc32f88b5deb00c51913d3f940a Mon Sep 17 00:00:00 2001 From: Keith Wall Date: Wed, 4 Jun 2025 11:15:46 +0100 Subject: [PATCH 66/83] Fix #2266: Ensure connection is close if SNI hostname fails to resolve to virtual cluster. Signed-off-by: Keith Wall Signed-off-by: Ujjwal --- kroxylicious-integration-test-support/pom.xml | 4 + .../kroxylicious/test/client/KafkaClient.java | 51 +++++++- .../kroxylicious/test/server/MockServer.java | 29 ++++- .../tester/DefaultKroxyliciousTester.java | 5 + .../test/tester/KroxyliciousClients.java | 16 ++- .../test/tester/KroxyliciousTester.java | 2 + .../test/client/KafkaClientTest.java | 111 ++++++++++++++++++ .../java/io/kroxylicious/proxy/TlsIT.java | 88 ++++++++++++-- .../proxy/internal/KafkaProxyInitializer.java | 13 +- 9 files changed, 294 insertions(+), 25 deletions(-) diff --git a/kroxylicious-integration-test-support/pom.xml b/kroxylicious-integration-test-support/pom.xml index adaaa60588..18b09e4e9f 100644 --- a/kroxylicious-integration-test-support/pom.xml +++ b/kroxylicious-integration-test-support/pom.xml @@ -90,6 +90,10 @@ io.netty netty-codec + + io.netty + netty-handler + io.netty netty-transport-classes-epoll diff --git a/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/client/KafkaClient.java b/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/client/KafkaClient.java index 4eaa7c3ff2..af9e742eec 100644 --- a/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/client/KafkaClient.java +++ b/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/client/KafkaClient.java @@ -6,11 +6,16 @@ package io.kroxylicious.test.client; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import javax.net.ssl.SSLException; +import javax.net.ssl.X509TrustManager; + import org.apache.kafka.common.message.RequestHeaderData; import io.netty.bootstrap.Bootstrap; @@ -22,6 +27,8 @@ import io.netty.channel.ChannelPipeline; import io.netty.channel.EventLoopGroup; import io.netty.channel.socket.SocketChannel; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; import io.kroxylicious.test.Request; import io.kroxylicious.test.Response; @@ -46,8 +53,10 @@ */ public final class KafkaClient implements AutoCloseable { + public static final SslContext TRUST_ALL_CLIENT_SSL_CONTEXT = buildTrustAllSslContext(); private final String host; private final int port; + private final SslContext sslContext; private final AtomicReference> connected = new AtomicReference<>(); @@ -56,15 +65,21 @@ public final class KafkaClient implements AutoCloseable { private final CorrelationManager correlationManager; private final KafkaClientHandler kafkaClientHandler; + public KafkaClient(String host, int port) { + this(host, port, null); + } + /** * create empty kafkaClient * * @param host host to connect to * @param port port to connect to + * @param clientSslContext client ssl context or null if TLS should not be used. */ - public KafkaClient(String host, int port) { + public KafkaClient(String host, int port, SslContext clientSslContext) { this.host = host; this.port = port; + this.sslContext = clientSslContext; this.eventGroupConfig = EventGroupConfig.create(); bossGroup = eventGroupConfig.newBossGroup(); correlationManager = new CorrelationManager(); @@ -130,6 +145,7 @@ public Response getSync(Request request) { private CompletableFuture ensureChannel(CorrelationManager correlationManager, KafkaClientHandler kafkaClientHandler) { var candidate = new CompletableFuture(); + if (connected.compareAndSet(null, candidate)) { Bootstrap b = new Bootstrap(); b.group(bossGroup) @@ -139,6 +155,9 @@ private CompletableFuture ensureChannel(CorrelationManager correlationM @Override public void initChannel(SocketChannel ch) { ChannelPipeline p = ch.pipeline(); + if (sslContext != null) { + p.addLast(sslContext.newHandler(ch.alloc(), host, port)); + } p.addLast(new KafkaRequestEncoder(correlationManager)); p.addLast(new KafkaResponseDecoder(correlationManager)); p.addLast(kafkaClientHandler); @@ -147,9 +166,7 @@ public void initChannel(SocketChannel ch) { ChannelFuture connect = b.connect(host, port); connect.addListeners((ChannelFutureListener) channelFuture -> candidate.complete(channelFuture.channel())); - connect.channel().closeFuture().addListener(future -> { - correlationManager.onChannelClose(); - }); + connect.channel().closeFuture().addListener(future -> correlationManager.onChannelClose()); return candidate; } else { @@ -189,4 +206,30 @@ private static Response toResponse(SequencedResponse sequencedResponse) { return new Response(new ResponsePayload(frame.apiKey(), frame.apiVersion(), frame.body()), sequencedResponse.sequenceNumber()); } + private static SslContext buildTrustAllSslContext() { + try { + return SslContextBuilder.forClient().trustManager(new TrustingTrustManager()).build(); + } + catch (SSLException e) { + throw new RuntimeException("Failed to build SslContext", e); + } + } + + private static class TrustingTrustManager implements X509TrustManager { + + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType) { + // we are trust all - nothing to do. + } + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { + // we are trust all - nothing to do. + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + } } diff --git a/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/server/MockServer.java b/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/server/MockServer.java index e3e9f436b2..21cdcb6abf 100644 --- a/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/server/MockServer.java +++ b/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/server/MockServer.java @@ -19,6 +19,7 @@ import io.netty.channel.ChannelPipeline; import io.netty.channel.EventLoopGroup; import io.netty.channel.socket.SocketChannel; +import io.netty.handler.ssl.SslContext; import io.kroxylicious.test.Request; import io.kroxylicious.test.ResponsePayload; @@ -40,8 +41,8 @@ public final class MockServer implements AutoCloseable { private final int port; private MockHandler serverHandler; - private MockServer(ResponsePayload response, int port) { - this.port = start(port, response); + private MockServer(ResponsePayload response, int port, SslContext serverSslContext) { + this.port = start(port, response, serverSslContext); } /** @@ -86,7 +87,7 @@ static Request toRequest(DecodedRequestFrame decodedRequestFrame) { * @return the created server */ public static MockServer startOnRandomPort() { - return new MockServer(null, 0); + return new MockServer(null, 0, null); } /** @@ -95,16 +96,31 @@ public static MockServer startOnRandomPort() { * @return the created server */ public static MockServer startOnRandomPort(ResponsePayload response) { - return new MockServer(response, 0); + return new MockServer(response, 0, null); + } + + /** + * Start mock server on a random port and serve this response + * @param response response to serve + * @param serverSslContext server ssl context, if null server will be plain. + * @return the created server + */ + public static MockServer startOnRandomPort(ResponsePayload response, SslContext serverSslContext) { + return new MockServer(response, 0, serverSslContext); } /** * Start the server + * * @param port port to bind to (0 to bind to an ephemeral) * @param response response to serve (nullable) + * @param serverSslContext server ssl context, if null server will be plain. * @return the port bound to */ - public int start(int port, ResponsePayload response) { + public int start(int port, ResponsePayload response, SslContext serverSslContext) { + if (serverSslContext != null && !serverSslContext.isServer()) { + throw new IllegalArgumentException("if using SSL, a SslContext configured for server required."); + } // Configure the server. final EventGroupConfig eventGroupConfig = EventGroupConfig.create(); bossGroup = eventGroupConfig.newBossGroup(); @@ -118,6 +134,9 @@ public int start(int port, ResponsePayload response) { @Override public void initChannel(SocketChannel ch) { ChannelPipeline p = ch.pipeline(); + if (serverSslContext != null) { + p.addLast(serverSslContext.newHandler(ch.alloc())); + } p.addLast(new KafkaRequestDecoder()); p.addLast(new KafkaResponseEncoder()); p.addLast(serverHandler); diff --git a/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/tester/DefaultKroxyliciousTester.java b/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/tester/DefaultKroxyliciousTester.java index cc0be6cc8b..ee437462d9 100644 --- a/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/tester/DefaultKroxyliciousTester.java +++ b/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/tester/DefaultKroxyliciousTester.java @@ -180,6 +180,11 @@ public KafkaClient simpleTestClient() { return clients().simpleTestClient(); } + @Override + public KafkaClient simpleTestClient(String address, boolean useTls) { + return clients().simpleTestClient(address, useTls); + } + @Override public KafkaClient simpleTestClient(String virtualCluster) { return clients(virtualCluster, DEFAULT_GATEWAY_NAME).simpleTestClient(); diff --git a/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/tester/KroxyliciousClients.java b/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/tester/KroxyliciousClients.java index a6e85ad2d2..c7b1cf1a01 100644 --- a/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/tester/KroxyliciousClients.java +++ b/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/tester/KroxyliciousClients.java @@ -21,6 +21,7 @@ import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serdes; @@ -94,8 +95,17 @@ public Consumer consumer(Serde keySerde, Serde valueSerde, Ma } public KafkaClient simpleTestClient() { - String[] hostPort = defaultClientConfiguration.get(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG).toString().split(":"); - return new KafkaClient(hostPort[0], Integer.parseInt(hostPort[1])); + var bootstrap = defaultClientConfiguration.get(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG).toString(); + var securityProtocol = SecurityProtocol + .forName(String.valueOf(defaultClientConfiguration.getOrDefault(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.PLAINTEXT))); + boolean useTls = securityProtocol != SecurityProtocol.PLAINTEXT && securityProtocol != SecurityProtocol.SASL_PLAINTEXT; + return simpleTestClient(bootstrap, useTls); + } + + public KafkaClient simpleTestClient(String address, boolean useTls) { + var hostPort = address.split(":", 2); + var clientSslContext = useTls ? KafkaClient.TRUST_ALL_CLIENT_SSL_CONTEXT : null; + return new KafkaClient(hostPort[0], Integer.parseInt(hostPort[1]), clientSslContext); } public void close() { @@ -146,4 +156,4 @@ default Producer newProducer(Map clientConfiguratio return CloseableProducer.wrap(new KafkaProducer<>(clientConfiguration, keySerializer, valueSerializer)); } } -} \ No newline at end of file +} diff --git a/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/tester/KroxyliciousTester.java b/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/tester/KroxyliciousTester.java index c781e02212..cf810e8f5a 100644 --- a/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/tester/KroxyliciousTester.java +++ b/kroxylicious-integration-test-support/src/main/java/io/kroxylicious/test/tester/KroxyliciousTester.java @@ -242,6 +242,8 @@ public interface KroxyliciousTester extends Closeable { */ KafkaClient simpleTestClient(); + KafkaClient simpleTestClient(String address, boolean useTls); + /** * Creates a Mock Request client configured with the kroxylicious bootstrap server * for a specific virtual cluster. This client can be used to send multiple diff --git a/kroxylicious-integration-test-support/src/test/java/io/kroxylicious/test/client/KafkaClientTest.java b/kroxylicious-integration-test-support/src/test/java/io/kroxylicious/test/client/KafkaClientTest.java index 2fbdde37fa..3d1f494b7c 100644 --- a/kroxylicious-integration-test-support/src/test/java/io/kroxylicious/test/client/KafkaClientTest.java +++ b/kroxylicious-integration-test-support/src/test/java/io/kroxylicious/test/client/KafkaClientTest.java @@ -6,29 +6,66 @@ package io.kroxylicious.test.client; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.security.GeneralSecurityException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLException; +import javax.net.ssl.TrustManagerFactory; + import org.apache.kafka.common.message.ApiVersionsRequestData; import org.apache.kafka.common.message.ApiVersionsResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.ApiVersionsRequest; import org.apache.kafka.common.requests.RequestHeader; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import io.netty.buffer.Unpooled; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; import io.kroxylicious.test.Request; import io.kroxylicious.test.Response; import io.kroxylicious.test.ResponsePayload; import io.kroxylicious.test.codec.OpaqueRequestFrame; import io.kroxylicious.test.server.MockServer; +import io.kroxylicious.testing.kafka.common.KeytoolCertificateGenerator; import static org.assertj.core.api.Assertions.assertThat; class KafkaClientTest { + + private KeytoolCertificateGenerator downstreamCertificateGenerator; + private Path clientTrustStore; + + @TempDir + private Path certsDirectory; + + @BeforeEach + void beforeEach() throws Exception { + // Note that the KeytoolCertificateGenerator generates key stores that are PKCS12 format. + this.downstreamCertificateGenerator = new KeytoolCertificateGenerator(); + this.downstreamCertificateGenerator.generateSelfSignedCertificateEntry("test@kroxylicious.io", "localhost", "KI", "kroxylicious.io", null, null, "US"); + this.clientTrustStore = certsDirectory.resolve("kafka.truststore.jks"); + this.downstreamCertificateGenerator.generateTrustStore(this.downstreamCertificateGenerator.getCertFilePath(), "client", + clientTrustStore.toAbsolutePath().toString()); + } + @Test void testClientCanSendOpaqueFrame() { ApiVersionsResponseData message = new ApiVersionsResponseData(); @@ -65,4 +102,78 @@ void testClientCanTolerateV0ApiVersionsResponseToHigherRequestVersion() { }); } } + + @Test + void shouldWorkWithTls() throws Exception { + shouldWorkWithTls(SslContextBuilder.forClient() + .trustManager( + trustManagerFactory(clientTrustStore.toFile(), downstreamCertificateGenerator.getKeyStoreType(), downstreamCertificateGenerator.getPassword())) + .build()); + } + + @Test + void shouldWorkWithTlsWhenTrustingAll() throws Exception { + shouldWorkWithTls(KafkaClient.TRUST_ALL_CLIENT_SSL_CONTEXT); + } + + private void shouldWorkWithTls(SslContext clientSslContext) throws SSLException { + var message = new ApiVersionsResponseData(); + + var file = new File(downstreamCertificateGenerator.getKeyStoreLocation()); + var serverSslContext = SslContextBuilder + .forServer(keyManagerFactory(file, downstreamCertificateGenerator.getKeyStoreType(), downstreamCertificateGenerator.getPassword())).build(); + + var serverResponse = new ResponsePayload(ApiKeys.API_VERSIONS, (short) 0, message); + + try (var mockServer = MockServer.startOnRandomPort(serverResponse, serverSslContext); + var kafkaClient = new KafkaClient("127.0.0.1", mockServer.port(), clientSslContext)) { + + CompletableFuture future = kafkaClient.get(new Request(ApiKeys.API_VERSIONS, (short) 0, "client", new ApiVersionsRequestData())); + assertThat(future) + .succeedsWithin(10, TimeUnit.SECONDS) + .satisfies(response -> assertThat(response.payload().message()).isInstanceOf(ApiVersionsResponseData.class)); + } + } + + private KeyManagerFactory keyManagerFactory(File storeFile, String keyStoreType, String password) { + try { + var passwordChars = Optional.ofNullable(password).map(String::toCharArray).orElse(null); + var keyStore = loadKeyStore(storeFile, keyStoreType, passwordChars); + var keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagerFactory.init(keyStore, passwordChars); + return keyManagerFactory; + } + catch (GeneralSecurityException e) { + throw new RuntimeException("Error building KeyManagerFactory from : " + storeFile, e); + } + } + + private TrustManagerFactory trustManagerFactory(File storeFile, String keyStoreType, String password) { + + try { + var passwordChars = Optional.ofNullable(password).map(String::toCharArray).orElse(null); + var keyStore = loadKeyStore(storeFile, keyStoreType, passwordChars); + var trustManagerFactory = TrustManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(keyStore); + return trustManagerFactory; + } + catch (GeneralSecurityException e) { + throw new RuntimeException("Error building TrustManagerFactory from : " + storeFile, e); + } + + } + + private KeyStore loadKeyStore(File storeFile, String keyStoreType, char[] passwordChars) { + try (var is = new FileInputStream(storeFile)) { + var keyStore = KeyStore.getInstance(keyStoreType); + keyStore.load(is, passwordChars); + return keyStore; + } + catch (CertificateException | KeyStoreException | NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } } diff --git a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/TlsIT.java b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/TlsIT.java index c7dc4411d6..31866d1246 100644 --- a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/TlsIT.java +++ b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/TlsIT.java @@ -34,7 +34,10 @@ import org.apache.kafka.clients.admin.CreateTopicsResult; import org.apache.kafka.clients.admin.DescribeClusterOptions; import org.apache.kafka.common.config.SslConfigs; +import org.apache.kafka.common.errors.SslAuthenticationException; import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.message.ApiVersionsRequestData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.assertj.core.api.InstanceOfAssertFactories; import org.junit.jupiter.api.BeforeEach; @@ -44,6 +47,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import io.kroxylicious.net.IntegrationTestInetAddressResolverProvider; import io.kroxylicious.proxy.config.ConfigurationBuilder; import io.kroxylicious.proxy.config.VirtualClusterBuilder; import io.kroxylicious.proxy.config.secret.FilePassword; @@ -52,6 +56,7 @@ import io.kroxylicious.proxy.config.tls.AllowDeny; import io.kroxylicious.proxy.config.tls.TlsClientAuth; import io.kroxylicious.proxy.service.HostPort; +import io.kroxylicious.test.Request; import io.kroxylicious.testing.kafka.api.KafkaCluster; import io.kroxylicious.testing.kafka.clients.CloseableAdmin; import io.kroxylicious.testing.kafka.common.KafkaClusterConfig; @@ -64,6 +69,7 @@ import static io.kroxylicious.test.tester.KroxyliciousConfigUtils.baseVirtualClusterBuilder; import static io.kroxylicious.test.tester.KroxyliciousConfigUtils.defaultPortIdentifiesNodeGatewayBuilder; +import static io.kroxylicious.test.tester.KroxyliciousConfigUtils.defaultSniHostIdentifiesNodeGatewayBuilder; import static io.kroxylicious.test.tester.KroxyliciousTesters.kroxyliciousTester; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -77,6 +83,11 @@ class TlsIT extends BaseIT { private static final HostPort PROXY_ADDRESS = HostPort.parse("localhost:9192"); private static final String TOPIC = "my-test-topic"; + + private static final String SNI_BASE_ADDRESS = IntegrationTestInetAddressResolverProvider.generateFullyQualifiedDomainName("sni"); + private static final String SNI_BROKER_ADDRESS_PATTERN = "broker-$(nodeId)." + SNI_BASE_ADDRESS; + private static final HostPort SNI_BOOTSTRAP_ADDRESS = HostPort.parse("bootstrap." + SNI_BASE_ADDRESS + ":9192"); + @TempDir private Path certsDirectory; private KeytoolCertificateGenerator downstreamCertificateGenerator; @@ -85,17 +96,17 @@ class TlsIT extends BaseIT { private Path proxyTrustStore; @BeforeEach - public void beforeEach() throws Exception { + void beforeEach() throws Exception { // Note that the KeytoolCertificateGenerator generates key stores that are PKCS12 format. this.downstreamCertificateGenerator = new KeytoolCertificateGenerator(); - this.downstreamCertificateGenerator.generateSelfSignedCertificateEntry("test@redhat.com", "localhost", "KI", "RedHat", null, null, "US"); + this.downstreamCertificateGenerator.generateSelfSignedCertificateEntry("test@kroxylicious.io", "localhost", "KI", "kroxylicious.io", null, null, "US"); this.clientTrustStore = certsDirectory.resolve("kafka.truststore.jks"); this.downstreamCertificateGenerator.generateTrustStore(this.downstreamCertificateGenerator.getCertFilePath(), "client", clientTrustStore.toAbsolutePath().toString()); // Generator for certificate that will identify the client this.clientCertGenerator = new KeytoolCertificateGenerator(); - this.clientCertGenerator.generateSelfSignedCertificateEntry("clientTest@kroxylicious.io", "client", "Dev", "Kroxylicious.io", null, null, "US"); + this.clientCertGenerator.generateSelfSignedCertificateEntry("clientTest@kroxylicious.io", "client", "Dev", "kroxylicious.io", null, null, "US"); this.proxyTrustStore = certsDirectory.resolve("proxy.truststore.jks"); this.clientCertGenerator.generateTrustStore(clientCertGenerator.getCertFilePath(), "proxy", proxyTrustStore.toAbsolutePath().toString()); @@ -411,6 +422,62 @@ void downstream_SuccessfulTlsWithProtocolsAllowed(KafkaCluster cluster) { } } + @Test + void downstream_UnrecognizedSniHostNameClosesConnection(KafkaCluster cluster) { + var duffBootstrap = "bootstrap." + IntegrationTestInetAddressResolverProvider.generateFullyQualifiedDomainName("duff") + ":" + SNI_BOOTSTRAP_ADDRESS.port(); + + var builder = new ConfigurationBuilder() + .addToVirtualClusters(baseVirtualClusterBuilder(cluster, "demo") + .addToGateways(defaultSniHostIdentifiesNodeGatewayBuilder(SNI_BOOTSTRAP_ADDRESS, SNI_BROKER_ADDRESS_PATTERN) + .withNewTls() + .withNewKeyStoreKey() + .withStoreFile(downstreamCertificateGenerator.getKeyStoreLocation()) + .withNewInlinePasswordStoreProvider(downstreamCertificateGenerator.getPassword()) + .endKeyStoreKey() + .endTls() + .build()) + .build()); + + try (var tester = kroxyliciousTester(builder); + var testClient = tester.simpleTestClient(duffBootstrap, true)) { + + // The request itself is unimportant, it won't actually reach the server + var request = new Request(ApiKeys.API_VERSIONS, ApiKeys.API_VERSIONS.latestVersion(), null, new ApiVersionsRequestData()); + var response = testClient.get(request); + assertThat(response) + .failsWithin(Duration.ofSeconds(5)) + .withThrowableThat() + .withMessageContaining("channel closed before response received!"); + } + } + + @Test + void downstream_UntrustedCertificateClosesConnection(KafkaCluster cluster) { + + var builder = new ConfigurationBuilder() + .addToVirtualClusters(baseVirtualClusterBuilder(cluster, "demo") + .addToGateways(defaultPortIdentifiesNodeGatewayBuilder(PROXY_ADDRESS) + .withNewTls() + .withNewKeyStoreKey() + .withStoreFile(downstreamCertificateGenerator.getKeyStoreLocation()) + .withNewInlinePasswordStoreProvider(downstreamCertificateGenerator.getPassword()) + .endKeyStoreKey() + .endTls() + .build()) + .build()); + + try (var tester = kroxyliciousTester(builder); + // admin won't trust the self signed cert of the broker. + var admin = tester.admin(Map.of(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SSL.name))) { + + assertThat(admin.describeCluster().clusterId()) + .failsWithin(Duration.ofSeconds(10)) + .withThrowableThat() + .withCauseInstanceOf(SslAuthenticationException.class) + .withMessageContaining("SSL handshake failed"); + } + } + @Test void downstream_UnsuccessfulTlsWithProtocolsAllowed(KafkaCluster cluster) { // Protocol we want to use @@ -744,7 +811,7 @@ void upstream_UnsuccessfulTlsWithCipherSuitesAllowed(@Tls KafkaCluster cluster) } @Test - void downstreamMutualTls_SuccessfulTlsClientAuthRequiredByDefault(KafkaCluster cluster) throws Exception { + void downstreamMutualTls_SuccessfulTlsClientAuthRequiredByDefault(KafkaCluster cluster) { try (var tester = kroxyliciousTester(constructMutualTlsBuilder(cluster, null)); var admin = tester.admin("demo", Map.of(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, @@ -760,7 +827,7 @@ void downstreamMutualTls_SuccessfulTlsClientAuthRequiredByDefault(KafkaCluster c } @Test - void downstreamMutualTls_UnsuccessfulTlsClientAuthRequiredByDefault(KafkaCluster cluster) throws Exception { + void downstreamMutualTls_UnsuccessfulTlsClientAuthRequiredByDefault(KafkaCluster cluster) { try (var tester = kroxyliciousTester(constructMutualTlsBuilder(cluster, null)); var admin = tester.admin("demo", Map.of(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, @@ -774,7 +841,7 @@ void downstreamMutualTls_UnsuccessfulTlsClientAuthRequiredByDefault(KafkaCluster } @Test - void downstreamMutualTls_SuccessfulTlsClientAuthRequired(KafkaCluster cluster) throws Exception { + void downstreamMutualTls_SuccessfulTlsClientAuthRequired(KafkaCluster cluster) { try (var tester = kroxyliciousTester(constructMutualTlsBuilder(cluster, TlsClientAuth.REQUIRED)); var admin = tester.admin("demo", Map.of(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, @@ -790,7 +857,7 @@ void downstreamMutualTls_SuccessfulTlsClientAuthRequired(KafkaCluster cluster) t } @Test - void downstreamMutualTls_UnsuccessfulTlsClientAuthRequired(KafkaCluster cluster) throws Exception { + void downstreamMutualTls_UnsuccessfulTlsClientAuthRequired(KafkaCluster cluster) { try (var tester = kroxyliciousTester(constructMutualTlsBuilder(cluster, TlsClientAuth.REQUIRED)); var admin = tester.admin("demo", Map.of(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, @@ -804,7 +871,7 @@ void downstreamMutualTls_UnsuccessfulTlsClientAuthRequired(KafkaCluster cluster) } @Test - void downstreamMutualTls_SuccessfulTlsClientAuthRequestedAndProvided(KafkaCluster cluster) throws Exception { + void downstreamMutualTls_SuccessfulTlsClientAuthRequestedAndProvided(KafkaCluster cluster) { try (var tester = kroxyliciousTester(constructMutualTlsBuilder(cluster, TlsClientAuth.REQUESTED)); var admin = tester.admin("demo", Map.of(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, @@ -834,7 +901,7 @@ void downstreamMutualTls_SuccessfulTlsClientAuthRequestedAndNotProvided(KafkaClu } @Test - void downstreamMutualTls_SuccessfulTlsClientAuthNoneAndProvided(KafkaCluster cluster) throws Exception { + void downstreamMutualTls_SuccessfulTlsClientAuthNoneAndProvided(KafkaCluster cluster) { try (var tester = kroxyliciousTester(constructMutualTlsBuilder(cluster, TlsClientAuth.NONE)); var admin = tester.admin("demo", Map.of(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, @@ -850,7 +917,7 @@ void downstreamMutualTls_SuccessfulTlsClientAuthNoneAndProvided(KafkaCluster clu } @Test - void downstreamMutualTls_SuccessfulTlsClientAuthNoneAndNotProvided(KafkaCluster cluster) throws Exception { + void downstreamMutualTls_SuccessfulTlsClientAuthNoneAndNotProvided(KafkaCluster cluster) { try (var tester = kroxyliciousTester(constructMutualTlsBuilder(cluster, TlsClientAuth.NONE)); var admin = tester.admin("demo", Map.of(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, @@ -904,7 +971,6 @@ private String writePasswordToFile(String password) { File tmp = File.createTempFile("password", ".txt"); tmp.deleteOnExit(); makeFileOwnerReadWriteOnly(tmp); - boolean ignore; Files.writeString(tmp.toPath(), password); // remove write from owner assertThat(tmp.setWritable(false, true)).isTrue(); diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java index 56e2988a50..04b6b790d7 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java @@ -171,8 +171,17 @@ private void initTlsChannel(SocketChannel ch, ChannelPipeline pipeline, Optional @Override protected void onLookupComplete(ChannelHandlerContext ctx, Future future) throws Exception { - super.onLookupComplete(ctx, future); - ctx.fireChannelActive(); + if (future.isSuccess()) { + super.onLookupComplete(ctx, future); + ctx.fireChannelActive(); + } + else { + // We've failed to look up the SslContext - this indicates that the SNI hostname was unrecognized + // or that the virtual cluster is somehow not configured for TLS. All we can do is close the + // connection. + ctx.close(); + } + } }); } From 4b07f93aa050b516043452b2126731b617dbf959 Mon Sep 17 00:00:00 2001 From: ShubhamRwt Date: Wed, 4 Jun 2025 14:56:08 +0530 Subject: [PATCH 67/83] Added connection metrics Signed-off-by: ShubhamRwt Signed-off-by: Ujjwal --- .../java/io/kroxylicious/proxy/MetricsIT.java | 275 +++++++++++++++++- .../internal/KafkaProxyFrontendHandler.java | 2 +- .../proxy/internal/KafkaProxyInitializer.java | 15 + .../internal/ProxyChannelStateMachine.java | 53 +++- .../proxy/internal/util/Metrics.java | 41 ++- .../KafkaProxyFrontendHandlerTest.java | 2 +- .../ProxyChannelStateMachineEndToEndTest.java | 2 +- .../ProxyChannelStateMachineTest.java | 2 +- 8 files changed, 382 insertions(+), 10 deletions(-) diff --git a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java index abceacc98f..7694a4bd5c 100644 --- a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java +++ b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java @@ -5,7 +5,12 @@ */ package io.kroxylicious.proxy; +import java.io.IOException; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.time.Duration; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -14,19 +19,25 @@ import java.util.concurrent.ExecutionException; import java.util.function.Consumer; import java.util.function.Predicate; +import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.stream.Stream; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.admin.DescribeClusterOptions; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartitionInfo; +import org.apache.kafka.common.config.SslConfigs; +import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.protocol.ApiKeys; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -35,15 +46,20 @@ import io.micrometer.core.instrument.Metrics; import io.netty.handler.codec.http.HttpResponseStatus; +import io.kroxylicious.net.IntegrationTestInetAddressResolverProvider; import io.kroxylicious.proxy.config.ConfigurationBuilder; import io.kroxylicious.proxy.config.MicrometerDefinitionBuilder; import io.kroxylicious.proxy.config.NamedFilterDefinition; import io.kroxylicious.proxy.config.NamedFilterDefinitionBuilder; +import io.kroxylicious.proxy.config.VirtualClusterGateway; import io.kroxylicious.proxy.micrometer.CommonTagsHook; import io.kroxylicious.proxy.micrometer.StandardBindersHook; +import io.kroxylicious.proxy.service.HostPort; +import io.kroxylicious.test.tester.KroxyliciousTester; import io.kroxylicious.test.tester.SimpleMetric; import io.kroxylicious.testing.kafka.api.KafkaCluster; import io.kroxylicious.testing.kafka.common.BrokerCluster; +import io.kroxylicious.testing.kafka.common.KeytoolCertificateGenerator; import io.kroxylicious.testing.kafka.junit5ext.KafkaClusterExtension; import io.kroxylicious.testing.kafka.junit5ext.Topic; import io.kroxylicious.testing.kafka.junit5ext.TopicPartitions; @@ -53,19 +69,32 @@ import static io.kroxylicious.proxy.internal.util.Metrics.API_KEY_LABEL; import static io.kroxylicious.proxy.internal.util.Metrics.DECODED_LABEL; import static io.kroxylicious.proxy.internal.util.Metrics.NODE_ID_LABEL; +import static io.kroxylicious.test.tester.KroxyliciousConfigUtils.defaultPortIdentifiesNodeGatewayBuilder; +import static io.kroxylicious.test.tester.KroxyliciousConfigUtils.defaultSniHostIdentifiesNodeGatewayBuilder; import static io.kroxylicious.test.tester.KroxyliciousConfigUtils.proxy; import static io.kroxylicious.test.tester.KroxyliciousTesters.kroxyliciousTester; import static io.kroxylicious.test.tester.SimpleMetricAssert.assertThat; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import static org.junit.jupiter.params.provider.Arguments.argumentSet; @ExtendWith(KafkaClusterExtension.class) @ExtendWith(NettyLeakDetectorExtension.class) class MetricsIT { + @TempDir + private static Path certsDirectory; + + private static final HostPort PORT_IDENTIFIES_BROKER_BOOTSTRAP = new HostPort("localhost", 9092); + private static final String SNI_IDENTIFIES_BROKER_BASE_ADDRESS = IntegrationTestInetAddressResolverProvider.generateFullyQualifiedDomainName("sni"); + + private static final HostPort SNI_IDENTIFIES_BROKER_BOOTSTRAP = new HostPort("bootstrap." + SNI_IDENTIFIES_BROKER_BASE_ADDRESS, 9192); + private static final String SNI_IDENTIFIES_BROKER_ADDRESS_PATTERN = "broker-$(nodeId)." + SNI_IDENTIFIES_BROKER_BASE_ADDRESS; + @BeforeEach void beforeEach() { assertThat(Metrics.globalRegistry.getMeters()).isEmpty(); + } @AfterEach @@ -324,6 +353,98 @@ void shouldIncrementCountOnMessageTransit(UnaryOperator bu } } + static Stream connectionMetricsScenarios() { + + return Stream.of( + argumentSet("counts connection from client to proxy for bootstrap", + (UnaryOperator) builder -> builder, + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_client_to_proxy_connections_total") + .filterByTag(NODE_ID_LABEL, "bootstrap") + .isEmpty(), + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_client_to_proxy_connections_total") + .filterByTag(NODE_ID_LABEL, "bootstrap") + .singleElement() + .value() + .isGreaterThan(0.0)), + argumentSet("counts connections to node, client to proxy", + (UnaryOperator) builder -> builder, + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_client_to_proxy_connections_total") + .filterByTag(NODE_ID_LABEL, "0") + .isEmpty(), + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_client_to_proxy_connections_total") + .filterByTag(NODE_ID_LABEL, "0") + .singleElement() + .value() + .isGreaterThan(0.0)), + argumentSet("count bootstrap connection from proxy to server ", + (UnaryOperator) builder -> builder, + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_proxy_to_server_connections_total") + .filterByTag(NODE_ID_LABEL, "bootstrap") + .isEmpty(), + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_proxy_to_server_connections_total") + .filterByTag(NODE_ID_LABEL, "bootstrap") + .singleElement() + .value() + .isGreaterThan(0.0)), + argumentSet("count node connection from proxy to server ", + (UnaryOperator) builder -> builder, + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_proxy_to_server_connections_total") + .filterByTag(NODE_ID_LABEL, "0") + .isEmpty(), + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_proxy_to_server_connections_total") + .filterByTag(NODE_ID_LABEL, "0") + .singleElement() + .value() + .isGreaterThan(0.0)), + argumentSet("count error from proxy to server (bootstrap)", + (UnaryOperator) builder -> builder, + (Consumer>) metricList -> assertThat(metricList) + .filterByName("kroxylicious_proxy_to_server_errors_total") + .filterByTag(NODE_ID_LABEL) + .isEmpty(), + (Consumer>) metricList -> assertThat(metricList).anySatisfy(simpleMetric -> { + assertThat(simpleMetric.name()).isEqualTo("kroxylicious_proxy_to_server_errors_total"); + assertThat(simpleMetric.value()).isGreaterThan(0); + }))); + } + + @ParameterizedTest + @MethodSource("connectionMetricsScenarios") + void shouldIncrementCountOnConnectionMetrics(UnaryOperator builder, + Consumer> beforeAssertion, + Consumer> afterAssertion, + KafkaCluster cluster) + throws Exception { + var config = configWithMetrics(cluster); + + // Given + try (var tester = kroxyliciousTester(builder.apply(config)); + var managementClient = tester.getManagementClient();) { + + var metricList = managementClient.scrapeMetrics(); + + beforeAssertion.accept(metricList); + + // When + var producer = tester.producer(); + var future = producer.send(new ProducerRecord<>("my-topic", "key", "value")); + assertThat(future).succeedsWithin(Duration.ofSeconds(5)); + cluster.close(); + + // Then + metricList = managementClient.scrapeMetrics(); + afterAssertion.accept(metricList); + } + } + @Test void countMetricsShouldBeDiscriminatedByNodeId(@BrokerCluster(numBrokers = 2) KafkaCluster cluster, @TopicPartitions(2) Topic topic) throws ExecutionException, InterruptedException { @@ -432,7 +553,8 @@ void shouldIncrementDownstreamMessagesOnProduceRequestWithFilter(KafkaCluster cl } @Test - void shouldIncrementConnectionMetrics(KafkaCluster cluster, Topic topic) throws ExecutionException, InterruptedException { + @Deprecated(since = "0.13.0", forRemoval = true) + void shouldIncrementConnectionMetrics(KafkaCluster cluster, Topic topic) throws Exception { var config = configWithMetrics(cluster); // Given @@ -454,6 +576,157 @@ void shouldIncrementConnectionMetrics(KafkaCluster cluster, Topic topic) throws } } + @Test + @Deprecated(since = "0.13.0", forRemoval = true) + void shouldIncrementUpstreamConnectionErrorMetrics(KafkaCluster cluster) throws Exception { + var config = proxy(cluster) + .addToFilterDefinitions() + .addToDefaultFilters() + .withNewManagement() + .withNewEndpoints() + .withNewPrometheus() + .endPrometheus() + .endEndpoints() + .endManagement(); + + // Given + try (var tester = kroxyliciousTester(config); + var managementClient = tester.getManagementClient()) { + var metricsList = managementClient.scrapeMetrics(); + assertMetricsDoesNotExist(metricsList, "kroxylicious_upstream_errors_total", null); + + // When + var producer = tester.producer(); + var future = producer.send(new ProducerRecord<>("my-topic", "key", "value")); + assertThat(future).succeedsWithin(Duration.ofSeconds(5)); + cluster.close(); + + // Then + // updated metrics after some message were produced + var updatedMetricsList = managementClient.scrapeMetrics(); + assertMetricsWithValue(updatedMetricsList, "kroxylicious_upstream_errors_total", null); + } + } + + static Stream createDownstreamConnectionError() throws Exception { + var failQuickDescribeCluster = new DescribeClusterOptions().timeoutMs(2_000); + // Note that the KeytoolCertificateGenerator generates key stores that are PKCS12 format. + var downstreamCertificateGenerator = new KeytoolCertificateGenerator(); + downstreamCertificateGenerator.generateSelfSignedCertificateEntry("test@gmail.com", "localhost", "foo", "bar", null, null, "US"); + var clientTrustStore = certsDirectory.resolve("kafka.truststore.jks"); + downstreamCertificateGenerator.generateTrustStore(downstreamCertificateGenerator.getCertFilePath(), "client", + clientTrustStore.toAbsolutePath().toString()); + + var list = new ArrayList(); + + { + list.add(argumentSet("gateway is tcp, client sends malformed kafka", + (Supplier) () -> defaultPortIdentifiesNodeGatewayBuilder(PORT_IDENTIFIES_BROKER_BOOTSTRAP) + .withName("default") + .build(), + (Consumer) kroxyliciousTester -> { + var hostPort = HostPort.parse(kroxyliciousTester.getBootstrapAddress()); + try (var sock = new Socket(hostPort.host(), hostPort.port()); + var output = sock.getOutputStream()) { + output.write("This ain't Kafka!".getBytes(StandardCharsets.UTF_8)); + } + catch (IOException e) { + throw new RuntimeException(e); + } + })); + } + + { + + list.add(argumentSet("gateway is tls, client connects using plain", + (Supplier) () -> defaultSniHostIdentifiesNodeGatewayBuilder(SNI_IDENTIFIES_BROKER_BOOTSTRAP, + SNI_IDENTIFIES_BROKER_ADDRESS_PATTERN) + .withName("default") + .withNewTls() + .withNewKeyStoreKey() + .withStoreFile(downstreamCertificateGenerator.getKeyStoreLocation()) + .withNewInlinePasswordStoreProvider(downstreamCertificateGenerator.getPassword()) + .endKeyStoreKey() + .endTls() + .build(), + (Consumer) kroxyliciousTester -> { + try (var admin = kroxyliciousTester.admin(Map.of(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "PLAINTEXT"))) { + assertThat(admin.describeCluster(failQuickDescribeCluster).clusterId()) + .failsWithin(Duration.ofSeconds(5)) + .withThrowableThat() + .withCauseInstanceOf(TimeoutException.class) + .havingCause() + .withMessageStartingWith("Timed out waiting for a node assignment."); + } + })); + } + + { + var unrecognisedSni = IntegrationTestInetAddressResolverProvider.generateFullyQualifiedDomainName("unrecognised"); + + list.add(argumentSet("gateway is tls/sni, unrecognised SNI name", + (Supplier) () -> defaultSniHostIdentifiesNodeGatewayBuilder(SNI_IDENTIFIES_BROKER_BOOTSTRAP, + SNI_IDENTIFIES_BROKER_ADDRESS_PATTERN) + .withName("default") + .withNewTls() + .withNewKeyStoreKey() + .withStoreFile(downstreamCertificateGenerator.getKeyStoreLocation()) + .withNewInlinePasswordStoreProvider(downstreamCertificateGenerator.getPassword()) + .endKeyStoreKey() + .endTls() + .build(), + (Consumer) kroxyliciousTester -> { + var hostPort = HostPort.parse(kroxyliciousTester.getBootstrapAddress()); + var unrecognisedHostPort = unrecognisedSni + ":" + hostPort.port(); + + try (var admin = kroxyliciousTester.admin( + Map.of(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, unrecognisedHostPort, + SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, clientTrustStore.toAbsolutePath().toString(), + SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, downstreamCertificateGenerator.getPassword()))) { + assertThat(admin.describeCluster(failQuickDescribeCluster).clusterId()) + .failsWithin(Duration.ofSeconds(5)) + .withThrowableThat() + .withCauseInstanceOf(TimeoutException.class) + .havingCause() + .withMessageStartingWith("Timed out waiting for a node assignment."); + } + })); + } + return list.stream(); + + } + + @ParameterizedTest + @MethodSource(value = "createDownstreamConnectionError") + void shouldIncrementDownstreamErrorOnConnectionError(Supplier gatewaySupplier, Consumer causeConnectionError, + KafkaCluster cluster) { + var config = proxy(cluster) + .withNewManagement() + .withNewEndpoints() + .withNewPrometheus() + .endPrometheus() + .endEndpoints() + .endManagement() + .editFirstVirtualCluster() + .withGateways(List.of(gatewaySupplier.get())) + .endVirtualCluster(); + + // Given + try (var tester = kroxyliciousTester(config); + var managementClient = tester.getManagementClient()) { + + // When + causeConnectionError.accept(tester); + + // Then + await().atMost(Duration.ofSeconds(30)) + .untilAsserted(() -> { + var updatedMetricsList = managementClient.scrapeMetrics(); + assertMetricsWithValue(updatedMetricsList, "kroxylicious_client_to_proxy_errors_total", null); + }); + } + } + @Test void shouldIncrementPayloadSizeBytesMetricsOnProduceRequest(KafkaCluster cluster, Topic topic) throws ExecutionException, InterruptedException { var config = configWithMetrics(cluster); diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java index 6ea2ac75f2..fb7ce68d74 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandler.java @@ -129,7 +129,7 @@ SslHandler getSslHandler(ChannelHandlerContext ctx) { @NonNull SaslDecodePredicate dp, @NonNull EndpointBinding endpointBinding, @NonNull String clusterName) { - this(netFilter, dp, endpointBinding, new ProxyChannelStateMachine(clusterName)); + this(netFilter, dp, endpointBinding, new ProxyChannelStateMachine(clusterName, endpointBinding.nodeId())); } @VisibleForTesting diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java index 04b6b790d7..d565ed689e 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java @@ -14,6 +14,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Tag; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.channel.ChannelInitializer; @@ -49,6 +51,10 @@ import io.kroxylicious.proxy.model.VirtualClusterModel; import io.kroxylicious.proxy.tag.VisibleForTesting; +import static io.kroxylicious.proxy.internal.util.Metrics.KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_TOTAL_METER_PROVIDER; +import static io.kroxylicious.proxy.internal.util.Metrics.NODE_ID_LABEL; +import static io.kroxylicious.proxy.internal.util.Metrics.VIRTUAL_CLUSTER_LABEL; + public class KafkaProxyInitializer extends ChannelInitializer { private static final Logger LOGGER = LoggerFactory.getLogger(KafkaProxyInitializer.class); @@ -65,6 +71,7 @@ public class KafkaProxyInitializer extends ChannelInitializer { private final PluginFactoryRegistry pfr; private final FilterChainFactory filterChainFactory; private final ApiVersionsServiceImpl apiVersionsService; + private final Counter clientToProxyErrorCounter; public KafkaProxyInitializer(FilterChainFactory filterChainFactory, PluginFactoryRegistry pfr, @@ -82,6 +89,8 @@ public KafkaProxyInitializer(FilterChainFactory filterChainFactory, this.bindingResolver = bindingResolver; this.filterChainFactory = filterChainFactory; this.apiVersionsService = apiVersionsService; + List newTags = Metrics.tags(VIRTUAL_CLUSTER_LABEL, "", NODE_ID_LABEL, "bootstrap"); + clientToProxyErrorCounter = KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_TOTAL_METER_PROVIDER.withTags(newTags); } @Override @@ -144,13 +153,17 @@ private void initTlsChannel(SocketChannel ch, ChannelPipeline pipeline, Optional try { if (t != null) { LOGGER.warn("Exception resolving Virtual Cluster Binding for endpoint {} and sniHostname {}: {}", endpoint, sniHostname, t.getMessage()); + clientToProxyErrorCounter.increment(); promise.setFailure(t); + ch.close(); // KW TODO - check right way to handle terminal errors within the SNIHandler. return null; } var gateway = binding.endpointGateway(); var sslContext = gateway.getDownstreamSslContext(); if (sslContext.isEmpty()) { + clientToProxyErrorCounter.increment(); promise.setFailure(new IllegalStateException("Virtual cluster %s does not provide SSL context".formatted(gateway))); + // close here too? } else { KafkaProxyInitializer.this.addHandlers(ch, binding); @@ -158,7 +171,9 @@ private void initTlsChannel(SocketChannel ch, ChannelPipeline pipeline, Optional } } catch (Throwable t1) { + clientToProxyErrorCounter.increment(); promise.setFailure(t1); + // and close here too? } return null; }); diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachine.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachine.java index 8365c819e1..4136b1ec96 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachine.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachine.java @@ -38,12 +38,17 @@ import edu.umd.cs.findbugs.annotations.Nullable; import static io.kroxylicious.proxy.internal.ProxyChannelState.Startup.STARTING_STATE; +import static io.kroxylicious.proxy.internal.util.Metrics.KROXYLICIOUS_CLIENT_TO_PROXY_CONNECTION_TOTAL_METER_PROVIDER; +import static io.kroxylicious.proxy.internal.util.Metrics.KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_TOTAL_METER_PROVIDER; import static io.kroxylicious.proxy.internal.util.Metrics.KROXYLICIOUS_DOWNSTREAM_CONNECTIONS; import static io.kroxylicious.proxy.internal.util.Metrics.KROXYLICIOUS_DOWNSTREAM_ERRORS; +import static io.kroxylicious.proxy.internal.util.Metrics.KROXYLICIOUS_PROXY_TO_SERVER_CONNECTION_TOTAL_METER_PROVIDER; +import static io.kroxylicious.proxy.internal.util.Metrics.KROXYLICIOUS_PROXY_TO_SERVER_ERROR_TOTAL_METER_PROVIDER; import static io.kroxylicious.proxy.internal.util.Metrics.KROXYLICIOUS_UPSTREAM_CONNECTIONS; import static io.kroxylicious.proxy.internal.util.Metrics.KROXYLICIOUS_UPSTREAM_CONNECTION_ATTEMPTS; import static io.kroxylicious.proxy.internal.util.Metrics.KROXYLICIOUS_UPSTREAM_CONNECTION_FAILURES; import static io.kroxylicious.proxy.internal.util.Metrics.KROXYLICIOUS_UPSTREAM_ERRORS; +import static io.kroxylicious.proxy.internal.util.Metrics.NODE_ID_LABEL; import static io.kroxylicious.proxy.internal.util.Metrics.taggedCounter; import static org.slf4j.LoggerFactory.getLogger; @@ -105,17 +110,53 @@ public class ProxyChannelStateMachine { private static final String DUPLICATE_INITIATE_CONNECT_ERROR = "NetFilter called NetFilterContext.initiateConnect() more than once"; private static final Logger LOGGER = getLogger(ProxyChannelStateMachine.class); + /** + * @deprecated replaced by `clientToProxyConnectionCounter` + */ + @Deprecated(since = "0.13.0", forRemoval = true) private final Counter downstreamConnectionsCounter; + /** + * @deprecated replaced by `proxyToServerConnectionCounter` + */ + @Deprecated(since = "0.13.0", forRemoval = true) private final Counter upstreamConnectionsCounter; + /** + * @deprecated replaced by `clientToProxyErrorCounter` + */ + @Deprecated(since = "0.13.0", forRemoval = true) private final Counter downstreamErrorCounter; + /** + * @deprecated replaced by `proxyToServerErrorCounter` + */ + @Deprecated(since = "0.13.0", forRemoval = true) private final Counter upstreamErrorCounter; + /** + * @deprecated replaced by `proxyToServerConnectionCounter` + */ + @Deprecated(since = "0.13.0", forRemoval = true) private final Counter connectionAttemptsCounter; + /** + * @deprecated replaced by `proxyToServerConnectionCounter` + */ + @Deprecated(since = "0.13.0", forRemoval = true) private final Counter upstreamConnectionFailureCounter; + private final Counter clientToProxyErrorCounter; + private final Counter clientToProxyConnectionCounter; + private final Counter proxyToServerConnectionCounter; + private final Counter proxyToServerErrorCounter; - public ProxyChannelStateMachine(String clusterName) { - List tags = Metrics.tags(Metrics.VIRTUAL_CLUSTER_TAG, clusterName); + public ProxyChannelStateMachine(String clusterName, @Nullable Integer nodeId) { + List tags = Metrics.tags(Metrics.VIRTUAL_CLUSTER_LABEL, clusterName); downstreamConnectionsCounter = taggedCounter(KROXYLICIOUS_DOWNSTREAM_CONNECTIONS, tags); + clientToProxyConnectionCounter = KROXYLICIOUS_CLIENT_TO_PROXY_CONNECTION_TOTAL_METER_PROVIDER + .withTags(Metrics.VIRTUAL_CLUSTER_LABEL, clusterName, NODE_ID_LABEL, nodeId != null ? nodeId.toString() : "bootstrap"); downstreamErrorCounter = taggedCounter(KROXYLICIOUS_DOWNSTREAM_ERRORS, tags); + clientToProxyErrorCounter = KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_TOTAL_METER_PROVIDER + .withTags(Metrics.VIRTUAL_CLUSTER_LABEL, clusterName, NODE_ID_LABEL, nodeId != null ? nodeId.toString() : "bootstrap"); + proxyToServerConnectionCounter = KROXYLICIOUS_PROXY_TO_SERVER_CONNECTION_TOTAL_METER_PROVIDER + .withTags(Metrics.VIRTUAL_CLUSTER_LABEL, clusterName, NODE_ID_LABEL, nodeId != null ? nodeId.toString() : "bootstrap"); + proxyToServerErrorCounter = KROXYLICIOUS_PROXY_TO_SERVER_ERROR_TOTAL_METER_PROVIDER + .withTags(Metrics.VIRTUAL_CLUSTER_LABEL, clusterName, NODE_ID_LABEL, nodeId != null ? nodeId.toString() : "bootstrap"); upstreamConnectionsCounter = taggedCounter(KROXYLICIOUS_UPSTREAM_CONNECTIONS, tags); connectionAttemptsCounter = taggedCounter(KROXYLICIOUS_UPSTREAM_CONNECTION_ATTEMPTS, tags); upstreamErrorCounter = taggedCounter(KROXYLICIOUS_UPSTREAM_ERRORS, tags); @@ -398,8 +439,10 @@ void onServerException(Throwable cause) { .log("Exception from the server channel: {}. Increase log level to DEBUG for stacktrace"); if (state instanceof ProxyChannelState.Connecting) { upstreamConnectionFailureCounter.increment(); + proxyToServerConnectionCounter.increment(); } upstreamErrorCounter.increment(); + proxyToServerErrorCounter.increment(); toClosed(cause); } @@ -426,6 +469,7 @@ void onClientException(Throwable cause, boolean tlsEnabled) { errorCodeEx = Errors.UNKNOWN_SERVER_ERROR.exception(); } downstreamErrorCounter.increment(); + clientToProxyErrorCounter.increment(); toClosed(errorCodeEx); } @@ -435,6 +479,7 @@ private void toClientActive( setState(clientActive); frontendHandler.inClientActive(); downstreamConnectionsCounter.increment(); + clientToProxyConnectionCounter.increment(); } private void toConnecting( @@ -445,12 +490,14 @@ private void toConnecting( backendHandler = new KafkaProxyBackendHandler(this, virtualClusterModel); frontendHandler.inConnecting(connecting.remote(), filters, backendHandler); connectionAttemptsCounter.increment(); + proxyToServerConnectionCounter.increment(); } private void toForwarding(Forwarding forwarding) { setState(forwarding); Objects.requireNonNull(frontendHandler).inForwarding(); upstreamConnectionsCounter.increment(); + proxyToServerConnectionCounter.increment(); } /** @@ -570,4 +617,4 @@ private static boolean isMessageApiVersionsRequest(Object msg) { return msg instanceof DecodedRequestFrame && ((DecodedRequestFrame) msg).apiKey() == ApiKeys.API_VERSIONS; } -} \ No newline at end of file +} diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java index 70d736b7a3..b67263892c 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java @@ -107,22 +107,59 @@ public class Metrics { @Deprecated(since = "0.13.0", forRemoval = true) public static final String KROXYLICIOUS_INBOUND_DOWNSTREAM_DECODED_MESSAGES = "kroxylicious_inbound_downstream_decoded_messages"; + static final String KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_BASE_METER_NAME = "kroxylicious_client_to_proxy_errors_total"; + static final String KROXYLICIOUS_PROXY_TO_SERVER_ERROR_BASE_METER_NAME = "kroxylicious_proxy_to_server_errors_total"; + static final String KROXYLICIOUS_CLIENT_TO_PROXY_CONNECTION_BASE_METER_NAME = "kroxylicious_client_to_proxy_connections_total"; + static final String KROXYLICIOUS_PROXY_TO_SERVER_CONNECTION_BASE_METER_NAME = "kroxylicious_proxy_to_server_connections_total"; + + public static MeterProvider KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_TOTAL_METER_PROVIDER = Counter.builder( + KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_BASE_METER_NAME) + .description("Incremented by one every time a connection is closed due to any downstream error.") + .withRegistry(globalRegistry); + + public static MeterProvider KROXYLICIOUS_PROXY_TO_SERVER_ERROR_TOTAL_METER_PROVIDER = Counter.builder( + KROXYLICIOUS_PROXY_TO_SERVER_ERROR_BASE_METER_NAME) + .description("Incremented by one every time a connection is closed due to any upstream error.") + .withRegistry(globalRegistry); + + public static MeterProvider KROXYLICIOUS_CLIENT_TO_PROXY_CONNECTION_TOTAL_METER_PROVIDER = Counter.builder( + KROXYLICIOUS_CLIENT_TO_PROXY_CONNECTION_BASE_METER_NAME) + .description("Incremented by one every time a connection is accepted from the downstream the proxy.") + .withRegistry(globalRegistry); + + public static MeterProvider KROXYLICIOUS_PROXY_TO_SERVER_CONNECTION_TOTAL_METER_PROVIDER = Counter.builder( + KROXYLICIOUS_PROXY_TO_SERVER_CONNECTION_BASE_METER_NAME) + .description("Incremented by one every time a connection is made to the upstream from the proxy.") + .withRegistry(globalRegistry); + private static final String KROXYLICIOUS_DOWNSTREAM = "kroxylicious_downstream_"; private static final String KROXYLICIOUS_UPSTREAM = "kroxylicious_upstream_"; + + @Deprecated(since = "0.13.0", forRemoval = true) public static final String KROXYLICIOUS_DOWNSTREAM_CONNECTIONS = KROXYLICIOUS_DOWNSTREAM + "connections"; + @Deprecated(since = "0.13.0", forRemoval = true) public static final String KROXYLICIOUS_DOWNSTREAM_ERRORS = KROXYLICIOUS_DOWNSTREAM + "errors"; + + @Deprecated(since = "0.13.0", forRemoval = true) public static final String KROXYLICIOUS_UPSTREAM_CONNECTIONS = KROXYLICIOUS_UPSTREAM + "connections"; + @Deprecated(since = "0.13.0", forRemoval = true) public static final String KROXYLICIOUS_UPSTREAM_CONNECTION_ATTEMPTS = KROXYLICIOUS_UPSTREAM + "connection_attempts"; + + @Deprecated(since = "0.13.0", forRemoval = true) public static final String KROXYLICIOUS_UPSTREAM_CONNECTION_FAILURES = KROXYLICIOUS_UPSTREAM + "connection_failures"; + + @Deprecated(since = "0.13.0", forRemoval = true) public static final String KROXYLICIOUS_UPSTREAM_ERRORS = KROXYLICIOUS_UPSTREAM + "errors"; + public static final String KROXYLICIOUS_PAYLOAD_SIZE_BYTES = "kroxylicious_payload_size_bytes"; public static final String FLOWING_TAG = "flowing"; public static final String VIRTUAL_CLUSTER_TAG = "virtualCluster"; + public static final String DOWNSTREAM = "downstream"; public static final String UPSTREAM = "upstream"; @@ -170,7 +207,7 @@ public static List tags(String... tagsAndValues) { @NonNull private static String required(String value) { - if ((Objects.isNull(value) || value.trim().isEmpty())) { + if ((Objects.isNull(value))) { throw new IllegalArgumentException("tag value supplied without a value"); } return value; @@ -186,4 +223,4 @@ public interface ClusterNodeSpecificMetricProviderCreator { MeterProvider create(String virtualCluster, Integer nodeId); } -} \ No newline at end of file +} diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandlerTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandlerTest.java index 2e87853a64..68423fbcbf 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandlerTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/KafkaProxyFrontendHandlerTest.java @@ -119,7 +119,7 @@ private DecodedRequestFrame decodedRequestFrame(short apiVersion, Ap public void buildChannel() { inboundChannel = new EmbeddedChannel(); corrId = 0; - proxyChannelStateMachine = new ProxyChannelStateMachine("RandomCluster"); + proxyChannelStateMachine = new ProxyChannelStateMachine("RandomCluster", null); } @AfterEach diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachineEndToEndTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachineEndToEndTest.java index 25a4b7ff1f..b9ac5afa04 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachineEndToEndTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachineEndToEndTest.java @@ -105,7 +105,7 @@ class ProxyChannelStateMachineEndToEndTest { @BeforeEach void setUp() { - proxyChannelStateMachine = new ProxyChannelStateMachine("RandomCluster"); + proxyChannelStateMachine = new ProxyChannelStateMachine("RandomCluster", null); } @AfterEach diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachineTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachineTest.java index 628fe80bb3..df87b42880 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachineTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachineTest.java @@ -92,7 +92,7 @@ class ProxyChannelStateMachineTest { @BeforeEach void setUp() { - proxyChannelStateMachine = new ProxyChannelStateMachine(CLUSTER_NAME); + proxyChannelStateMachine = new ProxyChannelStateMachine(CLUSTER_NAME, null); backendHandler = mock(KafkaProxyBackendHandler.class); frontendHandler = mock(KafkaProxyFrontendHandler.class); simpleMeterRegistry = new SimpleMeterRegistry(); From 0bd59dd4ac9bda535712f1ad040f744b7bf79e6d Mon Sep 17 00:00:00 2001 From: ShubhamRwt Date: Wed, 4 Jun 2025 15:13:41 +0530 Subject: [PATCH 68/83] ability to keep the label field empty Signed-off-by: ShubhamRwt Signed-off-by: Ujjwal --- .../proxy/internal/util/MetricsTest.java | 32 +------------------ 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/util/MetricsTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/util/MetricsTest.java index 0ea6f2d058..e1d9660875 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/util/MetricsTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/util/MetricsTest.java @@ -98,34 +98,4 @@ void shouldThrowIfOneOfSeveralTagsNameHasNoValue() { assertThatThrownBy(() -> Metrics.tags("TagA", "value1", "TagB", "value2", "TagC", "value3", "TagD")) .isInstanceOf(IllegalArgumentException.class); } - - @Test - void shouldThrowIfTagNameHasEmptyValue() { - // Given - - // When - // Then - assertThatThrownBy(() -> Metrics.tags("TagA", " ")) - .isInstanceOf(IllegalArgumentException.class); - } - - @Test - void shouldThrowIfOneOfTwoTagNameHasEmptyValue() { - // Given - - // When - // Then - assertThatThrownBy(() -> Metrics.tags("TagA", "value1", "TagB", "")) - .isInstanceOf(IllegalArgumentException.class); - } - - @Test - void shouldThrowIfOneOfSeveralTagsNameHasEmptyValue() { - // Given - - // When - // Then - assertThatThrownBy(() -> Metrics.tags("TagA", "value1", "TagB", "value2", "TagC", "value3", "TagD", " ")) - .isInstanceOf(IllegalArgumentException.class); - } -} \ No newline at end of file +} From 546a8bd30e0757b3c47dc7661e8ac05929642335 Mon Sep 17 00:00:00 2001 From: ShubhamRwt Date: Wed, 4 Jun 2025 18:12:49 +0530 Subject: [PATCH 69/83] fix sonar issues Signed-off-by: ShubhamRwt Signed-off-by: Ujjwal --- .../java/io/kroxylicious/proxy/MetricsIT.java | 17 ++--- .../internal/ProxyChannelStateMachine.java | 32 +++++---- .../proxy/internal/util/Metrics.java | 67 +++++++++++++------ 3 files changed, 70 insertions(+), 46 deletions(-) diff --git a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java index 7694a4bd5c..e94bc9f52b 100644 --- a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java +++ b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java @@ -403,17 +403,8 @@ static Stream connectionMetricsScenarios() { .filterByTag(NODE_ID_LABEL, "0") .singleElement() .value() - .isGreaterThan(0.0)), - argumentSet("count error from proxy to server (bootstrap)", - (UnaryOperator) builder -> builder, - (Consumer>) metricList -> assertThat(metricList) - .filterByName("kroxylicious_proxy_to_server_errors_total") - .filterByTag(NODE_ID_LABEL) - .isEmpty(), - (Consumer>) metricList -> assertThat(metricList).anySatisfy(simpleMetric -> { - assertThat(simpleMetric.name()).isEqualTo("kroxylicious_proxy_to_server_errors_total"); - assertThat(simpleMetric.value()).isGreaterThan(0); - }))); + .isGreaterThan(0.0)) + ); } @ParameterizedTest @@ -436,8 +427,8 @@ void shouldIncrementCountOnConnectionMetrics(UnaryOperator // When var producer = tester.producer(); var future = producer.send(new ProducerRecord<>("my-topic", "key", "value")); - assertThat(future).succeedsWithin(Duration.ofSeconds(5)); cluster.close(); + assertThat(future).succeedsWithin(Duration.ofSeconds(10)); // Then metricList = managementClient.scrapeMetrics(); @@ -598,8 +589,8 @@ void shouldIncrementUpstreamConnectionErrorMetrics(KafkaCluster cluster) throws // When var producer = tester.producer(); var future = producer.send(new ProducerRecord<>("my-topic", "key", "value")); - assertThat(future).succeedsWithin(Duration.ofSeconds(5)); cluster.close(); + assertThat(future).succeedsWithin(Duration.ofSeconds(10)); // Then // updated metrics after some message were produced diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachine.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachine.java index 4136b1ec96..518f5bcc6d 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachine.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachine.java @@ -107,36 +107,37 @@ * Thus when the proxy is notified that a peer is applying back pressure it results in action on the channel with the opposite peer. *

*/ +@SuppressWarnings("java:S1133") public class ProxyChannelStateMachine { private static final String DUPLICATE_INITIATE_CONNECT_ERROR = "NetFilter called NetFilterContext.initiateConnect() more than once"; private static final Logger LOGGER = getLogger(ProxyChannelStateMachine.class); /** - * @deprecated replaced by `clientToProxyConnectionCounter` + * @deprecated use `clientToProxyConnectionCounter` instead */ @Deprecated(since = "0.13.0", forRemoval = true) private final Counter downstreamConnectionsCounter; /** - * @deprecated replaced by `proxyToServerConnectionCounter` + * @deprecated use `proxyToServerConnectionCounter` instead */ @Deprecated(since = "0.13.0", forRemoval = true) private final Counter upstreamConnectionsCounter; /** - * @deprecated replaced by `clientToProxyErrorCounter` + * @deprecated use `clientToProxyErrorCounter` instead */ @Deprecated(since = "0.13.0", forRemoval = true) private final Counter downstreamErrorCounter; /** - * @deprecated replaced by `proxyToServerErrorCounter` + * @deprecated use `proxyToServerErrorCounter` instead */ @Deprecated(since = "0.13.0", forRemoval = true) private final Counter upstreamErrorCounter; /** - * @deprecated replaced by `proxyToServerConnectionCounter` + * @deprecated use `proxyToServerConnectionCounter` instead */ @Deprecated(since = "0.13.0", forRemoval = true) private final Counter connectionAttemptsCounter; /** - * @deprecated replaced by `proxyToServerConnectionCounter` + * @deprecated use `proxyToServerConnectionCounter` instead */ @Deprecated(since = "0.13.0", forRemoval = true) private final Counter upstreamConnectionFailureCounter; @@ -145,18 +146,20 @@ public class ProxyChannelStateMachine { private final Counter proxyToServerConnectionCounter; private final Counter proxyToServerErrorCounter; + @SuppressWarnings("java:S5738") public ProxyChannelStateMachine(String clusterName, @Nullable Integer nodeId) { - List tags = Metrics.tags(Metrics.VIRTUAL_CLUSTER_LABEL, clusterName); + List tags = Metrics.tags(Metrics.VIRTUAL_CLUSTER_TAG, clusterName); + String bootstrap = "bootstrap"; downstreamConnectionsCounter = taggedCounter(KROXYLICIOUS_DOWNSTREAM_CONNECTIONS, tags); clientToProxyConnectionCounter = KROXYLICIOUS_CLIENT_TO_PROXY_CONNECTION_TOTAL_METER_PROVIDER - .withTags(Metrics.VIRTUAL_CLUSTER_LABEL, clusterName, NODE_ID_LABEL, nodeId != null ? nodeId.toString() : "bootstrap"); + .withTags(Metrics.VIRTUAL_CLUSTER_LABEL, clusterName, NODE_ID_LABEL, nodeId != null ? nodeId.toString() : bootstrap); downstreamErrorCounter = taggedCounter(KROXYLICIOUS_DOWNSTREAM_ERRORS, tags); clientToProxyErrorCounter = KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_TOTAL_METER_PROVIDER - .withTags(Metrics.VIRTUAL_CLUSTER_LABEL, clusterName, NODE_ID_LABEL, nodeId != null ? nodeId.toString() : "bootstrap"); + .withTags(Metrics.VIRTUAL_CLUSTER_LABEL, clusterName, NODE_ID_LABEL, nodeId != null ? nodeId.toString() : bootstrap); proxyToServerConnectionCounter = KROXYLICIOUS_PROXY_TO_SERVER_CONNECTION_TOTAL_METER_PROVIDER - .withTags(Metrics.VIRTUAL_CLUSTER_LABEL, clusterName, NODE_ID_LABEL, nodeId != null ? nodeId.toString() : "bootstrap"); + .withTags(Metrics.VIRTUAL_CLUSTER_LABEL, clusterName, NODE_ID_LABEL, nodeId != null ? nodeId.toString() : bootstrap); proxyToServerErrorCounter = KROXYLICIOUS_PROXY_TO_SERVER_ERROR_TOTAL_METER_PROVIDER - .withTags(Metrics.VIRTUAL_CLUSTER_LABEL, clusterName, NODE_ID_LABEL, nodeId != null ? nodeId.toString() : "bootstrap"); + .withTags(Metrics.VIRTUAL_CLUSTER_LABEL, clusterName, NODE_ID_LABEL, nodeId != null ? nodeId.toString() : bootstrap); upstreamConnectionsCounter = taggedCounter(KROXYLICIOUS_UPSTREAM_CONNECTIONS, tags); connectionAttemptsCounter = taggedCounter(KROXYLICIOUS_UPSTREAM_CONNECTION_ATTEMPTS, tags); upstreamErrorCounter = taggedCounter(KROXYLICIOUS_UPSTREAM_ERRORS, tags); @@ -432,6 +435,7 @@ void onClientInactive() { * Notify the state machine that something exceptional and un-recoverable has happened on the upstream side. * @param cause the exception that triggered the issue */ + @SuppressWarnings("java:S5738") void onServerException(Throwable cause) { LOGGER.atWarn() .setCause(LOGGER.isDebugEnabled() ? cause : null) @@ -450,6 +454,7 @@ void onServerException(Throwable cause) { * Notify the state machine that something exceptional and un-recoverable has happened on the downstream side. * @param cause the exception that triggered the issue */ + @SuppressWarnings("java:S5738") void onClientException(Throwable cause, boolean tlsEnabled) { ApiException errorCodeEx; if (cause instanceof DecoderException de @@ -473,6 +478,7 @@ void onClientException(Throwable cause, boolean tlsEnabled) { toClosed(errorCodeEx); } + @SuppressWarnings("java:S5738") private void toClientActive( @NonNull ProxyChannelState.ClientActive clientActive, @NonNull KafkaProxyFrontendHandler frontendHandler) { @@ -482,6 +488,7 @@ private void toClientActive( clientToProxyConnectionCounter.increment(); } + @SuppressWarnings("java:S5738") private void toConnecting( ProxyChannelState.Connecting connecting, @NonNull List filters, @@ -493,6 +500,7 @@ private void toConnecting( proxyToServerConnectionCounter.increment(); } + @SuppressWarnings("java:S5738") private void toForwarding(Forwarding forwarding) { setState(forwarding); Objects.requireNonNull(frontendHandler).inForwarding(); @@ -525,7 +533,7 @@ else if (state() instanceof ProxyChannelState.SelectingServer) { } } - @SuppressWarnings("java:S1172") + @SuppressWarnings({"java:S1172", "java:S1135"}) // We keep dp as we should need it and it gives consistency with the other onClientRequestIn methods (sue me) private boolean onClientRequestInApiVersionsState(@NonNull SaslDecodePredicate dp, Object msg, ProxyChannelState.ApiVersions apiVersions) { if (msg instanceof RequestFrame) { diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java index b67263892c..f2b564d2d4 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java @@ -40,6 +40,10 @@ public class Metrics { private static final String KROXYLICIOUS_PROXY_TO_SERVER_REQUEST_BASE_METER_NAME = "kroxylicious_proxy_to_server_request"; private static final String KROXYLICIOUS_SERVER_TO_PROXY_RESPONSE_BASE_METER_NAME = "kroxylicious_server_to_proxy_response"; private static final String KROXYLICIOUS_PROXY_TO_CLIENT_RESPONSE_BASE_METER_NAME = "kroxylicious_proxy_to_client_response"; + private static final String KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_BASE_METER_NAME = "kroxylicious_client_to_proxy_errors_total"; + private static final String KROXYLICIOUS_PROXY_TO_SERVER_ERROR_BASE_METER_NAME = "kroxylicious_proxy_to_server_errors_total"; + private static final String KROXYLICIOUS_CLIENT_TO_PROXY_CONNECTION_BASE_METER_NAME = "kroxylicious_client_to_proxy_connections_total"; + private static final String KROXYLICIOUS_PROXY_TO_SERVER_CONNECTION_BASE_METER_NAME = "kroxylicious_proxy_to_server_connections_total"; // Meter Providers // (Callers use the providers to create the meters they need, augmenting with any tags). @@ -95,63 +99,84 @@ public class Metrics { .withRegistry( globalRegistry); - /** - * @deprecated use kroxylicious_client_to_proxy_request_count instead. - */ - @Deprecated(since = "0.13.0", forRemoval = true) - public static final String KROXYLICIOUS_INBOUND_DOWNSTREAM_MESSAGES = "kroxylicious_inbound_downstream_messages"; - - /** - * @deprecated use kroxylicious_client_to_proxy_request_count instead. - */ - @Deprecated(since = "0.13.0", forRemoval = true) - public static final String KROXYLICIOUS_INBOUND_DOWNSTREAM_DECODED_MESSAGES = "kroxylicious_inbound_downstream_decoded_messages"; - - static final String KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_BASE_METER_NAME = "kroxylicious_client_to_proxy_errors_total"; - static final String KROXYLICIOUS_PROXY_TO_SERVER_ERROR_BASE_METER_NAME = "kroxylicious_proxy_to_server_errors_total"; - static final String KROXYLICIOUS_CLIENT_TO_PROXY_CONNECTION_BASE_METER_NAME = "kroxylicious_client_to_proxy_connections_total"; - static final String KROXYLICIOUS_PROXY_TO_SERVER_CONNECTION_BASE_METER_NAME = "kroxylicious_proxy_to_server_connections_total"; - - public static MeterProvider KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_TOTAL_METER_PROVIDER = Counter.builder( + public static final MeterProvider KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_TOTAL_METER_PROVIDER = Counter.builder( KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_BASE_METER_NAME) .description("Incremented by one every time a connection is closed due to any downstream error.") .withRegistry(globalRegistry); - public static MeterProvider KROXYLICIOUS_PROXY_TO_SERVER_ERROR_TOTAL_METER_PROVIDER = Counter.builder( + public static final MeterProvider KROXYLICIOUS_PROXY_TO_SERVER_ERROR_TOTAL_METER_PROVIDER = Counter.builder( KROXYLICIOUS_PROXY_TO_SERVER_ERROR_BASE_METER_NAME) .description("Incremented by one every time a connection is closed due to any upstream error.") .withRegistry(globalRegistry); - public static MeterProvider KROXYLICIOUS_CLIENT_TO_PROXY_CONNECTION_TOTAL_METER_PROVIDER = Counter.builder( + public static final MeterProvider KROXYLICIOUS_CLIENT_TO_PROXY_CONNECTION_TOTAL_METER_PROVIDER = Counter.builder( KROXYLICIOUS_CLIENT_TO_PROXY_CONNECTION_BASE_METER_NAME) .description("Incremented by one every time a connection is accepted from the downstream the proxy.") .withRegistry(globalRegistry); - public static MeterProvider KROXYLICIOUS_PROXY_TO_SERVER_CONNECTION_TOTAL_METER_PROVIDER = Counter.builder( + public static final MeterProvider KROXYLICIOUS_PROXY_TO_SERVER_CONNECTION_TOTAL_METER_PROVIDER = Counter.builder( KROXYLICIOUS_PROXY_TO_SERVER_CONNECTION_BASE_METER_NAME) .description("Incremented by one every time a connection is made to the upstream from the proxy.") .withRegistry(globalRegistry); + /** + * @deprecated use kroxylicious_client_to_proxy_request_count instead. + */ + @Deprecated(since = "0.13.0", forRemoval = true) + @SuppressWarnings("java:S1133") + public static final String KROXYLICIOUS_INBOUND_DOWNSTREAM_MESSAGES = "kroxylicious_inbound_downstream_messages"; + + /** + * @deprecated use kroxylicious_client_to_proxy_request_count instead. + */ + @Deprecated(since = "0.13.0", forRemoval = true) + @SuppressWarnings("java:S1133") + public static final String KROXYLICIOUS_INBOUND_DOWNSTREAM_DECODED_MESSAGES = "kroxylicious_inbound_downstream_decoded_messages"; + private static final String KROXYLICIOUS_DOWNSTREAM = "kroxylicious_downstream_"; private static final String KROXYLICIOUS_UPSTREAM = "kroxylicious_upstream_"; + /** + * @deprecated use `clientToProxyConnectionCounter` instead + */ @Deprecated(since = "0.13.0", forRemoval = true) + @SuppressWarnings("java:S1133") public static final String KROXYLICIOUS_DOWNSTREAM_CONNECTIONS = KROXYLICIOUS_DOWNSTREAM + "connections"; + /** + * @deprecated use `clientToProxyErrorCounter` instead + */ @Deprecated(since = "0.13.0", forRemoval = true) + @SuppressWarnings("java:S1133") public static final String KROXYLICIOUS_DOWNSTREAM_ERRORS = KROXYLICIOUS_DOWNSTREAM + "errors"; + /** + * @deprecated use `proxyToServerConnectionCounter` instead + */ @Deprecated(since = "0.13.0", forRemoval = true) + @SuppressWarnings("java:S1133") public static final String KROXYLICIOUS_UPSTREAM_CONNECTIONS = KROXYLICIOUS_UPSTREAM + "connections"; + /** + * @deprecated use `proxyToServerConnectionCounter` instead + */ @Deprecated(since = "0.13.0", forRemoval = true) + @SuppressWarnings("java:S1133") public static final String KROXYLICIOUS_UPSTREAM_CONNECTION_ATTEMPTS = KROXYLICIOUS_UPSTREAM + "connection_attempts"; + /** + * @deprecated use `proxyToServerConnectionCounter` instead + */ @Deprecated(since = "0.13.0", forRemoval = true) + @SuppressWarnings("java:S1133") public static final String KROXYLICIOUS_UPSTREAM_CONNECTION_FAILURES = KROXYLICIOUS_UPSTREAM + "connection_failures"; + /** + * @deprecated use `proxyToServerErrorCounter` instead + */ @Deprecated(since = "0.13.0", forRemoval = true) + @SuppressWarnings("java:S1133") public static final String KROXYLICIOUS_UPSTREAM_ERRORS = KROXYLICIOUS_UPSTREAM + "errors"; public static final String KROXYLICIOUS_PAYLOAD_SIZE_BYTES = "kroxylicious_payload_size_bytes"; From a31e3e8d0468ac189041029745369e101be8f71b Mon Sep 17 00:00:00 2001 From: ShubhamRwt Date: Wed, 4 Jun 2025 22:56:38 +0530 Subject: [PATCH 70/83] fix test Signed-off-by: ShubhamRwt Signed-off-by: Ujjwal --- .../java/io/kroxylicious/proxy/MetricsIT.java | 105 +++++++++++------- .../proxy/internal/KafkaProxyInitializer.java | 4 +- .../internal/ProxyChannelStateMachine.java | 2 +- .../proxy/internal/util/MetricsTest.java | 32 ++++++ 4 files changed, 101 insertions(+), 42 deletions(-) diff --git a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java index e94bc9f52b..9c98b31133 100644 --- a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java +++ b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java @@ -51,15 +51,19 @@ import io.kroxylicious.proxy.config.MicrometerDefinitionBuilder; import io.kroxylicious.proxy.config.NamedFilterDefinition; import io.kroxylicious.proxy.config.NamedFilterDefinitionBuilder; +import io.kroxylicious.proxy.config.VirtualClusterBuilder; import io.kroxylicious.proxy.config.VirtualClusterGateway; +import io.kroxylicious.proxy.config.secret.InlinePassword; import io.kroxylicious.proxy.micrometer.CommonTagsHook; import io.kroxylicious.proxy.micrometer.StandardBindersHook; import io.kroxylicious.proxy.service.HostPort; import io.kroxylicious.test.tester.KroxyliciousTester; import io.kroxylicious.test.tester.SimpleMetric; +import io.kroxylicious.test.tester.SimpleMetricAssert; import io.kroxylicious.testing.kafka.api.KafkaCluster; import io.kroxylicious.testing.kafka.common.BrokerCluster; import io.kroxylicious.testing.kafka.common.KeytoolCertificateGenerator; +import io.kroxylicious.testing.kafka.common.Tls; import io.kroxylicious.testing.kafka.junit5ext.KafkaClusterExtension; import io.kroxylicious.testing.kafka.junit5ext.Topic; import io.kroxylicious.testing.kafka.junit5ext.TopicPartitions; @@ -86,6 +90,7 @@ class MetricsIT { private static Path certsDirectory; private static final HostPort PORT_IDENTIFIES_BROKER_BOOTSTRAP = new HostPort("localhost", 9092); + private static final HostPort PROXY_ADDRESS = HostPort.parse("localhost:9192"); private static final String SNI_IDENTIFIES_BROKER_BASE_ADDRESS = IntegrationTestInetAddressResolverProvider.generateFullyQualifiedDomainName("sni"); private static final HostPort SNI_IDENTIFIES_BROKER_BOOTSTRAP = new HostPort("bootstrap." + SNI_IDENTIFIES_BROKER_BASE_ADDRESS, 9192); @@ -403,8 +408,7 @@ static Stream connectionMetricsScenarios() { .filterByTag(NODE_ID_LABEL, "0") .singleElement() .value() - .isGreaterThan(0.0)) - ); + .isGreaterThan(0.0))); } @ParameterizedTest @@ -412,8 +416,7 @@ static Stream connectionMetricsScenarios() { void shouldIncrementCountOnConnectionMetrics(UnaryOperator builder, Consumer> beforeAssertion, Consumer> afterAssertion, - KafkaCluster cluster) - throws Exception { + KafkaCluster cluster) { var config = configWithMetrics(cluster); // Given @@ -427,8 +430,7 @@ void shouldIncrementCountOnConnectionMetrics(UnaryOperator // When var producer = tester.producer(); var future = producer.send(new ProducerRecord<>("my-topic", "key", "value")); - cluster.close(); - assertThat(future).succeedsWithin(Duration.ofSeconds(10)); + assertThat(future).succeedsWithin(Duration.ofSeconds(5)); // Then metricList = managementClient.scrapeMetrics(); @@ -567,38 +569,6 @@ void shouldIncrementConnectionMetrics(KafkaCluster cluster, Topic topic) throws } } - @Test - @Deprecated(since = "0.13.0", forRemoval = true) - void shouldIncrementUpstreamConnectionErrorMetrics(KafkaCluster cluster) throws Exception { - var config = proxy(cluster) - .addToFilterDefinitions() - .addToDefaultFilters() - .withNewManagement() - .withNewEndpoints() - .withNewPrometheus() - .endPrometheus() - .endEndpoints() - .endManagement(); - - // Given - try (var tester = kroxyliciousTester(config); - var managementClient = tester.getManagementClient()) { - var metricsList = managementClient.scrapeMetrics(); - assertMetricsDoesNotExist(metricsList, "kroxylicious_upstream_errors_total", null); - - // When - var producer = tester.producer(); - var future = producer.send(new ProducerRecord<>("my-topic", "key", "value")); - cluster.close(); - assertThat(future).succeedsWithin(Duration.ofSeconds(10)); - - // Then - // updated metrics after some message were produced - var updatedMetricsList = managementClient.scrapeMetrics(); - assertMetricsWithValue(updatedMetricsList, "kroxylicious_upstream_errors_total", null); - } - } - static Stream createDownstreamConnectionError() throws Exception { var failQuickDescribeCluster = new DescribeClusterOptions().timeoutMs(2_000); // Note that the KeytoolCertificateGenerator generates key stores that are PKCS12 format. @@ -718,6 +688,65 @@ void shouldIncrementDownstreamErrorOnConnectionError(Supplier newTags = Metrics.tags(VIRTUAL_CLUSTER_LABEL, "", NODE_ID_LABEL, "bootstrap"); - clientToProxyErrorCounter = KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_TOTAL_METER_PROVIDER.withTags(newTags); + clientToProxyErrorCounter = KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_TOTAL_METER_PROVIDER.withTags(VIRTUAL_CLUSTER_LABEL, "", NODE_ID_LABEL, "bootstrap"); } @Override diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachine.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachine.java index 518f5bcc6d..79a960300a 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachine.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachine.java @@ -533,7 +533,7 @@ else if (state() instanceof ProxyChannelState.SelectingServer) { } } - @SuppressWarnings({"java:S1172", "java:S1135"}) + @SuppressWarnings({ "java:S1172", "java:S1135" }) // We keep dp as we should need it and it gives consistency with the other onClientRequestIn methods (sue me) private boolean onClientRequestInApiVersionsState(@NonNull SaslDecodePredicate dp, Object msg, ProxyChannelState.ApiVersions apiVersions) { if (msg instanceof RequestFrame) { diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/util/MetricsTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/util/MetricsTest.java index e1d9660875..b33fd99d4e 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/util/MetricsTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/util/MetricsTest.java @@ -98,4 +98,36 @@ void shouldThrowIfOneOfSeveralTagsNameHasNoValue() { assertThatThrownBy(() -> Metrics.tags("TagA", "value1", "TagB", "value2", "TagC", "value3", "TagD")) .isInstanceOf(IllegalArgumentException.class); } + + @Test + void shouldThrowIfTagNameHasEmptyValue() { + // Given + + // When + // Then + assertThatThrownBy(() -> Metrics.tags("TagA", " ")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void shouldThrowIfOneOfTwoTagNameHasEmptyValue() { + // Given + + // When + // Then + assertThatThrownBy(() -> Metrics.tags("TagA", "value1", "TagB", "")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void shouldThrowIfOneOfSeveralTagsNameHasEmptyValue() { + // Given + + // When + // Then + assertThatThrownBy(() -> Metrics.tags("TagA", "value1", "TagB", "value2", "TagC", "value3", "TagD", " ")) + .isInstanceOf(IllegalArgumentException.class); + } } + + From f5425b37c647de6ac084c5ba1a8be37a5efe20ef Mon Sep 17 00:00:00 2001 From: ShubhamRwt Date: Wed, 4 Jun 2025 23:01:08 +0530 Subject: [PATCH 71/83] Revert changes Signed-off-by: ShubhamRwt Signed-off-by: Ujjwal --- .../main/java/io/kroxylicious/proxy/internal/util/Metrics.java | 2 +- .../java/io/kroxylicious/proxy/internal/util/MetricsTest.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java index f2b564d2d4..e28eb80e7f 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java @@ -232,7 +232,7 @@ public static List tags(String... tagsAndValues) { @NonNull private static String required(String value) { - if ((Objects.isNull(value))) { + if ((Objects.isNull(value) || value.trim().isEmpty())) { throw new IllegalArgumentException("tag value supplied without a value"); } return value; diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/util/MetricsTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/util/MetricsTest.java index b33fd99d4e..1bc4cc7e12 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/util/MetricsTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/util/MetricsTest.java @@ -128,6 +128,5 @@ void shouldThrowIfOneOfSeveralTagsNameHasEmptyValue() { assertThatThrownBy(() -> Metrics.tags("TagA", "value1", "TagB", "value2", "TagC", "value3", "TagD", " ")) .isInstanceOf(IllegalArgumentException.class); } -} - +} \ No newline at end of file From 2b97035950938177775f1113303fd4581dbc8629 Mon Sep 17 00:00:00 2001 From: ShubhamRwt Date: Thu, 5 Jun 2025 12:43:03 +0530 Subject: [PATCH 72/83] Add suggestions by Sam Signed-off-by: ShubhamRwt Signed-off-by: Ujjwal --- .../java/io/kroxylicious/proxy/MetricsIT.java | 152 ++++++++---------- .../proxy/internal/KafkaProxyInitializer.java | 4 +- .../internal/ProxyChannelStateMachine.java | 10 +- .../proxy/internal/util/Metrics.java | 70 +++++--- 4 files changed, 126 insertions(+), 110 deletions(-) diff --git a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java index 9c98b31133..0796654544 100644 --- a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java +++ b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java @@ -10,7 +10,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.time.Duration; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -372,7 +371,7 @@ static Stream connectionMetricsScenarios() { .filterByTag(NODE_ID_LABEL, "bootstrap") .singleElement() .value() - .isGreaterThan(0.0)), + .isGreaterThan(1.0)), argumentSet("counts connections to node, client to proxy", (UnaryOperator) builder -> builder, (Consumer>) metricList -> assertThat(metricList) @@ -384,7 +383,7 @@ static Stream connectionMetricsScenarios() { .filterByTag(NODE_ID_LABEL, "0") .singleElement() .value() - .isGreaterThan(0.0)), + .isGreaterThan(1.0)), argumentSet("count bootstrap connection from proxy to server ", (UnaryOperator) builder -> builder, (Consumer>) metricList -> assertThat(metricList) @@ -396,7 +395,7 @@ static Stream connectionMetricsScenarios() { .filterByTag(NODE_ID_LABEL, "bootstrap") .singleElement() .value() - .isGreaterThan(0.0)), + .isGreaterThan(1.0)), argumentSet("count node connection from proxy to server ", (UnaryOperator) builder -> builder, (Consumer>) metricList -> assertThat(metricList) @@ -408,7 +407,7 @@ static Stream connectionMetricsScenarios() { .filterByTag(NODE_ID_LABEL, "0") .singleElement() .value() - .isGreaterThan(0.0))); + .isGreaterThan(1.0))); } @ParameterizedTest @@ -578,83 +577,72 @@ static Stream createDownstreamConnectionError() throws Exception { downstreamCertificateGenerator.generateTrustStore(downstreamCertificateGenerator.getCertFilePath(), "client", clientTrustStore.toAbsolutePath().toString()); - var list = new ArrayList(); - - { - list.add(argumentSet("gateway is tcp, client sends malformed kafka", - (Supplier) () -> defaultPortIdentifiesNodeGatewayBuilder(PORT_IDENTIFIES_BROKER_BOOTSTRAP) - .withName("default") - .build(), - (Consumer) kroxyliciousTester -> { - var hostPort = HostPort.parse(kroxyliciousTester.getBootstrapAddress()); - try (var sock = new Socket(hostPort.host(), hostPort.port()); - var output = sock.getOutputStream()) { - output.write("This ain't Kafka!".getBytes(StandardCharsets.UTF_8)); - } - catch (IOException e) { - throw new RuntimeException(e); - } - })); - } - - { - - list.add(argumentSet("gateway is tls, client connects using plain", - (Supplier) () -> defaultSniHostIdentifiesNodeGatewayBuilder(SNI_IDENTIFIES_BROKER_BOOTSTRAP, - SNI_IDENTIFIES_BROKER_ADDRESS_PATTERN) - .withName("default") - .withNewTls() - .withNewKeyStoreKey() - .withStoreFile(downstreamCertificateGenerator.getKeyStoreLocation()) - .withNewInlinePasswordStoreProvider(downstreamCertificateGenerator.getPassword()) - .endKeyStoreKey() - .endTls() - .build(), - (Consumer) kroxyliciousTester -> { - try (var admin = kroxyliciousTester.admin(Map.of(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "PLAINTEXT"))) { - assertThat(admin.describeCluster(failQuickDescribeCluster).clusterId()) - .failsWithin(Duration.ofSeconds(5)) - .withThrowableThat() - .withCauseInstanceOf(TimeoutException.class) - .havingCause() - .withMessageStartingWith("Timed out waiting for a node assignment."); - } - })); - } + return Stream.of( + argumentSet("gateway is tcp, client sends malformed kafka", + (Supplier) () -> defaultPortIdentifiesNodeGatewayBuilder(PORT_IDENTIFIES_BROKER_BOOTSTRAP) + .withName("default") + .build(), + (Consumer) kroxyliciousTester -> { + var hostPort = HostPort.parse(kroxyliciousTester.getBootstrapAddress()); + try (var sock = new Socket(hostPort.host(), hostPort.port()); + var output = sock.getOutputStream()) { + output.write("This ain't Kafka!".getBytes(StandardCharsets.UTF_8)); + } + catch (IOException e) { + throw new RuntimeException(e); + } + }), - { - var unrecognisedSni = IntegrationTestInetAddressResolverProvider.generateFullyQualifiedDomainName("unrecognised"); - - list.add(argumentSet("gateway is tls/sni, unrecognised SNI name", - (Supplier) () -> defaultSniHostIdentifiesNodeGatewayBuilder(SNI_IDENTIFIES_BROKER_BOOTSTRAP, - SNI_IDENTIFIES_BROKER_ADDRESS_PATTERN) - .withName("default") - .withNewTls() - .withNewKeyStoreKey() - .withStoreFile(downstreamCertificateGenerator.getKeyStoreLocation()) - .withNewInlinePasswordStoreProvider(downstreamCertificateGenerator.getPassword()) - .endKeyStoreKey() - .endTls() - .build(), - (Consumer) kroxyliciousTester -> { - var hostPort = HostPort.parse(kroxyliciousTester.getBootstrapAddress()); - var unrecognisedHostPort = unrecognisedSni + ":" + hostPort.port(); - - try (var admin = kroxyliciousTester.admin( - Map.of(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, unrecognisedHostPort, - SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, clientTrustStore.toAbsolutePath().toString(), - SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, downstreamCertificateGenerator.getPassword()))) { - assertThat(admin.describeCluster(failQuickDescribeCluster).clusterId()) - .failsWithin(Duration.ofSeconds(5)) - .withThrowableThat() - .withCauseInstanceOf(TimeoutException.class) - .havingCause() - .withMessageStartingWith("Timed out waiting for a node assignment."); - } - })); - } - return list.stream(); + argumentSet("gateway is tls, client connects using plain", + (Supplier) () -> defaultSniHostIdentifiesNodeGatewayBuilder(SNI_IDENTIFIES_BROKER_BOOTSTRAP, + SNI_IDENTIFIES_BROKER_ADDRESS_PATTERN) + .withName("default") + .withNewTls() + .withNewKeyStoreKey() + .withStoreFile(downstreamCertificateGenerator.getKeyStoreLocation()) + .withNewInlinePasswordStoreProvider(downstreamCertificateGenerator.getPassword()) + .endKeyStoreKey() + .endTls() + .build(), + (Consumer) kroxyliciousTester -> { + try (var admin = kroxyliciousTester.admin(Map.of(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "PLAINTEXT"))) { + assertThat(admin.describeCluster(failQuickDescribeCluster).clusterId()) + .failsWithin(Duration.ofSeconds(5)) + .withThrowableThat() + .withCauseInstanceOf(TimeoutException.class) + .havingCause() + .withMessageStartingWith("Timed out waiting for a node assignment."); + } + }), + argumentSet("gateway is tls/sni, unrecognised SNI name", + (Supplier) () -> defaultSniHostIdentifiesNodeGatewayBuilder(SNI_IDENTIFIES_BROKER_BOOTSTRAP, + SNI_IDENTIFIES_BROKER_ADDRESS_PATTERN) + .withName("default") + .withNewTls() + .withNewKeyStoreKey() + .withStoreFile(downstreamCertificateGenerator.getKeyStoreLocation()) + .withNewInlinePasswordStoreProvider(downstreamCertificateGenerator.getPassword()) + .endKeyStoreKey() + .endTls() + .build(), + (Consumer) kroxyliciousTester -> { + var hostPort = HostPort.parse(kroxyliciousTester.getBootstrapAddress()); + var unrecognisedSni = IntegrationTestInetAddressResolverProvider.generateFullyQualifiedDomainName("unrecognised"); + var unrecognisedHostPort = unrecognisedSni + ":" + hostPort.port(); + + try (var admin = kroxyliciousTester.admin( + Map.of(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, unrecognisedHostPort, + SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, clientTrustStore.toAbsolutePath().toString(), + SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, downstreamCertificateGenerator.getPassword()))) { + assertThat(admin.describeCluster(failQuickDescribeCluster).clusterId()) + .failsWithin(Duration.ofSeconds(5)) + .withThrowableThat() + .withCauseInstanceOf(TimeoutException.class) + .havingCause() + .withMessageStartingWith("Timed out waiting for a node assignment."); + } + })); } @ParameterizedTest @@ -736,14 +724,14 @@ void shouldIncrementUpstreamErrorOnConnectionError(@Tls KafkaCluster cluster) { .filterByTag(NODE_ID_LABEL, "bootstrap") .singleElement() .value() - .isGreaterThan(0.0); + .isGreaterThan(1.0); SimpleMetricAssert.assertThat(metricList) .filterByName("kroxylicious_proxy_to_server_errors_total") .filterByTag(NODE_ID_LABEL, "bootstrap") .singleElement() .value() - .isGreaterThan(0.0); + .isGreaterThan(1.0); } } diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java index c6e8906e46..fe48eeb7a9 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java @@ -51,8 +51,6 @@ import io.kroxylicious.proxy.tag.VisibleForTesting; import static io.kroxylicious.proxy.internal.util.Metrics.KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_TOTAL_METER_PROVIDER; -import static io.kroxylicious.proxy.internal.util.Metrics.NODE_ID_LABEL; -import static io.kroxylicious.proxy.internal.util.Metrics.VIRTUAL_CLUSTER_LABEL; public class KafkaProxyInitializer extends ChannelInitializer { @@ -88,7 +86,7 @@ public KafkaProxyInitializer(FilterChainFactory filterChainFactory, this.bindingResolver = bindingResolver; this.filterChainFactory = filterChainFactory; this.apiVersionsService = apiVersionsService; - clientToProxyErrorCounter = KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_TOTAL_METER_PROVIDER.withTags(VIRTUAL_CLUSTER_LABEL, "", NODE_ID_LABEL, "bootstrap"); + clientToProxyErrorCounter = KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_TOTAL_METER_PROVIDER.create("", null).withTags(); } @Override diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachine.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachine.java index 79a960300a..87234406f8 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachine.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachine.java @@ -48,7 +48,6 @@ import static io.kroxylicious.proxy.internal.util.Metrics.KROXYLICIOUS_UPSTREAM_CONNECTION_ATTEMPTS; import static io.kroxylicious.proxy.internal.util.Metrics.KROXYLICIOUS_UPSTREAM_CONNECTION_FAILURES; import static io.kroxylicious.proxy.internal.util.Metrics.KROXYLICIOUS_UPSTREAM_ERRORS; -import static io.kroxylicious.proxy.internal.util.Metrics.NODE_ID_LABEL; import static io.kroxylicious.proxy.internal.util.Metrics.taggedCounter; import static org.slf4j.LoggerFactory.getLogger; @@ -149,17 +148,16 @@ public class ProxyChannelStateMachine { @SuppressWarnings("java:S5738") public ProxyChannelStateMachine(String clusterName, @Nullable Integer nodeId) { List tags = Metrics.tags(Metrics.VIRTUAL_CLUSTER_TAG, clusterName); - String bootstrap = "bootstrap"; downstreamConnectionsCounter = taggedCounter(KROXYLICIOUS_DOWNSTREAM_CONNECTIONS, tags); clientToProxyConnectionCounter = KROXYLICIOUS_CLIENT_TO_PROXY_CONNECTION_TOTAL_METER_PROVIDER - .withTags(Metrics.VIRTUAL_CLUSTER_LABEL, clusterName, NODE_ID_LABEL, nodeId != null ? nodeId.toString() : bootstrap); + .create(clusterName, nodeId).withTags(); downstreamErrorCounter = taggedCounter(KROXYLICIOUS_DOWNSTREAM_ERRORS, tags); clientToProxyErrorCounter = KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_TOTAL_METER_PROVIDER - .withTags(Metrics.VIRTUAL_CLUSTER_LABEL, clusterName, NODE_ID_LABEL, nodeId != null ? nodeId.toString() : bootstrap); + .create(clusterName, nodeId).withTags(); proxyToServerConnectionCounter = KROXYLICIOUS_PROXY_TO_SERVER_CONNECTION_TOTAL_METER_PROVIDER - .withTags(Metrics.VIRTUAL_CLUSTER_LABEL, clusterName, NODE_ID_LABEL, nodeId != null ? nodeId.toString() : bootstrap); + .create(clusterName, nodeId).withTags(); proxyToServerErrorCounter = KROXYLICIOUS_PROXY_TO_SERVER_ERROR_TOTAL_METER_PROVIDER - .withTags(Metrics.VIRTUAL_CLUSTER_LABEL, clusterName, NODE_ID_LABEL, nodeId != null ? nodeId.toString() : bootstrap); + .create(clusterName, nodeId).withTags(); upstreamConnectionsCounter = taggedCounter(KROXYLICIOUS_UPSTREAM_CONNECTIONS, tags); connectionAttemptsCounter = taggedCounter(KROXYLICIOUS_UPSTREAM_CONNECTION_ATTEMPTS, tags); upstreamErrorCounter = taggedCounter(KROXYLICIOUS_UPSTREAM_ERRORS, tags); diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java index e28eb80e7f..0b535b755a 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java @@ -99,25 +99,57 @@ public class Metrics { .withRegistry( globalRegistry); - public static final MeterProvider KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_TOTAL_METER_PROVIDER = Counter.builder( - KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_BASE_METER_NAME) - .description("Incremented by one every time a connection is closed due to any downstream error.") - .withRegistry(globalRegistry); - - public static final MeterProvider KROXYLICIOUS_PROXY_TO_SERVER_ERROR_TOTAL_METER_PROVIDER = Counter.builder( - KROXYLICIOUS_PROXY_TO_SERVER_ERROR_BASE_METER_NAME) - .description("Incremented by one every time a connection is closed due to any upstream error.") - .withRegistry(globalRegistry); - - public static final MeterProvider KROXYLICIOUS_CLIENT_TO_PROXY_CONNECTION_TOTAL_METER_PROVIDER = Counter.builder( - KROXYLICIOUS_CLIENT_TO_PROXY_CONNECTION_BASE_METER_NAME) - .description("Incremented by one every time a connection is accepted from the downstream the proxy.") - .withRegistry(globalRegistry); - - public static final MeterProvider KROXYLICIOUS_PROXY_TO_SERVER_CONNECTION_TOTAL_METER_PROVIDER = Counter.builder( - KROXYLICIOUS_PROXY_TO_SERVER_CONNECTION_BASE_METER_NAME) - .description("Incremented by one every time a connection is made to the upstream from the proxy.") - .withRegistry(globalRegistry); + public static final ClusterNodeSpecificMetricProviderCreator KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_TOTAL_METER_PROVIDER = (virtualClusterName, + nodeId) -> Counter.builder( + KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_BASE_METER_NAME) + .description( + "Incremented by one every time a connection is closed due to any downstream error.") + .tag(VIRTUAL_CLUSTER_LABEL, + virtualClusterName) + .tag(NODE_ID_LABEL, + nodeIdToLabelValue( + nodeId)) + .withRegistry( + globalRegistry); + + public static final ClusterNodeSpecificMetricProviderCreator KROXYLICIOUS_PROXY_TO_SERVER_ERROR_TOTAL_METER_PROVIDER = (virtualClusterName, + nodeId) -> Counter.builder( + KROXYLICIOUS_PROXY_TO_SERVER_ERROR_BASE_METER_NAME) + .description( + "Incremented by one every time a connection is closed due to any upstream error.") + .tag(VIRTUAL_CLUSTER_LABEL, + virtualClusterName) + .tag(NODE_ID_LABEL, + nodeIdToLabelValue( + nodeId)) + .withRegistry( + globalRegistry); + + public static final ClusterNodeSpecificMetricProviderCreator KROXYLICIOUS_CLIENT_TO_PROXY_CONNECTION_TOTAL_METER_PROVIDER = (virtualClusterName, + nodeId) -> Counter.builder( + KROXYLICIOUS_CLIENT_TO_PROXY_CONNECTION_BASE_METER_NAME) + .description( + "Incremented by one every time a connection is accepted from the downstream the proxy.") + .tag(VIRTUAL_CLUSTER_LABEL, + virtualClusterName) + .tag(NODE_ID_LABEL, + nodeIdToLabelValue( + nodeId)) + .withRegistry( + globalRegistry); + + public static final ClusterNodeSpecificMetricProviderCreator KROXYLICIOUS_PROXY_TO_SERVER_CONNECTION_TOTAL_METER_PROVIDER = (virtualClusterName, + nodeId) -> Counter.builder( + KROXYLICIOUS_PROXY_TO_SERVER_CONNECTION_BASE_METER_NAME) + .description( + "Incremented by one every time a connection is made to the upstream from the proxy.") + .tag(VIRTUAL_CLUSTER_LABEL, + virtualClusterName) + .tag(NODE_ID_LABEL, + nodeIdToLabelValue( + nodeId)) + .withRegistry( + globalRegistry); /** * @deprecated use kroxylicious_client_to_proxy_request_count instead. From 63a3ac718a82c3b50dcc0c3f0460bf77bb41382b Mon Sep 17 00:00:00 2001 From: ShubhamRwt Date: Thu, 5 Jun 2025 12:50:50 +0530 Subject: [PATCH 73/83] use greaterThanOrEqual() method Signed-off-by: ShubhamRwt Signed-off-by: Ujjwal --- .../src/test/java/io/kroxylicious/proxy/MetricsIT.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java index 0796654544..09ce913b6a 100644 --- a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java +++ b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java @@ -371,7 +371,7 @@ static Stream connectionMetricsScenarios() { .filterByTag(NODE_ID_LABEL, "bootstrap") .singleElement() .value() - .isGreaterThan(1.0)), + .isGreaterThanOrEqualTo(1.0)), argumentSet("counts connections to node, client to proxy", (UnaryOperator) builder -> builder, (Consumer>) metricList -> assertThat(metricList) @@ -383,7 +383,7 @@ static Stream connectionMetricsScenarios() { .filterByTag(NODE_ID_LABEL, "0") .singleElement() .value() - .isGreaterThan(1.0)), + .isGreaterThanOrEqualTo(1.0)), argumentSet("count bootstrap connection from proxy to server ", (UnaryOperator) builder -> builder, (Consumer>) metricList -> assertThat(metricList) @@ -395,7 +395,7 @@ static Stream connectionMetricsScenarios() { .filterByTag(NODE_ID_LABEL, "bootstrap") .singleElement() .value() - .isGreaterThan(1.0)), + .isGreaterThanOrEqualTo(1.0)), argumentSet("count node connection from proxy to server ", (UnaryOperator) builder -> builder, (Consumer>) metricList -> assertThat(metricList) @@ -407,7 +407,7 @@ static Stream connectionMetricsScenarios() { .filterByTag(NODE_ID_LABEL, "0") .singleElement() .value() - .isGreaterThan(1.0))); + .isGreaterThanOrEqualTo(1.0))); } @ParameterizedTest From d934740884989bce179b22a58d1dfa5789e78abf Mon Sep 17 00:00:00 2001 From: ShubhamRwt Date: Thu, 5 Jun 2025 15:13:45 +0530 Subject: [PATCH 74/83] update w.r.t to Keith's changes for SNI handling Signed-off-by: ShubhamRwt Signed-off-by: Ujjwal --- .../kroxylicious/proxy/internal/KafkaProxyInitializer.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java index fe48eeb7a9..ed61754381 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java @@ -149,15 +149,13 @@ private void initTlsChannel(SocketChannel ch, ChannelPipeline pipeline, Optional try { if (t != null) { LOGGER.warn("Exception resolving Virtual Cluster Binding for endpoint {} and sniHostname {}: {}", endpoint, sniHostname, t.getMessage()); - clientToProxyErrorCounter.increment(); promise.setFailure(t); - ch.close(); // KW TODO - check right way to handle terminal errors within the SNIHandler. + ch.close(); return null; } var gateway = binding.endpointGateway(); var sslContext = gateway.getDownstreamSslContext(); if (sslContext.isEmpty()) { - clientToProxyErrorCounter.increment(); promise.setFailure(new IllegalStateException("Virtual cluster %s does not provide SSL context".formatted(gateway))); // close here too? } @@ -167,7 +165,6 @@ private void initTlsChannel(SocketChannel ch, ChannelPipeline pipeline, Optional } } catch (Throwable t1) { - clientToProxyErrorCounter.increment(); promise.setFailure(t1); // and close here too? } @@ -190,6 +187,7 @@ protected void onLookupComplete(ChannelHandlerContext ctx, Future fu // We've failed to look up the SslContext - this indicates that the SNI hostname was unrecognized // or that the virtual cluster is somehow not configured for TLS. All we can do is close the // connection. + clientToProxyErrorCounter.increment(); ctx.close(); } From d7ae6e8afe14b3bf077269df2462390755b1de08 Mon Sep 17 00:00:00 2001 From: ShubhamRwt Date: Thu, 5 Jun 2025 17:28:59 +0530 Subject: [PATCH 75/83] Added suggestions by Keith and Sam Signed-off-by: ShubhamRwt Signed-off-by: Ujjwal --- .../java/io/kroxylicious/proxy/MetricsIT.java | 308 ++++++++---------- .../proxy/internal/KafkaProxyInitializer.java | 3 - .../internal/ProxyChannelStateMachine.java | 15 +- .../proxy/internal/util/Metrics.java | 2 +- .../proxy/internal/util/MetricsTest.java | 1 - 5 files changed, 148 insertions(+), 181 deletions(-) diff --git a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java index 09ce913b6a..b998b7de64 100644 --- a/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java +++ b/kroxylicious-integration-tests/src/test/java/io/kroxylicious/proxy/MetricsIT.java @@ -50,9 +50,7 @@ import io.kroxylicious.proxy.config.MicrometerDefinitionBuilder; import io.kroxylicious.proxy.config.NamedFilterDefinition; import io.kroxylicious.proxy.config.NamedFilterDefinitionBuilder; -import io.kroxylicious.proxy.config.VirtualClusterBuilder; import io.kroxylicious.proxy.config.VirtualClusterGateway; -import io.kroxylicious.proxy.config.secret.InlinePassword; import io.kroxylicious.proxy.micrometer.CommonTagsHook; import io.kroxylicious.proxy.micrometer.StandardBindersHook; import io.kroxylicious.proxy.service.HostPort; @@ -60,9 +58,9 @@ import io.kroxylicious.test.tester.SimpleMetric; import io.kroxylicious.test.tester.SimpleMetricAssert; import io.kroxylicious.testing.kafka.api.KafkaCluster; +import io.kroxylicious.testing.kafka.api.TerminationStyle; import io.kroxylicious.testing.kafka.common.BrokerCluster; import io.kroxylicious.testing.kafka.common.KeytoolCertificateGenerator; -import io.kroxylicious.testing.kafka.common.Tls; import io.kroxylicious.testing.kafka.junit5ext.KafkaClusterExtension; import io.kroxylicious.testing.kafka.junit5ext.Topic; import io.kroxylicious.testing.kafka.junit5ext.TopicPartitions; @@ -89,7 +87,6 @@ class MetricsIT { private static Path certsDirectory; private static final HostPort PORT_IDENTIFIES_BROKER_BOOTSTRAP = new HostPort("localhost", 9092); - private static final HostPort PROXY_ADDRESS = HostPort.parse("localhost:9192"); private static final String SNI_IDENTIFIES_BROKER_BASE_ADDRESS = IntegrationTestInetAddressResolverProvider.generateFullyQualifiedDomainName("sni"); private static final HostPort SNI_IDENTIFIES_BROKER_BOOTSTRAP = new HostPort("bootstrap." + SNI_IDENTIFIES_BROKER_BASE_ADDRESS, 9192); @@ -98,7 +95,6 @@ class MetricsIT { @BeforeEach void beforeEach() { assertThat(Metrics.globalRegistry.getMeters()).isEmpty(); - } @AfterEach @@ -361,7 +357,6 @@ static Stream connectionMetricsScenarios() { return Stream.of( argumentSet("counts connection from client to proxy for bootstrap", - (UnaryOperator) builder -> builder, (Consumer>) metricList -> assertThat(metricList) .filterByName("kroxylicious_client_to_proxy_connections_total") .filterByTag(NODE_ID_LABEL, "bootstrap") @@ -373,7 +368,6 @@ static Stream connectionMetricsScenarios() { .value() .isGreaterThanOrEqualTo(1.0)), argumentSet("counts connections to node, client to proxy", - (UnaryOperator) builder -> builder, (Consumer>) metricList -> assertThat(metricList) .filterByName("kroxylicious_client_to_proxy_connections_total") .filterByTag(NODE_ID_LABEL, "0") @@ -385,7 +379,6 @@ static Stream connectionMetricsScenarios() { .value() .isGreaterThanOrEqualTo(1.0)), argumentSet("count bootstrap connection from proxy to server ", - (UnaryOperator) builder -> builder, (Consumer>) metricList -> assertThat(metricList) .filterByName("kroxylicious_proxy_to_server_connections_total") .filterByTag(NODE_ID_LABEL, "bootstrap") @@ -397,7 +390,6 @@ static Stream connectionMetricsScenarios() { .value() .isGreaterThanOrEqualTo(1.0)), argumentSet("count node connection from proxy to server ", - (UnaryOperator) builder -> builder, (Consumer>) metricList -> assertThat(metricList) .filterByName("kroxylicious_proxy_to_server_connections_total") .filterByTag(NODE_ID_LABEL, "0") @@ -412,14 +404,13 @@ static Stream connectionMetricsScenarios() { @ParameterizedTest @MethodSource("connectionMetricsScenarios") - void shouldIncrementCountOnConnectionMetrics(UnaryOperator builder, - Consumer> beforeAssertion, + void shouldIncrementCountOnConnectionMetrics(Consumer> beforeAssertion, Consumer> afterAssertion, KafkaCluster cluster) { var config = configWithMetrics(cluster); // Given - try (var tester = kroxyliciousTester(builder.apply(config)); + try (var tester = kroxyliciousTester(config); var managementClient = tester.getManagementClient();) { var metricList = managementClient.scrapeMetrics(); @@ -437,137 +428,6 @@ void shouldIncrementCountOnConnectionMetrics(UnaryOperator } } - @Test - void countMetricsShouldBeDiscriminatedByNodeId(@BrokerCluster(numBrokers = 2) KafkaCluster cluster, @TopicPartitions(2) Topic topic) - throws ExecutionException, InterruptedException { - var config = configWithMetrics(cluster); - - try (var tester = kroxyliciousTester(config); - var managementClient = tester.getManagementClient(); - var producer = tester.producer(); - var admin = tester.admin()) { - - // Given - var describeFuture = admin.describeTopics(List.of(topic.name())).allTopicNames(); - assertThat(describeFuture).succeedsWithin(Duration.ofSeconds(5)); - - var partitions = describeFuture.get().get(topic.name()).partitions(); - var partitionZeroNodeId = getNodeIdForPartition(partitions, 0); - var partitionOneNodeId = getNodeIdForPartition(partitions, 1); - assertThat(partitionZeroNodeId) - .describedAs("expecting topic partitions to be distributed amongst brokers") - .isNotEqualTo(partitionOneNodeId); - - // When - List.of(new ProducerRecord<>(topic.name(), 0, "my-key", "0-1"), - new ProducerRecord<>(topic.name(), 0, "my-key", "0-2"), - new ProducerRecord<>(topic.name(), 1, "my-key", "0-2")) - .forEach(pr -> assertThat(producer.send(pr)).succeedsWithin(Duration.ofSeconds(5))); - - // Then - var metricList = managementClient.scrapeMetrics(); - - assertThat(metricList) - .filterByName("kroxylicious_client_to_proxy_request_total") - .filterByTag(API_KEY_LABEL, ApiKeys.PRODUCE.name()) - .filterByTag(NODE_ID_LABEL, "" + partitionZeroNodeId) - .singleElement() - .value() - .isEqualTo(2.0); - - assertThat(metricList) - .filterByName("kroxylicious_client_to_proxy_request_total") - .filterByTag(API_KEY_LABEL, ApiKeys.PRODUCE.name()) - .filterByTag(NODE_ID_LABEL, "" + partitionOneNodeId) - .singleElement() - .value() - .isEqualTo(1.0); - } - } - - @Test - @Deprecated(since = "0.13.0", forRemoval = true) - void shouldIncrementDownstreamMessagesOnProduceRequestWithoutFilter(KafkaCluster cluster, Topic topic) throws ExecutionException, InterruptedException { - var config = configWithMetrics(cluster); - - // Given - try (var tester = kroxyliciousTester(config); - var managementClient = tester.getManagementClient(); - var producer = tester.producer()) { - var metricList = managementClient.scrapeMetrics(); - var inboundDownstreamMessagesMetricsValue = getMetricsValue(metricList, "kroxylicious_inbound_downstream_messages_total"); - var inboundDownstreamDecodedMessagesMetricsValue = getMetricsValue(metricList, "kroxylicious_inbound_downstream_decoded_messages_total"); - - // When - producer.send(new ProducerRecord<>(topic.name(), "my-key", "hello-world")).get(); - - // Then - // updated metrics after some message were produced - var updatedMetricsList = managementClient.scrapeMetrics(); - var updatedInboundDownstreamMessagesMetricsValue = getMetricsValue(updatedMetricsList, "kroxylicious_inbound_downstream_messages_total"); - var updatedInboundDownstreamDecodedMessagesMetricsValue = getMetricsValue(updatedMetricsList, "kroxylicious_inbound_downstream_decoded_messages_total"); - assertThat(updatedInboundDownstreamMessagesMetricsValue).isGreaterThan(inboundDownstreamMessagesMetricsValue); - assertThat(updatedInboundDownstreamDecodedMessagesMetricsValue).isGreaterThan(inboundDownstreamDecodedMessagesMetricsValue); - } - } - - @Test - @Deprecated(since = "0.13.0", forRemoval = true) - void shouldIncrementDownstreamMessagesOnProduceRequestWithFilter(KafkaCluster cluster, Topic topic) throws ExecutionException, InterruptedException { - - // the downstream messages and decoded messages is not yet differentiated by ApiKey - final UUID configInstance = UUID.randomUUID(); - - NamedFilterDefinition namedFilterDefinition = new NamedFilterDefinitionBuilder("filter", - CreateTopicRequest.class.getName()).withConfig("configInstanceId", configInstance).build(); - - var config = addFilterToConfig(configWithMetrics(cluster), namedFilterDefinition); - - // Given - try (var tester = kroxyliciousTester(config); - var managementClient = tester.getManagementClient(); - var producer = tester.producer()) { - var metricList = managementClient.scrapeMetrics(); - var inboundDownstreamMessagesMetricsValue = getMetricsValue(metricList, "kroxylicious_inbound_downstream_messages_total"); - var inboundDownstreamDecodedMessagesMetricsValue = getMetricsValue(metricList, "kroxylicious_inbound_downstream_decoded_messages_total"); - - // When - producer.send(new ProducerRecord<>(topic.name(), "my-key", "hello-world")).get(); - - // Then - // updated metrics after some message were produced - var updatedMetricsList = managementClient.scrapeMetrics(); - var updatedInboundDownstreamMessagesMetricsValue = getMetricsValue(updatedMetricsList, "kroxylicious_inbound_downstream_messages_total"); - var updatedInboundDownstreamDecodedMessagesMetricsValue = getMetricsValue(updatedMetricsList, "kroxylicious_inbound_downstream_decoded_messages_total"); - assertThat(updatedInboundDownstreamMessagesMetricsValue).isGreaterThan(inboundDownstreamMessagesMetricsValue); - assertThat(updatedInboundDownstreamDecodedMessagesMetricsValue).isGreaterThan(inboundDownstreamDecodedMessagesMetricsValue); - } - } - - @Test - @Deprecated(since = "0.13.0", forRemoval = true) - void shouldIncrementConnectionMetrics(KafkaCluster cluster, Topic topic) throws Exception { - var config = configWithMetrics(cluster); - - // Given - try (var tester = kroxyliciousTester(config); - var managementClient = tester.getManagementClient()) { - var metricsList = managementClient.scrapeMetrics(); - assertMetricsDoesNotExist(metricsList, "kroxylicious_downstream_connections_total", null); - assertMetricsDoesNotExist(metricsList, "kroxylicious_upstream_connections_total", null); - - // When - var producer = tester.producer(); - producer.send(new ProducerRecord<>(topic.name(), "my-key", "hello-world")).get(); - - // Then - // updated metrics after some message were produced - var updatedMetricsList = managementClient.scrapeMetrics(); - assertMetricsWithValue(updatedMetricsList, "kroxylicious_downstream_connections_total", null); - assertMetricsWithValue(updatedMetricsList, "kroxylicious_upstream_connections_total", null); - } - } - static Stream createDownstreamConnectionError() throws Exception { var failQuickDescribeCluster = new DescribeClusterOptions().timeoutMs(2_000); // Note that the KeytoolCertificateGenerator generates key stores that are PKCS12 format. @@ -677,39 +537,64 @@ void shouldIncrementDownstreamErrorOnConnectionError(Supplier(topic.name(), 0, "my-key", "0-1"), + new ProducerRecord<>(topic.name(), 0, "my-key", "0-2"), + new ProducerRecord<>(topic.name(), 1, "my-key", "0-2")) + .forEach(pr -> assertThat(producer.send(pr)).succeedsWithin(Duration.ofSeconds(5))); + + // Then + var metricList = managementClient.scrapeMetrics(); + + assertThat(metricList) + .filterByName("kroxylicious_client_to_proxy_request_total") + .filterByTag(API_KEY_LABEL, ApiKeys.PRODUCE.name()) + .filterByTag(NODE_ID_LABEL, "" + partitionZeroNodeId) + .singleElement() + .value() + .isEqualTo(2.0); + + assertThat(metricList) + .filterByName("kroxylicious_client_to_proxy_request_total") + .filterByTag(API_KEY_LABEL, ApiKeys.PRODUCE.name()) + .filterByTag(NODE_ID_LABEL, "" + partitionOneNodeId) + .singleElement() + .value() + .isEqualTo(1.0); + } + } + + @Test + void shouldIncrementUpstreamErrorOnConnectionError(KafkaCluster cluster) { + var config = configWithMetrics(cluster); // Given - try (var tester = kroxyliciousTester(builder); + try (var tester = kroxyliciousTester(config); var admin = tester.admin("demo"); var managementClient = tester.getManagementClient()) { // when + cluster.stopNodes(u -> true, TerminationStyle.GRACEFUL); assertThat(admin.describeCluster(new DescribeClusterOptions().timeoutMs(2_000)).clusterId()) .failsWithin(Duration.ofSeconds(5)) .withThrowableThat() @@ -735,6 +620,89 @@ void shouldIncrementUpstreamErrorOnConnectionError(@Tls KafkaCluster cluster) { } } + @Test + @Deprecated(since = "0.13.0", forRemoval = true) + void shouldIncrementDownstreamMessagesOnProduceRequestWithoutFilter(KafkaCluster cluster, Topic topic) throws ExecutionException, InterruptedException { + var config = configWithMetrics(cluster); + + // Given + try (var tester = kroxyliciousTester(config); + var managementClient = tester.getManagementClient(); + var producer = tester.producer()) { + var metricList = managementClient.scrapeMetrics(); + var inboundDownstreamMessagesMetricsValue = getMetricsValue(metricList, "kroxylicious_inbound_downstream_messages_total"); + var inboundDownstreamDecodedMessagesMetricsValue = getMetricsValue(metricList, "kroxylicious_inbound_downstream_decoded_messages_total"); + + // When + producer.send(new ProducerRecord<>(topic.name(), "my-key", "hello-world")).get(); + + // Then + // updated metrics after some message were produced + var updatedMetricsList = managementClient.scrapeMetrics(); + var updatedInboundDownstreamMessagesMetricsValue = getMetricsValue(updatedMetricsList, "kroxylicious_inbound_downstream_messages_total"); + var updatedInboundDownstreamDecodedMessagesMetricsValue = getMetricsValue(updatedMetricsList, "kroxylicious_inbound_downstream_decoded_messages_total"); + assertThat(updatedInboundDownstreamMessagesMetricsValue).isGreaterThan(inboundDownstreamMessagesMetricsValue); + assertThat(updatedInboundDownstreamDecodedMessagesMetricsValue).isGreaterThan(inboundDownstreamDecodedMessagesMetricsValue); + } + } + + @Test + @Deprecated(since = "0.13.0", forRemoval = true) + void shouldIncrementDownstreamMessagesOnProduceRequestWithFilter(KafkaCluster cluster, Topic topic) throws ExecutionException, InterruptedException { + + // the downstream messages and decoded messages is not yet differentiated by ApiKey + final UUID configInstance = UUID.randomUUID(); + + NamedFilterDefinition namedFilterDefinition = new NamedFilterDefinitionBuilder("filter", + CreateTopicRequest.class.getName()).withConfig("configInstanceId", configInstance).build(); + + var config = addFilterToConfig(configWithMetrics(cluster), namedFilterDefinition); + + // Given + try (var tester = kroxyliciousTester(config); + var managementClient = tester.getManagementClient(); + var producer = tester.producer()) { + var metricList = managementClient.scrapeMetrics(); + var inboundDownstreamMessagesMetricsValue = getMetricsValue(metricList, "kroxylicious_inbound_downstream_messages_total"); + var inboundDownstreamDecodedMessagesMetricsValue = getMetricsValue(metricList, "kroxylicious_inbound_downstream_decoded_messages_total"); + + // When + producer.send(new ProducerRecord<>(topic.name(), "my-key", "hello-world")).get(); + + // Then + // updated metrics after some message were produced + var updatedMetricsList = managementClient.scrapeMetrics(); + var updatedInboundDownstreamMessagesMetricsValue = getMetricsValue(updatedMetricsList, "kroxylicious_inbound_downstream_messages_total"); + var updatedInboundDownstreamDecodedMessagesMetricsValue = getMetricsValue(updatedMetricsList, "kroxylicious_inbound_downstream_decoded_messages_total"); + assertThat(updatedInboundDownstreamMessagesMetricsValue).isGreaterThan(inboundDownstreamMessagesMetricsValue); + assertThat(updatedInboundDownstreamDecodedMessagesMetricsValue).isGreaterThan(inboundDownstreamDecodedMessagesMetricsValue); + } + } + + @Test + @Deprecated(since = "0.13.0", forRemoval = true) + void shouldIncrementConnectionMetrics(KafkaCluster cluster, Topic topic) throws Exception { + var config = configWithMetrics(cluster); + + // Given + try (var tester = kroxyliciousTester(config); + var managementClient = tester.getManagementClient()) { + var metricsList = managementClient.scrapeMetrics(); + assertMetricsDoesNotExist(metricsList, "kroxylicious_downstream_connections_total", null); + assertMetricsDoesNotExist(metricsList, "kroxylicious_upstream_connections_total", null); + + // When + var producer = tester.producer(); + producer.send(new ProducerRecord<>(topic.name(), "my-key", "hello-world")).get(); + + // Then + // updated metrics after some message were produced + var updatedMetricsList = managementClient.scrapeMetrics(); + assertMetricsWithValue(updatedMetricsList, "kroxylicious_downstream_connections_total", null); + assertMetricsWithValue(updatedMetricsList, "kroxylicious_upstream_connections_total", null); + } + } + @Test void shouldIncrementPayloadSizeBytesMetricsOnProduceRequest(KafkaCluster cluster, Topic topic) throws ExecutionException, InterruptedException { var config = configWithMetrics(cluster); diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java index ed61754381..c347868398 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/KafkaProxyInitializer.java @@ -150,14 +150,12 @@ private void initTlsChannel(SocketChannel ch, ChannelPipeline pipeline, Optional if (t != null) { LOGGER.warn("Exception resolving Virtual Cluster Binding for endpoint {} and sniHostname {}: {}", endpoint, sniHostname, t.getMessage()); promise.setFailure(t); - ch.close(); return null; } var gateway = binding.endpointGateway(); var sslContext = gateway.getDownstreamSslContext(); if (sslContext.isEmpty()) { promise.setFailure(new IllegalStateException("Virtual cluster %s does not provide SSL context".formatted(gateway))); - // close here too? } else { KafkaProxyInitializer.this.addHandlers(ch, binding); @@ -166,7 +164,6 @@ private void initTlsChannel(SocketChannel ch, ChannelPipeline pipeline, Optional } catch (Throwable t1) { promise.setFailure(t1); - // and close here too? } return null; }); diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachine.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachine.java index 87234406f8..d6c9614eba 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachine.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/ProxyChannelStateMachine.java @@ -140,6 +140,8 @@ public class ProxyChannelStateMachine { */ @Deprecated(since = "0.13.0", forRemoval = true) private final Counter upstreamConnectionFailureCounter; + + // New connection metrics private final Counter clientToProxyErrorCounter; private final Counter clientToProxyConnectionCounter; private final Counter proxyToServerConnectionCounter; @@ -147,17 +149,20 @@ public class ProxyChannelStateMachine { @SuppressWarnings("java:S5738") public ProxyChannelStateMachine(String clusterName, @Nullable Integer nodeId) { - List tags = Metrics.tags(Metrics.VIRTUAL_CLUSTER_TAG, clusterName); - downstreamConnectionsCounter = taggedCounter(KROXYLICIOUS_DOWNSTREAM_CONNECTIONS, tags); + // New connection metrics clientToProxyConnectionCounter = KROXYLICIOUS_CLIENT_TO_PROXY_CONNECTION_TOTAL_METER_PROVIDER .create(clusterName, nodeId).withTags(); - downstreamErrorCounter = taggedCounter(KROXYLICIOUS_DOWNSTREAM_ERRORS, tags); clientToProxyErrorCounter = KROXYLICIOUS_CLIENT_TO_PROXY_ERROR_TOTAL_METER_PROVIDER .create(clusterName, nodeId).withTags(); proxyToServerConnectionCounter = KROXYLICIOUS_PROXY_TO_SERVER_CONNECTION_TOTAL_METER_PROVIDER .create(clusterName, nodeId).withTags(); proxyToServerErrorCounter = KROXYLICIOUS_PROXY_TO_SERVER_ERROR_TOTAL_METER_PROVIDER .create(clusterName, nodeId).withTags(); + + // These connections metrics are deprecated and are replaced by the metrics mentioned above + List tags = Metrics.tags(Metrics.VIRTUAL_CLUSTER_TAG, clusterName); + downstreamConnectionsCounter = taggedCounter(KROXYLICIOUS_DOWNSTREAM_CONNECTIONS, tags); + downstreamErrorCounter = taggedCounter(KROXYLICIOUS_DOWNSTREAM_ERRORS, tags); upstreamConnectionsCounter = taggedCounter(KROXYLICIOUS_UPSTREAM_CONNECTIONS, tags); connectionAttemptsCounter = taggedCounter(KROXYLICIOUS_UPSTREAM_CONNECTION_ATTEMPTS, tags); upstreamErrorCounter = taggedCounter(KROXYLICIOUS_UPSTREAM_ERRORS, tags); @@ -441,7 +446,6 @@ void onServerException(Throwable cause) { .log("Exception from the server channel: {}. Increase log level to DEBUG for stacktrace"); if (state instanceof ProxyChannelState.Connecting) { upstreamConnectionFailureCounter.increment(); - proxyToServerConnectionCounter.increment(); } upstreamErrorCounter.increment(); proxyToServerErrorCounter.increment(); @@ -503,7 +507,6 @@ private void toForwarding(Forwarding forwarding) { setState(forwarding); Objects.requireNonNull(frontendHandler).inForwarding(); upstreamConnectionsCounter.increment(); - proxyToServerConnectionCounter.increment(); } /** @@ -623,4 +626,4 @@ private static boolean isMessageApiVersionsRequest(Object msg) { return msg instanceof DecodedRequestFrame && ((DecodedRequestFrame) msg).apiKey() == ApiKeys.API_VERSIONS; } -} +} \ No newline at end of file diff --git a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java index 0b535b755a..874da3a39b 100644 --- a/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java +++ b/kroxylicious-runtime/src/main/java/io/kroxylicious/proxy/internal/util/Metrics.java @@ -280,4 +280,4 @@ public interface ClusterNodeSpecificMetricProviderCreator { MeterProvider create(String virtualCluster, Integer nodeId); } -} +} \ No newline at end of file diff --git a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/util/MetricsTest.java b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/util/MetricsTest.java index 1bc4cc7e12..0ea6f2d058 100644 --- a/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/util/MetricsTest.java +++ b/kroxylicious-runtime/src/test/java/io/kroxylicious/proxy/internal/util/MetricsTest.java @@ -128,5 +128,4 @@ void shouldThrowIfOneOfSeveralTagsNameHasEmptyValue() { assertThatThrownBy(() -> Metrics.tags("TagA", "value1", "TagB", "value2", "TagC", "value3", "TagD", " ")) .isInstanceOf(IllegalArgumentException.class); } - } \ No newline at end of file From b5275ef2a0075ccb917ba333fca7374d503818b4 Mon Sep 17 00:00:00 2001 From: Kroxylicious Robot <153509303+kroxylicious-robot@users.noreply.github.com> Date: Thu, 5 Jun 2025 13:45:43 +0100 Subject: [PATCH 76/83] Update localstack/localstack Docker tag to v4.5.0 (#2274) Signed-off-by: k-wall <18440250+k-wall@users.noreply.github.com> Co-authored-by: k-wall <18440250+k-wall@users.noreply.github.com> Signed-off-by: Ujjwal --- .../kroxylicious/kms/provider/aws/kms/AwsKmsTestKmsFacade.java | 2 +- .../src/main/resources/helm_localstack_overrides.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kroxylicious-kms-provider-aws-kms-test-support/src/main/java/io/kroxylicious/kms/provider/aws/kms/AwsKmsTestKmsFacade.java b/kroxylicious-kms-provider-aws-kms-test-support/src/main/java/io/kroxylicious/kms/provider/aws/kms/AwsKmsTestKmsFacade.java index 75dd69d27c..7b71f08ab2 100644 --- a/kroxylicious-kms-provider-aws-kms-test-support/src/main/java/io/kroxylicious/kms/provider/aws/kms/AwsKmsTestKmsFacade.java +++ b/kroxylicious-kms-provider-aws-kms-test-support/src/main/java/io/kroxylicious/kms/provider/aws/kms/AwsKmsTestKmsFacade.java @@ -18,7 +18,7 @@ public class AwsKmsTestKmsFacade extends AbstractAwsKmsTestKmsFacade { private static final Logger LOG = LoggerFactory.getLogger(AwsKmsTestKmsFacade.class); - private static final DockerImageName LOCALSTACK_IMAGE = DockerImageName.parse("localstack/localstack:4.4.0"); + private static final DockerImageName LOCALSTACK_IMAGE = DockerImageName.parse("localstack/localstack:4.5.0"); private LocalStackContainer localStackContainer; @Override diff --git a/kroxylicious-systemtests/src/main/resources/helm_localstack_overrides.yaml b/kroxylicious-systemtests/src/main/resources/helm_localstack_overrides.yaml index 6660685677..8b992f1172 100644 --- a/kroxylicious-systemtests/src/main/resources/helm_localstack_overrides.yaml +++ b/kroxylicious-systemtests/src/main/resources/helm_localstack_overrides.yaml @@ -6,7 +6,7 @@ # Helm Overrides File for localstack used by the system tests. image: - tag: 4.4.0 + tag: 4.5.0 enableStartupScripts: true startupScriptContent: | #!/bin/bash From ce6be81d0c85df225e2cdcca1c7295f19a57ee6a Mon Sep 17 00:00:00 2001 From: Keith Wall Date: Thu, 5 Jun 2025 16:19:50 +0100 Subject: [PATCH 77/83] Strimzi Quickstart For Examples (#2275) Signed-off-by: Keith Wall Signed-off-by: Ujjwal --- kroxylicious-operator/examples/README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 kroxylicious-operator/examples/README.md diff --git a/kroxylicious-operator/examples/README.md b/kroxylicious-operator/examples/README.md new file mode 100644 index 0000000000..19b0bc07b8 --- /dev/null +++ b/kroxylicious-operator/examples/README.md @@ -0,0 +1,20 @@ +# Examples + +These examples assume a Kafka cluster running the Kubernetes Cluster. + +The [Strimzi Quickstart](https://strimzi.io/quickstarts/) fulfils the requirement: + +## Install Strimzi + +``` +kubectl create ns kafka +kubectl apply -f https://strimzi.io/examples/latest/kafka/kafka-single-node.yaml -n kafka +kubectl wait deployment/strimzi-cluster-operator --for=condition=Available=True --timeout=300s -n kafka +``` + +## Deploy the Quickstart + +``` +kubectl apply -f https://strimzi.io/examples/latest/kafka/kafka-single-node.yaml -n kafka +kubectl wait -n kafka kafka my-cluster --for=condition=Ready=True +``` From 9d91722596de9429c9e8740a53a9f2fe260abbdc Mon Sep 17 00:00:00 2001 From: Robert Young Date: Wed, 4 Jun 2025 17:03:05 +1200 Subject: [PATCH 78/83] Use SNI node identification for TLS ClusterIP ingress For TLS clusterIP ingress we manifest: 1. a bootstrap service named ${virtualClusterName}-${ingressName}, the same naming we use for the service carrying the TCP bootstrap port. 2. a service for each upstream node named like ${virtualClusterName}-${ingressName}-${upstreamNodeId}. 3. we configure the proxy virtual cluster gateway like: sniHostIdentifiesNode: bootstrapAddress: "${virtualClusterName}-${ingressName}.${proxyNamespace}.svc.cluster.local:9191" advertisedBrokerAddressPattern: "${virtualClusterName}-${ingressName}-$(nodeId).${proxyNamespace}.svc.cluster.local:9292" The user must ensure their downstream TLS certificates contain SANs for the bootstrap and each upstream node. Why: Previously we limited the user to a single VKC+ingress using identifying ports due to the risks of using port for identification (during reconciliation, the identity of a port can change, so clients could connect to an unexpected vcluster/node). By changing to SNI as the mechanism, the identity is unambiguous as VKCs/gateways are added and removed, as the hostnames should be stable and uniquely address a gateway bootstrap or node. So we can safely allow numerous TLS clusterIP ingresses to co-exist within a cluster without implementing a safer port allocation strategy for identifying ports. Signed-off-by: Robert Young Signed-off-by: Ujjwal --- .../06.Certificate.server-certificate.yaml | 3 + .../kubernetes/operator/ResourcesUtil.java | 9 + .../VirtualKafkaClusterReconciler.java | 2 +- .../model/networking/NetworkingPlanner.java | 47 ++- ...usterIPClusterIngressNetworkingModel.java} | 27 +- ...lusterIPClusterIngressNetworkingModel.java | 122 +++++++ .../operator/KafkaProxyReconcilerIT.java | 134 +++++++- .../networking/NetworkingPlannerTest.java | 133 ++++++-- ...rIPClusterIngressNetworkingModelTest.java} | 65 ++-- ...erIPClusterIngressNetworkingModelTest.java | 320 ++++++++++++++++++ .../out-ConfigMap-minimal-proxy-config.yaml | 10 +- .../out-Deployment-minimal.yaml | 8 - .../out-Service-one-cluster-ip-0.yaml | 40 +++ .../out-Service-one-cluster-ip-1.yaml | 40 +++ .../out-Service-one-cluster-ip-2.yaml | 40 +++ .../out-Service-one-cluster-ip.yaml | 17 +- .../out-ConfigMap-minimal-proxy-config.yaml | 10 +- .../out-Deployment-minimal.yaml | 10 +- .../out-Service-one-cluster-ip-0.yaml | 40 +++ .../out-Service-one-cluster-ip-1.yaml | 40 +++ .../out-Service-one-cluster-ip-2.yaml | 40 +++ .../out-Service-one-cluster-ip.yaml | 17 +- .../out-ConfigMap-minimal-proxy-config.yaml | 10 +- .../out-Deployment-minimal.yaml | 10 +- .../out-Service-one-cluster-ip-0.yaml | 40 +++ .../out-Service-one-cluster-ip-1.yaml | 40 +++ .../out-Service-one-cluster-ip-2.yaml | 40 +++ .../out-Service-one-cluster-ip.yaml | 17 +- .../out-ConfigMap-minimal-proxy-config.yaml | 10 +- .../out-Deployment-minimal.yaml | 10 +- .../out-Service-one-cluster-ip-0.yaml | 40 +++ .../out-Service-one-cluster-ip-1.yaml | 40 +++ .../out-Service-one-cluster-ip-2.yaml | 40 +++ .../out-Service-one-cluster-ip.yaml | 17 +- .../out-ConfigMap-minimal-proxy-config.yaml | 10 +- .../out-Deployment-minimal.yaml | 10 +- .../out-Service-one-cluster-ip-0.yaml | 40 +++ .../out-Service-one-cluster-ip-1.yaml | 40 +++ .../out-Service-one-cluster-ip-2.yaml | 40 +++ .../out-Service-one-cluster-ip.yaml | 17 +- .../out-ConfigMap-minimal-proxy-config.yaml | 10 +- .../out-Deployment-minimal.yaml | 10 +- .../out-Service-one-cluster-ip-0.yaml | 40 +++ .../out-Service-one-cluster-ip-1.yaml | 40 +++ .../out-Service-one-cluster-ip-2.yaml | 40 +++ .../out-Service-one-cluster-ip.yaml | 17 +- .../out-ConfigMap-minimal-proxy-config.yaml | 10 +- .../out-Deployment-minimal.yaml | 10 +- .../out-Service-one-cluster-ip-0.yaml | 40 +++ .../out-Service-one-cluster-ip-1.yaml | 40 +++ .../out-Service-one-cluster-ip-2.yaml | 40 +++ .../out-Service-one-cluster-ip.yaml | 17 +- 52 files changed, 1657 insertions(+), 302 deletions(-) rename kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/{ClusterIPClusterIngressNetworkingModel.java => TcpClusterIPClusterIngressNetworkingModel.java} (85%) create mode 100644 kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/TlsClusterIPClusterIngressNetworkingModel.java rename kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/{ClusterIPClusterIngressNetworkingModelTest.java => TcpClusterIPClusterIngressNetworkingModelTest.java} (84%) create mode 100644 kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/TlsClusterIPClusterIngressNetworkingModelTest.java create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-one-cluster-ip-0.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-one-cluster-ip-1.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-one-cluster-ip-2.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-Service-one-cluster-ip-0.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-Service-one-cluster-ip-1.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-Service-one-cluster-ip-2.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-Service-one-cluster-ip-0.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-Service-one-cluster-ip-1.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-Service-one-cluster-ip-2.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-Service-one-cluster-ip-0.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-Service-one-cluster-ip-1.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-Service-one-cluster-ip-2.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-Service-one-cluster-ip-0.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-Service-one-cluster-ip-1.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-Service-one-cluster-ip-2.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-Service-one-cluster-ip-0.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-Service-one-cluster-ip-1.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-Service-one-cluster-ip-2.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-Service-one-cluster-ip-0.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-Service-one-cluster-ip-1.yaml create mode 100644 kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-Service-one-cluster-ip-2.yaml diff --git a/kroxylicious-operator/examples/downstream-tls/06.Certificate.server-certificate.yaml b/kroxylicious-operator/examples/downstream-tls/06.Certificate.server-certificate.yaml index 0ce05dd83b..d81301ac55 100644 --- a/kroxylicious-operator/examples/downstream-tls/06.Certificate.server-certificate.yaml +++ b/kroxylicious-operator/examples/downstream-tls/06.Certificate.server-certificate.yaml @@ -19,6 +19,9 @@ spec: size: 4096 dnsNames: - my-cluster-cluster-ip.my-proxy.svc.cluster.local + - my-cluster-cluster-ip-0.my-proxy.svc.cluster.local + - my-cluster-cluster-ip-1.my-proxy.svc.cluster.local + - my-cluster-cluster-ip-2.my-proxy.svc.cluster.local usages: - server auth issuerRef: diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ResourcesUtil.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ResourcesUtil.java index a3213eb0eb..931c768067 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ResourcesUtil.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/ResourcesUtil.java @@ -531,4 +531,13 @@ private static boolean isSupportedFileType(String key) { && !key.endsWith(".p12") && !key.endsWith(".jks"); } + + /** + * @return an address that any pod in the same k8s cluster can use to address the service, regardless of which namespace the pod is in + * @param serviceName service name + * @param namespacedResource resource in the namespace of the Service + */ + public static String crossNamespaceServiceAddress(String serviceName, HasMetadata namespacedResource) { + return serviceName + "." + namespace(namespacedResource) + ".svc.cluster.local"; + } } diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java index 64dcf860eb..9eb34e25d7 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/VirtualKafkaClusterReconciler.java @@ -72,7 +72,7 @@ import static io.kroxylicious.kubernetes.operator.ResourcesUtil.name; import static io.kroxylicious.kubernetes.operator.ResourcesUtil.namespace; import static io.kroxylicious.kubernetes.operator.ResourcesUtil.toLocalRef; -import static io.kroxylicious.kubernetes.operator.model.networking.ClusterIPClusterIngressNetworkingModel.bootstrapServiceName; +import static io.kroxylicious.kubernetes.operator.model.networking.TcpClusterIPClusterIngressNetworkingModel.bootstrapServiceName; /** * Reconciles a {@link VirtualKafkaCluster} by checking whether the resources diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/NetworkingPlanner.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/NetworkingPlanner.java index 1459f5ed9b..4c73264956 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/NetworkingPlanner.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/NetworkingPlanner.java @@ -145,7 +145,15 @@ private static ClusterIngressNetworkingDefinition clusterIngressNetworkingDefini ClusterIP clusterIP = ingress.getSpec().getClusterIP(); LoadBalancer loadBalancer = ingress.getSpec().getLoadBalancer(); if (clusterIP != null) { - return new ClusterIPClusterIngressNetworkingDefinition(ingress, cluster, primary, nodeIdRanges, tls); + switch (clusterIP.getProtocol()) { + case TCP -> { + return new TcpClusterIPClusterIngressNetworkingDefinition(ingress, cluster, primary, nodeIdRanges); + } + case TLS -> { + return new TlsClusterIPClusterIngressNetworkingDefinition(ingress, cluster, primary, nodeIdRanges, tls); + } + default -> throw new IllegalStateException("Unexpected clusterIP protocol: " + clusterIP.getProtocol()); + } } else if (loadBalancer != null) { return new LoadBalancerClusterIngressNetworkingDefinition(ingress, cluster, loadBalancer, tls); @@ -199,12 +207,11 @@ default boolean requiresSharedSniPort() { } } - private record ClusterIPClusterIngressNetworkingDefinition( - KafkaProxyIngress ingress, - VirtualKafkaCluster cluster, - KafkaProxy primary, - List nodeIdRanges, - @Nullable Tls tls) + private record TcpClusterIPClusterIngressNetworkingDefinition( + KafkaProxyIngress ingress, + VirtualKafkaCluster cluster, + KafkaProxy primary, + List nodeIdRanges) implements ClusterIngressNetworkingDefinition { @Override @@ -212,14 +219,36 @@ public ClusterIngressNetworkingModel createNetworkingModel(@Nullable Integer fir @Nullable Integer sharedSniPort) { validateNotNull(firstIdentifyingPort, "firstIdentifyingPort must be non null for ClusterIP ingress"); validateNotNull(lastIdentifyingPort, "lastIdentifyingPort must be non null for ClusterIP ingress"); - return new ClusterIPClusterIngressNetworkingModel(primary, cluster, ingress, nodeIdRanges, tls, firstIdentifyingPort, lastIdentifyingPort); + return new TcpClusterIPClusterIngressNetworkingModel(primary, cluster, ingress, nodeIdRanges, firstIdentifyingPort, lastIdentifyingPort); } @Override public int numIdentifyingPortsRequired() { - return ClusterIPClusterIngressNetworkingModel.numIdentifyingPortsRequired(nodeIdRanges); + return TcpClusterIPClusterIngressNetworkingModel.numIdentifyingPortsRequired(nodeIdRanges); + } + + } + + private record TlsClusterIPClusterIngressNetworkingDefinition( + KafkaProxyIngress ingress, + VirtualKafkaCluster cluster, + KafkaProxy primary, + List nodeIdRanges, + @Nullable Tls tls) + implements ClusterIngressNetworkingDefinition { + + @Override + public ClusterIngressNetworkingModel createNetworkingModel(@Nullable Integer firstIdentifyingPort, @Nullable Integer lastIdentifyingPort, + @Nullable Integer sharedSniPort) { + validateNotNull(sharedSniPort, "sharedSniPort must be non null for TLS ClusterIP ingress"); + validateNotNull(tls, "tls must be non null for TLS ClusterIP ingress"); + return new TlsClusterIPClusterIngressNetworkingModel(primary, cluster, ingress, nodeIdRanges, tls, sharedSniPort); } + @Override + public boolean requiresSharedSniPort() { + return true; + } } private record LoadBalancerClusterIngressNetworkingDefinition( diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/ClusterIPClusterIngressNetworkingModel.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/TcpClusterIPClusterIngressNetworkingModel.java similarity index 85% rename from kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/ClusterIPClusterIngressNetworkingModel.java rename to kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/TcpClusterIPClusterIngressNetworkingModel.java index bdf998feae..3d67f8ae1a 100644 --- a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/ClusterIPClusterIngressNetworkingModel.java +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/TcpClusterIPClusterIngressNetworkingModel.java @@ -32,23 +32,21 @@ import io.kroxylicious.proxy.service.HostPort; import edu.umd.cs.findbugs.annotations.NonNull; -import edu.umd.cs.findbugs.annotations.Nullable; import static io.kroxylicious.kubernetes.operator.Labels.standardLabels; import static io.kroxylicious.kubernetes.operator.ResourcesUtil.name; import static io.kroxylicious.kubernetes.operator.ResourcesUtil.namespace; import static java.lang.Math.toIntExact; -public record ClusterIPClusterIngressNetworkingModel(KafkaProxy proxy, - VirtualKafkaCluster cluster, - KafkaProxyIngress ingress, - List nodeIdRanges, - @Nullable Tls tls, - int firstIdentifyingPort, - int lastIdentifyingPort) +public record TcpClusterIPClusterIngressNetworkingModel(KafkaProxy proxy, + VirtualKafkaCluster cluster, + KafkaProxyIngress ingress, + List nodeIdRanges, + int firstIdentifyingPort, + int lastIdentifyingPort) implements ClusterIngressNetworkingModel { - public ClusterIPClusterIngressNetworkingModel { + public TcpClusterIPClusterIngressNetworkingModel { Objects.requireNonNull(proxy); Objects.requireNonNull(cluster); Objects.requireNonNull(ingress); @@ -127,21 +125,14 @@ public NodeIdentificationStrategy nodeIdentificationStrategy() { return new NamedRange(name, toIntExact(range.getStart()), toIntExact(range.getEnd())); }).toList(); return new PortIdentifiesNodeIdentificationStrategy(new HostPort("localhost", firstIdentifyingPort()), - crossNamespaceServiceAddress(), + ResourcesUtil.crossNamespaceServiceAddress(bootstrapServiceName(), cluster), null, portRanges); } - /** - * @return an address that any pod in the same k8s cluster can use to address the service, regardless of which namespace the pod is in - */ - private String crossNamespaceServiceAddress() { - return bootstrapServiceName() + "." + namespace(cluster) + ".svc.cluster.local"; - } - @Override public Optional downstreamTls() { - return Optional.ofNullable(tls); + return Optional.empty(); } public static int numIdentifyingPortsRequired(List nodeIdRanges) { diff --git a/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/TlsClusterIPClusterIngressNetworkingModel.java b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/TlsClusterIPClusterIngressNetworkingModel.java new file mode 100644 index 0000000000..5d0b5e9e6c --- /dev/null +++ b/kroxylicious-operator/src/main/java/io/kroxylicious/kubernetes/operator/model/networking/TlsClusterIPClusterIngressNetworkingModel.java @@ -0,0 +1,122 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.kubernetes.operator.model.networking; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import io.fabric8.kubernetes.api.model.ContainerPort; +import io.fabric8.kubernetes.api.model.IntOrString; +import io.fabric8.kubernetes.api.model.ObjectMeta; +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.kubernetes.api.model.ServiceBuilder; +import io.fabric8.kubernetes.api.model.ServicePortBuilder; + +import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxy; +import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngress; +import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaCluster; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaservicespec.NodeIdRanges; +import io.kroxylicious.kubernetes.api.v1alpha1.virtualkafkaclusterspec.ingresses.Tls; +import io.kroxylicious.kubernetes.operator.ProxyDeploymentDependentResource; +import io.kroxylicious.kubernetes.operator.ResourcesUtil; +import io.kroxylicious.proxy.config.NodeIdentificationStrategy; +import io.kroxylicious.proxy.config.SniHostIdentifiesNodeIdentificationStrategy; +import io.kroxylicious.proxy.service.HostPort; + +import edu.umd.cs.findbugs.annotations.NonNull; + +import static io.kroxylicious.kubernetes.operator.Labels.standardLabels; +import static io.kroxylicious.kubernetes.operator.ResourcesUtil.crossNamespaceServiceAddress; +import static io.kroxylicious.kubernetes.operator.ResourcesUtil.name; +import static io.kroxylicious.kubernetes.operator.ResourcesUtil.namespace; +import static java.lang.Math.toIntExact; + +public record TlsClusterIPClusterIngressNetworkingModel(KafkaProxy proxy, + VirtualKafkaCluster cluster, + KafkaProxyIngress ingress, + List nodeIdRanges, + Tls tls, + int sharedSniPort) + implements ClusterIngressNetworkingModel { + + public static final int CLIENT_FACING_PORT = 9292; + + public TlsClusterIPClusterIngressNetworkingModel { + Objects.requireNonNull(proxy); + Objects.requireNonNull(cluster); + Objects.requireNonNull(ingress); + Objects.requireNonNull(nodeIdRanges); + Objects.requireNonNull(tls); + if (nodeIdRanges.isEmpty()) { + throw new IllegalArgumentException("nodeIdRanges cannot be empty"); + } + } + + @Override + public Stream services() { + String serviceName = bootstrapServiceName(); + var bootstrapService = createService(serviceName); + var nodeServices = nodeIdRanges.stream() + .flatMapToInt(nodeIdRange -> IntStream.rangeClosed(toIntExact(nodeIdRange.getStart()), toIntExact(nodeIdRange.getEnd()))) + .mapToObj(upstreamNodeId -> createService(bootstrapServiceName() + "-" + upstreamNodeId)); + return Stream.concat(Stream.of(bootstrapService), nodeServices); + } + + private ServiceBuilder createService(String serviceName) { + return new ServiceBuilder() + .withMetadata(serviceMetadata(serviceName)) + .withNewSpec() + .withSelector(ProxyDeploymentDependentResource.podLabels(proxy)) + .withPorts(new ServicePortBuilder().withProtocol("TCP").withPort(CLIENT_FACING_PORT).withTargetPort(new IntOrString(sharedSniPort)).build()) + .endSpec(); + } + + @NonNull + private String bootstrapServiceName() { + // we want to ensure TLS and TCP have the same service name for the service that will be used to bootstrap + // this is because the VKC reconciler loads the Service by name. + return TcpClusterIPClusterIngressNetworkingModel.bootstrapServiceName(cluster, name(ingress)); + } + + ObjectMeta serviceMetadata(String name) { + return new ObjectMetaBuilder() + .withName(name) + .withNamespace(namespace(cluster)) + .addToLabels(standardLabels(proxy)) + .addNewOwnerReferenceLike(ResourcesUtil.newOwnerReferenceTo(proxy)).endOwnerReference() + .addNewOwnerReferenceLike(ResourcesUtil.newOwnerReferenceTo(cluster)).endOwnerReference() + .addNewOwnerReferenceLike(ResourcesUtil.newOwnerReferenceTo(ingress)).endOwnerReference() + .build(); + } + + @Override + public Stream identifyingProxyContainerPorts() { + return Stream.of(); + } + + @Override + public boolean requiresSharedSniContainerPort() { + return true; + } + + @Override + public NodeIdentificationStrategy nodeIdentificationStrategy() { + HostPort bootstrapAddress = new HostPort(crossNamespaceServiceAddress(bootstrapServiceName(), proxy), sharedSniPort); + HostPort advertisedBrokerAddressPattern = new HostPort(crossNamespaceServiceAddress(bootstrapServiceName() + "-$(nodeId)", proxy), CLIENT_FACING_PORT); + return new SniHostIdentifiesNodeIdentificationStrategy(bootstrapAddress.toString(), + advertisedBrokerAddressPattern.toString()); + } + + @Override + public Optional downstreamTls() { + return Optional.of(tls); + } + +} diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/KafkaProxyReconcilerIT.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/KafkaProxyReconcilerIT.java index e5ca5bff08..7a5cc98579 100644 --- a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/KafkaProxyReconcilerIT.java +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/KafkaProxyReconcilerIT.java @@ -76,6 +76,7 @@ import io.kroxylicious.kubernetes.operator.assertj.OperatorAssertions; import io.kroxylicious.kubernetes.operator.assertj.ProxyConfigAssert; import io.kroxylicious.kubernetes.operator.model.networking.LoadBalancerClusterIngressNetworkingModel; +import io.kroxylicious.kubernetes.operator.model.networking.TlsClusterIPClusterIngressNetworkingModel; import io.kroxylicious.proxy.config.ConfigParser; import io.kroxylicious.proxy.config.Configuration; import io.kroxylicious.proxy.config.VirtualCluster; @@ -232,8 +233,119 @@ void virtualClusterWithClusterIpIngressWithTrustAnchor() { updateStatusObservedGeneration(testActor.create(resource)); // Then + int clientFacingPort = TlsClusterIPClusterIngressNetworkingModel.CLIENT_FACING_PORT; + int proxyListenPort = ProxyDeploymentDependentResource.SHARED_SNI_PORT; + assertProxyConfigContents(proxy, Set.of(downstreamTrustAnchorName), Set.of()); + String baseServiceName = name(resource) + "-" + name(ingress); + + String expectedBootstrapHost = baseServiceName + "." + extension.getNamespace() + ".svc.cluster.local"; + String expectedAdvertisedBrokerAddressPattern = baseServiceName + "-$(nodeId)." + extension.getNamespace() + ".svc.cluster.local"; + AWAIT.alias("proxy config - gateway configured for clusterIP SNI ingress").untilAsserted(() -> { + assertProxyConfigInConfigMap(proxy) + .cluster(name(resource)) + .gateway(name(ingress)) + .sniHostIdentifiesNode() + .hasBootstrapAddress(new HostPort(expectedBootstrapHost, proxyListenPort).toString()) + .hasAdvertisedBrokerAddressPattern(new HostPort(expectedAdvertisedBrokerAddressPattern, clientFacingPort).toString()); + }); + + assertDeploymentMountsConfigMap(proxy, downstreamTrustAnchorName); + assertSharedSniPortExposedOnProxyDeployment(proxy, proxyListenPort); + AWAIT.alias("SNI clusterIp services manifested").untilAsserted(() -> { + assertTlsClusterIpServiceManifested(baseServiceName, proxy, clientFacingPort, proxyListenPort); + assertTlsClusterIpServiceManifested(baseServiceName + "-0", proxy, clientFacingPort, proxyListenPort); + assertTlsClusterIpServiceManifested(baseServiceName + "-1", proxy, clientFacingPort, proxyListenPort); + assertTlsClusterIpServiceManifested(baseServiceName + "-2", proxy, clientFacingPort, proxyListenPort); + }); + } + + @Test + void virtualClusterWithMultipleClusterIpIngressWithTrustAnchor() { + // Given + KafkaProxy proxy = testActor.create(kafkaProxy(PROXY_A)); + + KafkaService kafkaService = updateStatusObservedGeneration(testActor.create(kafkaService(CLUSTER_BAR_REF, CLUSTER_BAR_BOOTSTRAP))); + + KafkaProxyIngress ingress = updateStatusObservedGeneration(testActor.create(clusterIpIngress(CLUSTER_BAR_CLUSTERIP_INGRESS, proxy, TLS))); + KafkaProxyIngress ingress2 = updateStatusObservedGeneration(testActor.create(clusterIpIngress("another-cluster-ip", proxy, TLS))); + + Secret tlsServerCert = testActor.create(tlsKeyAndCertSecret("downstream-tls-certificate")); + String downstreamTrustAnchorName = "downstream-tls-trust-anchor"; + ConfigMap trustAnchor = testActor.create(trustAnchorConfigMap(downstreamTrustAnchorName, "tls.pem")); + + Ingresses clusterIngress = new IngressesBuilder() + .withIngressRef(toIngressRef(ingress)) + .withNewTls() + .withCertificateRef(toCertificateRef(tlsServerCert)) + .withTrustAnchorRef(toTrustAnchorRef(trustAnchor)) + .endTls() + .build(); + + Ingresses clusterIngress2 = new IngressesBuilder() + .withIngressRef(toIngressRef(ingress2)) + .withNewTls() + .withCertificateRef(toCertificateRef(tlsServerCert)) + .withTrustAnchorRef(toTrustAnchorRef(trustAnchor)) + .endTls() + .build(); + + VirtualKafkaCluster resource = virtualKafkaCluster(CLUSTER_BAR, proxy, kafkaService, List.of(clusterIngress, clusterIngress2), Optional.empty()); + + // When + updateStatusObservedGeneration(testActor.create(resource)); + + // Then + int clientFacingPort = TlsClusterIPClusterIngressNetworkingModel.CLIENT_FACING_PORT; + int proxyListenPort = ProxyDeploymentDependentResource.SHARED_SNI_PORT; assertProxyConfigContents(proxy, Set.of(downstreamTrustAnchorName), Set.of()); + String baseServiceName = name(resource) + "-" + name(ingress); + String baseServiceName2 = name(resource) + "-" + name(ingress2); + + assertProxyGatewayConfiguredForTlsClusterIP(baseServiceName, proxy, resource, ingress, proxyListenPort, clientFacingPort); + assertProxyGatewayConfiguredForTlsClusterIP(baseServiceName2, proxy, resource, ingress2, proxyListenPort, clientFacingPort); + assertDeploymentMountsConfigMap(proxy, downstreamTrustAnchorName); + assertSharedSniPortExposedOnProxyDeployment(proxy, proxyListenPort); + AWAIT.alias("SNI clusterIp services manifested").untilAsserted(() -> { + assertTlsClusterIpServiceManifested(baseServiceName, proxy, clientFacingPort, proxyListenPort); + assertTlsClusterIpServiceManifested(baseServiceName + "-0", proxy, clientFacingPort, proxyListenPort); + assertTlsClusterIpServiceManifested(baseServiceName + "-1", proxy, clientFacingPort, proxyListenPort); + assertTlsClusterIpServiceManifested(baseServiceName + "-2", proxy, clientFacingPort, proxyListenPort); + assertTlsClusterIpServiceManifested(baseServiceName2, proxy, clientFacingPort, proxyListenPort); + assertTlsClusterIpServiceManifested(baseServiceName2 + "-0", proxy, clientFacingPort, proxyListenPort); + assertTlsClusterIpServiceManifested(baseServiceName2 + "-1", proxy, clientFacingPort, proxyListenPort); + assertTlsClusterIpServiceManifested(baseServiceName2 + "-2", proxy, clientFacingPort, proxyListenPort); + }); + } + + private void assertProxyGatewayConfiguredForTlsClusterIP(String baseServiceName, KafkaProxy proxy, VirtualKafkaCluster resource, KafkaProxyIngress ingress, + int proxyListenPort, int clientFacingPort) { + String expectedBootstrapHost = baseServiceName + "." + extension.getNamespace() + ".svc.cluster.local"; + String expectedAdvertisedBrokerAddressPattern = baseServiceName + "-$(nodeId)." + extension.getNamespace() + ".svc.cluster.local"; + AWAIT.alias("proxy config - gateway configured for clusterIP SNI ingress").untilAsserted(() -> { + assertProxyConfigInConfigMap(proxy) + .cluster(name(resource)) + .gateway(name(ingress)) + .sniHostIdentifiesNode() + .hasBootstrapAddress(new HostPort(expectedBootstrapHost, proxyListenPort).toString()) + .hasAdvertisedBrokerAddressPattern(new HostPort(expectedAdvertisedBrokerAddressPattern, clientFacingPort).toString()); + }); + } + + private void assertTlsClusterIpServiceManifested(String serviceName, KafkaProxy proxy, int clientFacingPort, int proxyListenPort) { + var service = testActor.get(Service.class, serviceName); + assertThat(service).isNotNull() + .describedAs( + "Expect Service '" + serviceName + " to exist") + .extracting(svc -> svc.getSpec().getSelector()) + .describedAs("Service's selector should select proxy pods") + .isEqualTo(ProxyDeploymentDependentResource.podLabels(proxy)); + assertThat(service.getSpec().getType()).isEqualTo("ClusterIP"); + assertThat(service.getSpec().getPorts()).singleElement().satisfies(onlyPort -> { + assertThat(onlyPort.getProtocol()).isEqualTo("TCP"); + assertThat(onlyPort.getPort()).isEqualTo(clientFacingPort); + assertThat(onlyPort.getTargetPort()).isEqualTo(new IntOrString(proxyListenPort)); + }); } @Test @@ -286,6 +398,19 @@ void virtualClusterWithLoadBalancerIngressWithTrustAnchor() { }); }); + assertSharedSniPortExposedOnProxyDeployment(proxy, proxyListenPort); + + AWAIT.alias("proxy config - gateway configured for SNI loadbalancer ingress").untilAsserted(() -> { + assertProxyConfigInConfigMap(proxy) + .cluster(name(cluster)) + .gateway(name(loadBalancerIngress)) + .sniHostIdentifiesNode() + .hasBootstrapAddress(new HostPort(loadbalancerBootstrap, proxyListenPort).toString()) + .hasAdvertisedBrokerAddressPattern(new HostPort(loadbalancerBrokerAddressPattern, clientFacingPort).toString()); + }); + } + + private void assertSharedSniPortExposedOnProxyDeployment(KafkaProxy proxy, int proxyListenPort) { AWAIT.alias("proxy deployment exposes shared sni port").untilAsserted(() -> { var deployment = testActor.get(Deployment.class, ProxyDeploymentDependentResource.deploymentName(proxy)); assertThat(deployment).isNotNull(); @@ -300,15 +425,6 @@ void virtualClusterWithLoadBalancerIngressWithTrustAnchor() { bootstrapContainerPort); }); }); - - AWAIT.alias("proxy config - gateway configured for SNI loadbalancer ingress").untilAsserted(() -> { - assertProxyConfigInConfigMap(proxy) - .cluster(name(cluster)) - .gateway(name(loadBalancerIngress)) - .sniHostIdentifiesNode() - .hasBootstrapAddress(new HostPort(loadbalancerBootstrap, proxyListenPort).toString()) - .hasAdvertisedBrokerAddressPattern(new HostPort(loadbalancerBrokerAddressPattern, clientFacingPort).toString()); - }); } @Test diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/NetworkingPlannerTest.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/NetworkingPlannerTest.java index c88a541d37..9ea93d7047 100644 --- a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/NetworkingPlannerTest.java +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/NetworkingPlannerTest.java @@ -38,6 +38,8 @@ class NetworkingPlannerTest { private static final String PROXY_NAME = "my-proxy"; private static final String CLUSTER_IP_INGRESS_NAME = "my-cluster-ip-ingress"; + private static final String TLS_CLUSTER_IP_INGRESS_NAME = "my-tls-cluster-ip-ingress"; + private static final String TLS_CLUSTER_IP_2_INGRESS_NAME = "my-tls-cluster-ip-ingress-2"; private static final String CLUSTER_IP_2_INGRESS_NAME = "my-cluster-ip-ingress-2"; private static final String LOAD_BALANCER_INGRESS_NAME = "my-load-balancer-ingress"; private static final String LOAD_BALANCER_INGRESS_2_NAME = "my-load-balancer-ingress-2"; @@ -67,6 +69,34 @@ class NetworkingPlannerTest { .build(); // @formatter:on + // @formatter:off + private static final KafkaProxyIngress TLS_CLUSTER_IP_INGRESS = new KafkaProxyIngressBuilder() + .withNewMetadata() + .withName(TLS_CLUSTER_IP_INGRESS_NAME) + .withNamespace(NAMESPACE) + .endMetadata() + .withNewSpec() + .withNewClusterIP() + .withProtocol(ClusterIP.Protocol.TLS) + .endClusterIP() + .endSpec() + .build(); + // @formatter:on + + // @formatter:off + private static final KafkaProxyIngress TLS_CLUSTER_IP_2_INGRESS = new KafkaProxyIngressBuilder() + .withNewMetadata() + .withName(TLS_CLUSTER_IP_2_INGRESS_NAME) + .withNamespace(NAMESPACE) + .endMetadata() + .withNewSpec() + .withNewClusterIP() + .withProtocol(ClusterIP.Protocol.TLS) + .endClusterIP() + .endSpec() + .build(); + // @formatter:on + // @formatter:off private static final KafkaProxyIngress CLUSTER_IP_INGRESS_2 = new KafkaProxyIngressBuilder() .withNewMetadata() @@ -109,6 +139,10 @@ private static KafkaProxyIngress createLoadBalancerIngress(String name, LoadBala // @formatter:on public static final Ingresses CLUSTER_IP_CLUSTER_INGRESSES = new IngressesBuilder().withNewIngressRef().withName(CLUSTER_IP_INGRESS_NAME).endIngressRef().build(); + public static final Ingresses TLS_CLUSTER_IP_CLUSTER_INGRESSES = new IngressesBuilder().withTls(TLS).withNewIngressRef().withName(TLS_CLUSTER_IP_INGRESS_NAME) + .endIngressRef().build(); + public static final Ingresses TLS_CLUSTER_IP_CLUSTER_2_INGRESSES = new IngressesBuilder().withTls(TLS_2).withNewIngressRef().withName(TLS_CLUSTER_IP_2_INGRESS_NAME) + .endIngressRef().build(); public static final Ingresses CLUSTER_IP_2_CLUSTER_INGRESSES = new IngressesBuilder().withNewIngressRef().withName(CLUSTER_IP_2_INGRESS_NAME).endIngressRef().build(); public static final Ingresses LOAD_BALANCER_INGRESSES = new IngressesBuilder().withTls(TLS).withNewIngressRef().withName(LOAD_BALANCER_INGRESS_NAME).endIngressRef() @@ -146,13 +180,13 @@ private static VirtualKafkaCluster clusterWithIngress(String clusterName, Ingres // @formatter:on @Test - public void noClustersResolved() { + void noClustersResolved() { ProxyNetworkingModel networkingModel = NetworkingPlanner.planNetworking(PROXY, new ProxyResolutionResult(Set.of())); assertThat(networkingModel.clusterNetworkingModels()).isEmpty(); } @Test - public void clusterIpIngress() { + void clusterIpIngress() { VirtualKafkaCluster virtualKafkaCluster = clusterWithIngress(CLUSTER_NAME, CLUSTER_IP_CLUSTER_INGRESSES); ClusterResolutionResult clusterResolutionResult = new ClusterResolutionResult(virtualKafkaCluster, ResolutionResult.resolved(virtualKafkaCluster, PROXY), List.of(), ResolutionResult.resolved(virtualKafkaCluster, KAFKA_SERVICE), @@ -166,15 +200,68 @@ public void clusterIpIngress() { assertThat(clusterNetworkingModel.ingressExceptions()).isEmpty(); assertThat(clusterNetworkingModel.clusterIngressNetworkingModelResults()).singleElement().satisfies(model -> { assertThat(model.exception()).isNull(); - assertThat(model.clusterIngressNetworkingModel()).isEqualTo(new ClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster, CLUSTER_IP_INGRESS, - List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), null, FIRST_IDENTIFYING_PORT, + assertThat(model.clusterIngressNetworkingModel()).isEqualTo(new TcpClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster, CLUSTER_IP_INGRESS, + List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), FIRST_IDENTIFYING_PORT, FIRST_IDENTIFYING_PORT + 3)); }); }); } @Test - public void loadBalancerIngress() { + void tlsClusterIpIngress() { + VirtualKafkaCluster virtualKafkaCluster = clusterWithIngress(CLUSTER_NAME, TLS_CLUSTER_IP_CLUSTER_INGRESSES); + ClusterResolutionResult clusterResolutionResult = new ClusterResolutionResult(virtualKafkaCluster, ResolutionResult.resolved(virtualKafkaCluster, PROXY), + List.of(), ResolutionResult.resolved(virtualKafkaCluster, KAFKA_SERVICE), + List.of(new IngressResolutionResult(ResolutionResult.resolved(virtualKafkaCluster, TLS_CLUSTER_IP_INGRESS), + ResolutionResult.resolved(TLS_CLUSTER_IP_INGRESS, PROXY), TLS_CLUSTER_IP_CLUSTER_INGRESSES))); + ProxyNetworkingModel networkingModel = NetworkingPlanner.planNetworking(PROXY, new ProxyResolutionResult(Set.of(clusterResolutionResult))); + assertThat(networkingModel.clusterIngressModel(virtualKafkaCluster)).isNotNull(); + + assertThat(networkingModel.clusterNetworkingModels()).singleElement().satisfies(clusterNetworkingModel -> { + assertThat(clusterNetworkingModel.cluster()).isEqualTo(virtualKafkaCluster); + assertThat(clusterNetworkingModel.ingressExceptions()).isEmpty(); + assertThat(clusterNetworkingModel.clusterIngressNetworkingModelResults()).singleElement().satisfies(model -> { + assertThat(model.exception()).isNull(); + assertThat(model.clusterIngressNetworkingModel()) + .isEqualTo(new TlsClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster, TLS_CLUSTER_IP_INGRESS, + List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), TLS, 9291)); + }); + }); + } + + @Test + void mulipleTlsClusterIpIngress() { + VirtualKafkaCluster virtualKafkaCluster = clusterWithIngress(CLUSTER_NAME, TLS_CLUSTER_IP_CLUSTER_INGRESSES, TLS_CLUSTER_IP_CLUSTER_2_INGRESSES); + ClusterResolutionResult clusterResolutionResult = new ClusterResolutionResult(virtualKafkaCluster, ResolutionResult.resolved(virtualKafkaCluster, PROXY), + List.of(), ResolutionResult.resolved(virtualKafkaCluster, KAFKA_SERVICE), + List.of(new IngressResolutionResult(ResolutionResult.resolved(virtualKafkaCluster, TLS_CLUSTER_IP_INGRESS), + ResolutionResult.resolved(TLS_CLUSTER_IP_INGRESS, PROXY), TLS_CLUSTER_IP_CLUSTER_INGRESSES), + new IngressResolutionResult(ResolutionResult.resolved(virtualKafkaCluster, TLS_CLUSTER_IP_2_INGRESS), + ResolutionResult.resolved(TLS_CLUSTER_IP_2_INGRESS, PROXY), TLS_CLUSTER_IP_CLUSTER_2_INGRESSES))); + ProxyNetworkingModel networkingModel = NetworkingPlanner.planNetworking(PROXY, new ProxyResolutionResult(Set.of(clusterResolutionResult))); + assertThat(networkingModel.clusterIngressModel(virtualKafkaCluster)).isNotNull(); + + assertThat(networkingModel.clusterNetworkingModels()).singleElement().satisfies(clusterNetworkingModel -> { + assertThat(clusterNetworkingModel.cluster()).isEqualTo(virtualKafkaCluster); + assertThat(clusterNetworkingModel.ingressExceptions()).isEmpty(); + var listAssert = assertThat(clusterNetworkingModel.clusterIngressNetworkingModelResults()).hasSize(2); + listAssert.element(0).satisfies(model -> { + assertThat(model.exception()).isNull(); + assertThat(model.clusterIngressNetworkingModel()) + .isEqualTo(new TlsClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster, TLS_CLUSTER_IP_INGRESS, + List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), TLS, 9291)); + }); + listAssert.element(1).satisfies(model -> { + assertThat(model.exception()).isNull(); + assertThat(model.clusterIngressNetworkingModel()) + .isEqualTo(new TlsClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster, TLS_CLUSTER_IP_2_INGRESS, + List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), TLS_2, 9291)); + }); + }); + } + + @Test + void loadBalancerIngress() { VirtualKafkaCluster virtualKafkaCluster = clusterWithIngress(CLUSTER_NAME, LOAD_BALANCER_INGRESSES); ClusterResolutionResult clusterResolutionResult = new ClusterResolutionResult(virtualKafkaCluster, ResolutionResult.resolved(virtualKafkaCluster, PROXY), List.of(), ResolutionResult.resolved(virtualKafkaCluster, KAFKA_SERVICE), @@ -195,7 +282,7 @@ public void loadBalancerIngress() { } @Test - public void clusterIpAndLoadBalancerIngress() { + void clusterIpAndLoadBalancerIngress() { VirtualKafkaCluster virtualKafkaCluster = clusterWithIngress(CLUSTER_NAME, CLUSTER_IP_CLUSTER_INGRESSES, LOAD_BALANCER_INGRESSES); ClusterResolutionResult clusterResolutionResult = new ClusterResolutionResult(virtualKafkaCluster, ResolutionResult.resolved(virtualKafkaCluster, PROXY), List.of(), ResolutionResult.resolved(virtualKafkaCluster, KAFKA_SERVICE), @@ -212,8 +299,8 @@ public void clusterIpAndLoadBalancerIngress() { var listAssert = assertThat(clusterNetworkingModel.clusterIngressNetworkingModelResults()).hasSize(2); listAssert.element(0).satisfies(model -> { assertThat(model.exception()).isNull(); - assertThat(model.clusterIngressNetworkingModel()).isEqualTo(new ClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster, CLUSTER_IP_INGRESS, - List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), null, FIRST_IDENTIFYING_PORT, + assertThat(model.clusterIngressNetworkingModel()).isEqualTo(new TcpClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster, CLUSTER_IP_INGRESS, + List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), FIRST_IDENTIFYING_PORT, FIRST_IDENTIFYING_PORT + 3)); }); listAssert.element(1).satisfies(model -> { @@ -226,7 +313,7 @@ public void clusterIpAndLoadBalancerIngress() { } @Test - public void multipleLoadBalancerIngress() { + void multipleLoadBalancerIngress() { VirtualKafkaCluster virtualKafkaCluster = clusterWithIngress(CLUSTER_NAME, LOAD_BALANCER_INGRESSES, LOAD_BALANCER_2_INGRESSES); ClusterResolutionResult clusterResolutionResult = new ClusterResolutionResult(virtualKafkaCluster, ResolutionResult.resolved(virtualKafkaCluster, PROXY), List.of(), ResolutionResult.resolved(virtualKafkaCluster, KAFKA_SERVICE), @@ -255,7 +342,7 @@ public void multipleLoadBalancerIngress() { } @Test - public void multipleClustersWithLoadBalancerIngress() { + void multipleClustersWithLoadBalancerIngress() { VirtualKafkaCluster virtualKafkaCluster = clusterWithIngress(CLUSTER_NAME, LOAD_BALANCER_INGRESSES); VirtualKafkaCluster virtualKafkaCluster2 = clusterWithIngress(CLUSTER_NAME_2, LOAD_BALANCER_2_INGRESSES); ClusterResolutionResult clusterResolutionResult = new ClusterResolutionResult(virtualKafkaCluster, ResolutionResult.resolved(virtualKafkaCluster, PROXY), @@ -294,7 +381,7 @@ public void multipleClustersWithLoadBalancerIngress() { } @Test - public void multipleClusterIpIngresses() { + void multipleClusterIpIngresses() { VirtualKafkaCluster virtualKafkaCluster = clusterWithIngress(CLUSTER_NAME, CLUSTER_IP_CLUSTER_INGRESSES, CLUSTER_IP_2_CLUSTER_INGRESSES); ClusterResolutionResult clusterResolutionResult = new ClusterResolutionResult(virtualKafkaCluster, ResolutionResult.resolved(virtualKafkaCluster, PROXY), List.of(), ResolutionResult.resolved(virtualKafkaCluster, KAFKA_SERVICE), @@ -314,8 +401,8 @@ public void multipleClusterIpIngresses() { var listAssert = assertThat(clusterNetworkingModel.clusterIngressNetworkingModelResults()).hasSize(2); listAssert.element(0).satisfies(model -> { assertThat(model.exception()).isNull(); - assertThat(model.clusterIngressNetworkingModel()).isEqualTo(new ClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster, CLUSTER_IP_INGRESS, - List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), null, FIRST_IDENTIFYING_PORT, + assertThat(model.clusterIngressNetworkingModel()).isEqualTo(new TcpClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster, CLUSTER_IP_INGRESS, + List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), FIRST_IDENTIFYING_PORT, FIRST_IDENTIFYING_PORT + 3)); }); listAssert.element(1).satisfies(model -> { @@ -323,15 +410,16 @@ public void multipleClusterIpIngresses() { "Currently we do not support a virtual cluster with multiple ingresses that need unique ports to" + " identify which node the client is connecting to"); // ports are still allocated to try and keep the model stable - assertThat(model.clusterIngressNetworkingModel()).isEqualTo(new ClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster, CLUSTER_IP_INGRESS_2, - List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), null, FIRST_IDENTIFYING_PORT + 4, - FIRST_IDENTIFYING_PORT + 7)); + assertThat(model.clusterIngressNetworkingModel()) + .isEqualTo(new TcpClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster, CLUSTER_IP_INGRESS_2, + List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), FIRST_IDENTIFYING_PORT + 4, + FIRST_IDENTIFYING_PORT + 7)); }); }); } @Test - public void multipleClustersWithClusterIpIngress() { + void multipleClustersWithClusterIpIngress() { VirtualKafkaCluster virtualKafkaCluster = clusterWithIngress(CLUSTER_NAME, CLUSTER_IP_CLUSTER_INGRESSES); ClusterResolutionResult clusterResolutionResult = new ClusterResolutionResult(virtualKafkaCluster, ResolutionResult.resolved(virtualKafkaCluster, PROXY), List.of(), ResolutionResult.resolved(virtualKafkaCluster, KAFKA_SERVICE), @@ -354,8 +442,8 @@ public void multipleClustersWithClusterIpIngress() { assertThat(clusterNetworkingModel.ingressExceptions()).isEmpty(); assertThat(clusterNetworkingModel.clusterIngressNetworkingModelResults()).singleElement().satisfies(model -> { assertThat(model.exception()).isNull(); - assertThat(model.clusterIngressNetworkingModel()).isEqualTo(new ClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster, CLUSTER_IP_INGRESS, - List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), null, FIRST_IDENTIFYING_PORT, + assertThat(model.clusterIngressNetworkingModel()).isEqualTo(new TcpClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster, CLUSTER_IP_INGRESS, + List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), FIRST_IDENTIFYING_PORT, FIRST_IDENTIFYING_PORT + 3)); }); @@ -371,9 +459,10 @@ public void multipleClustersWithClusterIpIngress() { assertThat(model.exception()).isNotNull().hasMessageContaining( "Currently we do not support a virtual cluster with multiple ingresses that need unique ports to" + " identify which node the client is connecting to"); - assertThat(model.clusterIngressNetworkingModel()).isEqualTo(new ClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster2, CLUSTER_IP_INGRESS_2, - List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), null, FIRST_IDENTIFYING_PORT + 4, - FIRST_IDENTIFYING_PORT + 7)); + assertThat(model.clusterIngressNetworkingModel()) + .isEqualTo(new TcpClusterIPClusterIngressNetworkingModel(PROXY, virtualKafkaCluster2, CLUSTER_IP_INGRESS_2, + List.of(new NodeIdRangesBuilder().withName("default").withStart(0L).withEnd(2L).build()), FIRST_IDENTIFYING_PORT + 4, + FIRST_IDENTIFYING_PORT + 7)); }); }); diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/ClusterIPClusterIngressNetworkingModelTest.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/TcpClusterIPClusterIngressNetworkingModelTest.java similarity index 84% rename from kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/ClusterIPClusterIngressNetworkingModelTest.java rename to kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/TcpClusterIPClusterIngressNetworkingModelTest.java index fdf338d791..b08e772430 100644 --- a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/ClusterIPClusterIngressNetworkingModelTest.java +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/TcpClusterIPClusterIngressNetworkingModelTest.java @@ -46,7 +46,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.params.provider.Arguments.argumentSet; -class ClusterIPClusterIngressNetworkingModelTest { +class TcpClusterIPClusterIngressNetworkingModelTest { private static final String INGRESS_NAME = "my-ingress"; public static final String CLUSTER_NAME = "my-cluster"; @@ -90,7 +90,7 @@ class ClusterIPClusterIngressNetworkingModelTest { @Test void numIdentifyingPortsRequiredSingleRange() { // 3 ports for the nodeId range + 1 for bootstrap - int numIdentifyingPortsRequired = ClusterIPClusterIngressNetworkingModel.numIdentifyingPortsRequired(List.of(NODE_ID_RANGE)); + int numIdentifyingPortsRequired = TcpClusterIPClusterIngressNetworkingModel.numIdentifyingPortsRequired(List.of(NODE_ID_RANGE)); assertThat(numIdentifyingPortsRequired).isEqualTo(4); } @@ -99,8 +99,8 @@ void createInstancesWithExpectedNumber() { // given List nodeIdRange = List.of(createNodeIdRange("a", 1L, 3L)); // when - ClusterIPClusterIngressNetworkingModel model = new ClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, - nodeIdRange, null, 1, 4); + TcpClusterIPClusterIngressNetworkingModel model = new TcpClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, + nodeIdRange, 1, 4); // then assertThat(model).isNotNull(); } @@ -112,8 +112,8 @@ void createInstancesWithTooManyPorts() { // when // then // more ports than the 4 required by the node id range + 1 for bootstrap - assertThatThrownBy(() -> new ClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, - nodeIdRange, null, 1, 5)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new TcpClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, + nodeIdRange, 1, 5)).isInstanceOf(IllegalArgumentException.class); } @Test @@ -124,8 +124,8 @@ void createInstancesWithNotEnoughPorts() { // when // then // more ports than the 4 required by the node id range + 1 for bootstrap - assertThatThrownBy(() -> new ClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, - nodeIdRange, null, 1, 3)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new TcpClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, + nodeIdRange, 1, 3)).isInstanceOf(IllegalArgumentException.class); } @Test @@ -136,8 +136,8 @@ void gatewayConfigSingleRange() { int rangeEnd = 3; List nodeIdRange = List.of(createNodeIdRange(rangeName, rangeStart, rangeEnd)); // 3 ports for the nodeId range + 1 for bootstrap - ClusterIngressNetworkingModel instance = new ClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, - nodeIdRange, null, 1, 4); + ClusterIngressNetworkingModel instance = new TcpClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, + nodeIdRange, 1, 4); assertThat(instance).isNotNull(); // when @@ -162,9 +162,8 @@ void gatewayConfigSingleRange() { void serviceMetadata() { // given // 3 ports for the nodeId range + 1 for bootstrap - ClusterIngressNetworkingModel instance = new ClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, - List.of(createNodeIdRange("a", 1, 3)), null, 1, 4); - assertThat(instance).isNotNull(); + ClusterIngressNetworkingModel instance = new TcpClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, + List.of(createNodeIdRange("a", 1, 3)), 1, 4); assertThat(instance).isNotNull(); // when @@ -203,8 +202,8 @@ void serviceMetadata() { @Test void serviceCommonSpec() { // given - ClusterIngressNetworkingModel instance = new ClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, - List.of(createNodeIdRange("a", 1, 3)), null, 1, 4); + ClusterIngressNetworkingModel instance = new TcpClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, + List.of(createNodeIdRange("a", 1, 3)), 1, 4); // when List serviceBuilders = instance.services().toList(); @@ -225,8 +224,8 @@ void serviceCommonSpec() { void servicesSingleRangePorts() { // given // 3 ports for the nodeId range + 1 for bootstrap - ClusterIngressNetworkingModel instance = new ClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, - List.of(createNodeIdRange("a", 1, 3)), null, 1, 4); + ClusterIngressNetworkingModel instance = new TcpClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, + List.of(createNodeIdRange("a", 1, 3)), 1, 4); assertThat(instance).isNotNull(); @@ -253,8 +252,8 @@ void servicesSingleRangePorts() { void servicesMultipleRangesPorts() { // given // 2 ports for the nodeId ranges + 1 for bootstrap - ClusterIngressNetworkingModel instance = new ClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, - List.of(createNodeIdRange("a", 1, 1), createNodeIdRange("b", 3, 3)), null, 1, 3); + ClusterIngressNetworkingModel instance = new TcpClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, + List.of(createNodeIdRange("a", 1, 1), createNodeIdRange("b", 3, 3)), 1, 3); assertThat(instance).isNotNull(); @@ -287,8 +286,8 @@ void gatewayConfigNameDefault() { List nodeIdRange = List.of(createNodeIdRange(null, rangeStart, rangeEnd), createNodeIdRange(null, rangeStart2, rangeEnd2)); // 5 ports for the nodeId ranges + 1 for bootstrap - ClusterIngressNetworkingModel instance = new ClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, - nodeIdRange, null, 2, 7); + ClusterIngressNetworkingModel instance = new TcpClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, + nodeIdRange, 2, 7); assertThat(instance).isNotNull(); @@ -322,8 +321,8 @@ void gatewayConfigMultipleRange() { int rangeStart2 = 5; int rangeEnd2 = 6; // 5 ports for the nodeId ranges + 1 for bootstrap - ClusterIngressNetworkingModel instance = new ClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, - List.of(createNodeIdRange(rangeName, rangeStart, rangeEnd), createNodeIdRange(rangeName2, rangeStart2, rangeEnd2)), null, 2, 7); + ClusterIngressNetworkingModel instance = new TcpClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, + List.of(createNodeIdRange(rangeName, rangeStart, rangeEnd), createNodeIdRange(rangeName2, rangeStart2, rangeEnd2)), 2, 7); assertThat(instance).isNotNull(); @@ -349,20 +348,20 @@ void gatewayConfigMultipleRange() { @Test void numIdentifyingPortsRequiredMultipleRanges() { // 5 ports for the nodeId ranges + 1 for bootstrap - assertThat(ClusterIPClusterIngressNetworkingModel.numIdentifyingPortsRequired(List.of(NODE_ID_RANGE, createNodeIdRange("a", 5L, 6L)))).isEqualTo(6); + assertThat(TcpClusterIPClusterIngressNetworkingModel.numIdentifyingPortsRequired(List.of(NODE_ID_RANGE, createNodeIdRange("a", 5L, 6L)))).isEqualTo(6); } @Test void rangesMustBeNonEmpty() { List nodeIdRanges = List.of(); - assertThatThrownBy(() -> new ClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, nodeIdRanges, null, 1, 3)) + assertThatThrownBy(() -> new TcpClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, nodeIdRanges, 1, 3)) .isInstanceOf(IllegalArgumentException.class); } @Test void identifyingProxyContainerPortsSingleRange() { - ClusterIngressNetworkingModel instance = new ClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, - List.of(createNodeIdRange("a", 1, 3)), null, 1, 4); + ClusterIngressNetworkingModel instance = new TcpClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, + List.of(createNodeIdRange("a", 1, 3)), 1, 4); assertThat(instance).isNotNull(); assertThat(instance.identifyingProxyContainerPorts()) @@ -374,8 +373,8 @@ void identifyingProxyContainerPortsSingleRange() { @Test void identifyingProxyContainerPortsMultipleRanges() { - ClusterIngressNetworkingModel instance = new ClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, - List.of(createNodeIdRange("a", 1L, 1L), createNodeIdRange("b", 3L, 4L)), null, 1, 4); + ClusterIngressNetworkingModel instance = new TcpClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, + List.of(createNodeIdRange("a", 1L, 1L), createNodeIdRange("b", 3L, 4L)), 1, 4); assertThat(instance).isNotNull(); assertThat(instance.identifyingProxyContainerPorts()) @@ -387,10 +386,10 @@ void identifyingProxyContainerPortsMultipleRanges() { public static Stream constructorArgsMustBeNonNull() { List nodeIdRanges = List.of(NODE_ID_RANGE); - ThrowingCallable nullIngress = () -> new ClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, null, nodeIdRanges, null, 1, 4); - ThrowingCallable nullVirtualCluster = () -> new ClusterIPClusterIngressNetworkingModel(PROXY, null, INGRESS, nodeIdRanges, null, 1, 4); - ThrowingCallable nullProxy = () -> new ClusterIPClusterIngressNetworkingModel(null, VIRTUAL_KAFKA_CLUSTER, INGRESS, nodeIdRanges, null, 1, 4); - ThrowingCallable nullRanges = () -> new ClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, null, null, 1, 4); + ThrowingCallable nullIngress = () -> new TcpClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, null, nodeIdRanges, 1, 4); + ThrowingCallable nullVirtualCluster = () -> new TcpClusterIPClusterIngressNetworkingModel(PROXY, null, INGRESS, nodeIdRanges, 1, 4); + ThrowingCallable nullProxy = () -> new TcpClusterIPClusterIngressNetworkingModel(null, VIRTUAL_KAFKA_CLUSTER, INGRESS, nodeIdRanges, 1, 4); + ThrowingCallable nullRanges = () -> new TcpClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, null, 1, 4); return Stream.of(argumentSet("ingress", nullIngress), argumentSet("virtualCluster", nullVirtualCluster), argumentSet("proxy", nullProxy), diff --git a/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/TlsClusterIPClusterIngressNetworkingModelTest.java b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/TlsClusterIPClusterIngressNetworkingModelTest.java new file mode 100644 index 0000000000..7db7a77713 --- /dev/null +++ b/kroxylicious-operator/src/test/java/io/kroxylicious/kubernetes/operator/model/networking/TlsClusterIPClusterIngressNetworkingModelTest.java @@ -0,0 +1,320 @@ +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.kubernetes.operator.model.networking; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import io.fabric8.kubernetes.api.model.IntOrString; +import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.ServiceBuilder; +import io.fabric8.kubernetes.api.model.ServicePortBuilder; + +import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxy; +import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyBuilder; +import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngress; +import io.kroxylicious.kubernetes.api.v1alpha1.KafkaProxyIngressBuilder; +import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaCluster; +import io.kroxylicious.kubernetes.api.v1alpha1.VirtualKafkaClusterBuilder; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaproxyingressspec.ClusterIP; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaservicespec.NodeIdRanges; +import io.kroxylicious.kubernetes.api.v1alpha1.kafkaservicespec.NodeIdRangesBuilder; +import io.kroxylicious.kubernetes.api.v1alpha1.virtualkafkaclusterspec.ingresses.Tls; +import io.kroxylicious.kubernetes.api.v1alpha1.virtualkafkaclusterspec.ingresses.TlsBuilder; +import io.kroxylicious.kubernetes.operator.KafkaProxyReconciler; +import io.kroxylicious.proxy.config.VirtualClusterGateway; +import io.kroxylicious.proxy.service.HostPort; + +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.params.provider.Arguments.argumentSet; + +class TlsClusterIPClusterIngressNetworkingModelTest { + + private static final String INGRESS_NAME = "my-ingress"; + public static final String CLUSTER_NAME = "my-cluster"; + public static final String PROXY_NAME = "my-proxy"; + public static final String NAMESPACE = "my-namespace"; + + // @formatter:off + private static final KafkaProxyIngress INGRESS = new KafkaProxyIngressBuilder() + .withNewMetadata() + .withName(INGRESS_NAME) + .withNamespace(NAMESPACE) + .endMetadata() + .withNewSpec() + .withNewClusterIP() + .withProtocol(ClusterIP.Protocol.TLS) + .endClusterIP() + .endSpec() + .build(); + // @formatter:on + + private static final Tls TLS = new TlsBuilder().withNewCertificateRef().withName("server-cert").endCertificateRef().build(); + + // @formatter:off + private static final VirtualKafkaCluster VIRTUAL_KAFKA_CLUSTER = new VirtualKafkaClusterBuilder() + .withNewMetadata() + .withName(CLUSTER_NAME) + .withNamespace(NAMESPACE) + .endMetadata() + .withNewSpec() + .addNewIngress() + .withNewIngressRef() + .withName(INGRESS_NAME) + .endIngressRef() + .withTls(TLS) + .endIngress() + .endSpec() + .build(); + // @formatter:on + public static final String PROXY_UID = "my-proxy-uid"; + public static final KafkaProxy PROXY = new KafkaProxyBuilder().withNewMetadata().withName(PROXY_NAME).withUid(PROXY_UID).withNamespace(NAMESPACE).endMetadata() + .build(); + public static final NodeIdRanges NODE_ID_RANGE = createNodeIdRange("a", 1L, 3L); + + @Test + void createInstancesWithSharedSniPort() { + // given + List nodeIdRange = List.of(createNodeIdRange("a", 1L, 3L)); + // when + TlsClusterIPClusterIngressNetworkingModel model = new TlsClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, + nodeIdRange, TLS, 1); + // then + assertThat(model).isNotNull(); + } + + @Test + void gatewayConfig() { + // given + String rangeName = "a"; + int rangeStart = 1; + int rangeEnd = 3; + List nodeIdRange = List.of(createNodeIdRange(rangeName, rangeStart, rangeEnd)); + + int sharedSniPort = 5; + ClusterIngressNetworkingModel instance = new TlsClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, + nodeIdRange, TLS, sharedSniPort); + assertThat(instance).isNotNull(); + + // when + VirtualClusterGateway gateway = KafkaProxyReconciler.buildVirtualClusterGateway(instance).fragment(); + + // then + assertThat(gateway).isNotNull(); + assertThat(gateway.name()).isEqualTo(INGRESS_NAME); + assertThat(gateway.tls()).isNotNull(); + assertThat(gateway.sniHostIdentifiesNode()).isNotNull().satisfies(sni -> { + assertThat(sni.bootstrapAddress()).isEqualTo(new HostPort("my-cluster-my-ingress." + NAMESPACE + ".svc.cluster.local", sharedSniPort).toString()); + assertThat(sni.advertisedBrokerAddressPattern()).isEqualTo("my-cluster-my-ingress-$(nodeId)." + NAMESPACE + ".svc.cluster.local:" + 9292); + }); + } + + @Test + void serviceMetadata() { + // given + ClusterIngressNetworkingModel instance = new TlsClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, + List.of(createNodeIdRange("a", 1, 3)), TLS, 5); + assertThat(instance).isNotNull(); + + // when + List serviceBuilders = instance.services().toList(); + + // then + assertThat(serviceBuilders).isNotNull().isNotEmpty().allSatisfy(serviceBuild -> { + Service build = serviceBuild.build(); + Map orderedServiceLabels = commonLabels(PROXY_NAME); + assertThat(build.getMetadata()).isNotNull().satisfies(metadata -> { + assertThat(metadata.getNamespace()).isEqualTo(NAMESPACE); + assertThat(metadata.getLabels()).containsExactlyEntriesOf(orderedServiceLabels); + assertThat(metadata.getOwnerReferences()) + .satisfiesExactlyInAnyOrder( + ownerRef -> { + assertThat(ownerRef.getKind()).isEqualTo("KafkaProxy"); + assertThat(ownerRef.getApiVersion()).isEqualTo("kroxylicious.io/v1alpha1"); + assertThat(ownerRef.getName()).isEqualTo(PROXY_NAME); + assertThat(ownerRef.getUid()).isEqualTo(PROXY_UID); + }, + ownerRef -> { + assertThat(ownerRef.getKind()).isEqualTo("VirtualKafkaCluster"); + assertThat(ownerRef.getApiVersion()).isEqualTo("kroxylicious.io/v1alpha1"); + assertThat(ownerRef.getName()).isEqualTo(CLUSTER_NAME); + }, + ownerRef -> { + assertThat(ownerRef.getKind()).isEqualTo("KafkaProxyIngress"); + assertThat(ownerRef.getApiVersion()).isEqualTo("kroxylicious.io/v1alpha1"); + assertThat(ownerRef.getName()).isEqualTo(INGRESS_NAME); + }); + }); + }); + } + + @Test + void serviceCommonSpec() { + // given + ClusterIngressNetworkingModel instance = new TlsClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, + List.of(createNodeIdRange("a", 1, 3)), TLS, 5); + + // when + List serviceBuilders = instance.services().toList(); + + // then + assertThat(serviceBuilders).isNotNull().isNotEmpty().allSatisfy(serviceBuild -> { + Service build = serviceBuild.build(); + assertThat(build.getSpec()).isNotNull().satisfies(serviceSpec -> { + LinkedHashMap orderedSelectorLabels = new LinkedHashMap<>(); + orderedSelectorLabels.put("app", "kroxylicious"); + orderedSelectorLabels.putAll(commonLabels(PROXY_NAME)); + assertThat(serviceSpec.getSelector()).containsExactlyEntriesOf(orderedSelectorLabels); + }); + }); + } + + @Test + void servicesSingleRangeOfNodeIds() { + // given + int sharedSniPort = 5; + ClusterIngressNetworkingModel instance = new TlsClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, + List.of(createNodeIdRange("a", 1, 2)), TLS, sharedSniPort); + + assertThat(instance).isNotNull(); + + // when + List serviceBuilders = instance.services().toList(); + + // then + var listAssert = assertThat(serviceBuilders).isNotNull().hasSize(3); + + listAssert.element(0).satisfies(serviceBuild -> { + assertServiceNameAndPortMapping(serviceBuild, CLUSTER_NAME + "-" + INGRESS_NAME, 9292, sharedSniPort); + }); + + listAssert.element(1).satisfies(serviceBuild -> { + assertServiceNameAndPortMapping(serviceBuild, CLUSTER_NAME + "-" + INGRESS_NAME + "-1", 9292, sharedSniPort); + }); + + listAssert.element(2).satisfies(serviceBuild -> { + assertServiceNameAndPortMapping(serviceBuild, CLUSTER_NAME + "-" + INGRESS_NAME + "-2", 9292, sharedSniPort); + }); + } + + private static void assertServiceNameAndPortMapping(ServiceBuilder serviceBuild, String expectedName, int port, int targetPort) { + Service build = serviceBuild.build(); + assertThat(build.getMetadata()).isNotNull().satisfies(metadata -> { + assertThat(metadata.getName()).isEqualTo(expectedName); + }); + assertThat(build.getSpec()).isNotNull().satisfies(serviceSpec -> { + assertThat(serviceSpec.getPorts()) + .isNotNull() + .isNotEmpty() + .containsExactly(new ServicePortBuilder() + .withTargetPort(new IntOrString(targetPort)) + .withPort(port) + .withProtocol("TCP") + .build()); + }); + } + + @Test + void servicesMultipleRangesOfNodeIds() { + // given + int sharedSniPort = 5; + ClusterIngressNetworkingModel instance = new TlsClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, + List.of(createNodeIdRange("a", 1, 1), createNodeIdRange("b", 3, 3)), TLS, sharedSniPort); + + assertThat(instance).isNotNull(); + + // when + List serviceBuilders = instance.services().toList(); + + // then + var listAssert = assertThat(serviceBuilders).isNotNull().hasSize(3); + + listAssert.element(0).satisfies(serviceBuild -> { + assertServiceNameAndPortMapping(serviceBuild, CLUSTER_NAME + "-" + INGRESS_NAME, 9292, sharedSniPort); + }); + + listAssert.element(1).satisfies(serviceBuild -> { + assertServiceNameAndPortMapping(serviceBuild, CLUSTER_NAME + "-" + INGRESS_NAME + "-1", 9292, sharedSniPort); + }); + + listAssert.element(2).satisfies(serviceBuild -> { + assertServiceNameAndPortMapping(serviceBuild, CLUSTER_NAME + "-" + INGRESS_NAME + "-3", 9292, sharedSniPort); + }); + } + + @Test + void nodeIdRangesMustBeNonEmpty() { + List nodeIdRanges = List.of(); + assertThatThrownBy(() -> new TlsClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, nodeIdRanges, TLS, 5)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void requiresSharedSniContainerPort() { + ClusterIngressNetworkingModel instance = new TlsClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, + List.of(createNodeIdRange("a", 1, 3)), TLS, 5); + + assertThat(instance.requiresSharedSniContainerPort()).isTrue(); + } + + @Test + void identifyingProxyContainerPortsEmpty() { + ClusterIngressNetworkingModel instance = new TlsClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, + List.of(createNodeIdRange("a", 1, 3)), TLS, 5); + + assertThat(instance).isNotNull(); + assertThat(instance.identifyingProxyContainerPorts()) + .isEmpty(); + } + + public static Stream constructorArgsMustBeNonNull() { + List nodeIdRanges = List.of(NODE_ID_RANGE); + ThrowingCallable nullIngress = () -> new TlsClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, null, nodeIdRanges, TLS, 5); + ThrowingCallable nullVirtualCluster = () -> new TlsClusterIPClusterIngressNetworkingModel(PROXY, null, INGRESS, nodeIdRanges, TLS, 5); + ThrowingCallable nullProxy = () -> new TlsClusterIPClusterIngressNetworkingModel(null, VIRTUAL_KAFKA_CLUSTER, INGRESS, nodeIdRanges, TLS, 5); + ThrowingCallable nullRanges = () -> new TlsClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, null, TLS, 5); + ThrowingCallable nullTls = () -> new TlsClusterIPClusterIngressNetworkingModel(PROXY, VIRTUAL_KAFKA_CLUSTER, INGRESS, null, null, 5); + return Stream.of(argumentSet("ingress", nullIngress), + argumentSet("virtualCluster", nullVirtualCluster), + argumentSet("proxy", nullProxy), + argumentSet("ranges", nullRanges), + argumentSet("tls", nullTls)); + } + + @MethodSource + @ParameterizedTest + void constructorArgsMustBeNonNull(ThrowingCallable callable) { + assertThatThrownBy(callable).isInstanceOf(NullPointerException.class); + } + + private static @NonNull Map commonLabels(String proxyName) { + Map orderedLabels = new LinkedHashMap<>(); + orderedLabels.put("app.kubernetes.io/part-of", "kafka"); + orderedLabels.put("app.kubernetes.io/managed-by", "kroxylicious-operator"); + orderedLabels.put("app.kubernetes.io/name", "kroxylicious-proxy"); + orderedLabels.put("app.kubernetes.io/instance", proxyName); + orderedLabels.put("app.kubernetes.io/component", "proxy"); + return orderedLabels; + } + + private static NodeIdRanges createNodeIdRange(@Nullable String name, long start, long endInclusive) { + return new NodeIdRangesBuilder().withName(name).withStart(start).withEnd(endInclusive).build(); + } + +} diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-ConfigMap-minimal-proxy-config.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-ConfigMap-minimal-proxy-config.yaml index 51ee0b2c73..d998e8617c 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-ConfigMap-minimal-proxy-config.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-ConfigMap-minimal-proxy-config.yaml @@ -40,13 +40,9 @@ data: privateKeyFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.key" certificateFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.crt" - name: "cluster-ip" - portIdentifiesNode: - bootstrapAddress: "localhost:9292" - advertisedBrokerAddressPattern: "one-cluster-ip.proxy-ns.svc.cluster.local" - nodeIdRanges: - - name: "default" - start: 0 - end: 2 + sniHostIdentifiesNode: + bootstrapAddress: "one-cluster-ip.proxy-ns.svc.cluster.local:9291" + advertisedBrokerAddressPattern: "one-cluster-ip-$(nodeId).proxy-ns.svc.cluster.local:9292" tls: key: privateKeyFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.key" diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Deployment-minimal.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Deployment-minimal.yaml index 9c5ecd896f..4fbb76526d 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Deployment-minimal.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Deployment-minimal.yaml @@ -60,14 +60,6 @@ spec: ports: - containerPort: 9190 name: "management" - - containerPort: 9292 - name: "9292-bootstrap" - - containerPort: 9293 - name: "9293-node" - - containerPort: 9294 - name: "9294-node" - - containerPort: 9295 - name: "9295-node" - containerPort: 9291 name: "shared-sni-port" securityContext: diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-one-cluster-ip-0.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-one-cluster-ip-0.yaml new file mode 100644 index 0000000000..d504bf82a6 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-one-cluster-ip-0.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-0" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-one-cluster-ip-1.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-one-cluster-ip-1.yaml new file mode 100644 index 0000000000..6da0b4f985 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-one-cluster-ip-1.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-1" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-one-cluster-ip-2.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-one-cluster-ip-2.yaml new file mode 100644 index 0000000000..083997cdbd --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-one-cluster-ip-2.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-2" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-one-cluster-ip.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-one-cluster-ip.yaml index f6f71092c7..164191a941 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-one-cluster-ip.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/loadbalancer-and-clusterip-ingress/out-Service-one-cluster-ip.yaml @@ -28,22 +28,9 @@ metadata: name: "cluster-ip" spec: ports: - - name: "one-9292" - port: 9292 + - port: 9292 protocol: "TCP" - targetPort: 9292 - - name: "one-9293" - port: 9293 - protocol: "TCP" - targetPort: 9293 - - name: "one-9294" - port: 9294 - protocol: "TCP" - targetPort: 9294 - - name: "one-9295" - port: 9295 - protocol: "TCP" - targetPort: 9295 + targetPort: 9291 selector: app: "kroxylicious" app.kubernetes.io/part-of: "kafka" diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-ConfigMap-minimal-proxy-config.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-ConfigMap-minimal-proxy-config.yaml index 56deaa0b24..aec71a01e8 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-ConfigMap-minimal-proxy-config.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-ConfigMap-minimal-proxy-config.yaml @@ -32,13 +32,9 @@ data: bootstrapServers: "my-cluster-kafka-bootstrap.kafka.svc.cluster.local:9092" gateways: - name: "cluster-ip" - portIdentifiesNode: - bootstrapAddress: "localhost:9292" - advertisedBrokerAddressPattern: "one-cluster-ip.proxy-ns.svc.cluster.local" - nodeIdRanges: - - name: "default" - start: 0 - end: 2 + sniHostIdentifiesNode: + bootstrapAddress: "one-cluster-ip.proxy-ns.svc.cluster.local:9291" + advertisedBrokerAddressPattern: "one-cluster-ip-$(nodeId).proxy-ns.svc.cluster.local:9292" tls: key: privateKeyFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.key" diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-Deployment-minimal.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-Deployment-minimal.yaml index df9f20d7d4..77ada6951a 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-Deployment-minimal.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-Deployment-minimal.yaml @@ -60,14 +60,8 @@ spec: ports: - containerPort: 9190 name: "management" - - containerPort: 9292 - name: "9292-bootstrap" - - containerPort: 9293 - name: "9293-node" - - containerPort: 9294 - name: "9294-node" - - containerPort: 9295 - name: "9295-node" + - containerPort: 9291 + name: "shared-sni-port" securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-Service-one-cluster-ip-0.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-Service-one-cluster-ip-0.yaml new file mode 100644 index 0000000000..d504bf82a6 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-Service-one-cluster-ip-0.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-0" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-Service-one-cluster-ip-1.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-Service-one-cluster-ip-1.yaml new file mode 100644 index 0000000000..6da0b4f985 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-Service-one-cluster-ip-1.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-1" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-Service-one-cluster-ip-2.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-Service-one-cluster-ip-2.yaml new file mode 100644 index 0000000000..083997cdbd --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-Service-one-cluster-ip-2.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-2" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-Service-one-cluster-ip.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-Service-one-cluster-ip.yaml index f6f71092c7..164191a941 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-Service-one-cluster-ip.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-controls/out-Service-one-cluster-ip.yaml @@ -28,22 +28,9 @@ metadata: name: "cluster-ip" spec: ports: - - name: "one-9292" - port: 9292 + - port: 9292 protocol: "TCP" - targetPort: 9292 - - name: "one-9293" - port: 9293 - protocol: "TCP" - targetPort: 9293 - - name: "one-9294" - port: 9294 - protocol: "TCP" - targetPort: 9294 - - name: "one-9295" - port: 9295 - protocol: "TCP" - targetPort: 9295 + targetPort: 9291 selector: app: "kroxylicious" app.kubernetes.io/part-of: "kafka" diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-ConfigMap-minimal-proxy-config.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-ConfigMap-minimal-proxy-config.yaml index 490edd4565..41d9aa02ba 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-ConfigMap-minimal-proxy-config.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-ConfigMap-minimal-proxy-config.yaml @@ -32,13 +32,9 @@ data: bootstrapServers: "my-cluster-kafka-bootstrap.kafka.svc.cluster.local:9092" gateways: - name: "cluster-ip" - portIdentifiesNode: - bootstrapAddress: "localhost:9292" - advertisedBrokerAddressPattern: "one-cluster-ip.proxy-ns.svc.cluster.local" - nodeIdRanges: - - name: "default" - start: 0 - end: 2 + sniHostIdentifiesNode: + bootstrapAddress: "one-cluster-ip.proxy-ns.svc.cluster.local:9291" + advertisedBrokerAddressPattern: "one-cluster-ip-$(nodeId).proxy-ns.svc.cluster.local:9292" tls: key: privateKeyFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.key" diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-Deployment-minimal.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-Deployment-minimal.yaml index c3d5d9f2e6..32fdb43783 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-Deployment-minimal.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-Deployment-minimal.yaml @@ -60,14 +60,8 @@ spec: ports: - containerPort: 9190 name: "management" - - containerPort: 9292 - name: "9292-bootstrap" - - containerPort: 9293 - name: "9293-node" - - containerPort: 9294 - name: "9294-node" - - containerPort: 9295 - name: "9295-node" + - containerPort: 9291 + name: "shared-sni-port" securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-Service-one-cluster-ip-0.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-Service-one-cluster-ip-0.yaml new file mode 100644 index 0000000000..d504bf82a6 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-Service-one-cluster-ip-0.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-0" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-Service-one-cluster-ip-1.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-Service-one-cluster-ip-1.yaml new file mode 100644 index 0000000000..6da0b4f985 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-Service-one-cluster-ip-1.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-1" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-Service-one-cluster-ip-2.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-Service-one-cluster-ip-2.yaml new file mode 100644 index 0000000000..083997cdbd --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-Service-one-cluster-ip-2.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-2" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-Service-one-cluster-ip.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-Service-one-cluster-ip.yaml index f6f71092c7..164191a941 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-Service-one-cluster-ip.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-default-client-auth/out-Service-one-cluster-ip.yaml @@ -28,22 +28,9 @@ metadata: name: "cluster-ip" spec: ports: - - name: "one-9292" - port: 9292 + - port: 9292 protocol: "TCP" - targetPort: 9292 - - name: "one-9293" - port: 9293 - protocol: "TCP" - targetPort: 9293 - - name: "one-9294" - port: 9294 - protocol: "TCP" - targetPort: 9294 - - name: "one-9295" - port: 9295 - protocol: "TCP" - targetPort: 9295 + targetPort: 9291 selector: app: "kroxylicious" app.kubernetes.io/part-of: "kafka" diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-ConfigMap-minimal-proxy-config.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-ConfigMap-minimal-proxy-config.yaml index b9002ef653..ce34d2d770 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-ConfigMap-minimal-proxy-config.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-ConfigMap-minimal-proxy-config.yaml @@ -32,13 +32,9 @@ data: bootstrapServers: "my-cluster-kafka-bootstrap.kafka.svc.cluster.local:9092" gateways: - name: "cluster-ip" - portIdentifiesNode: - bootstrapAddress: "localhost:9292" - advertisedBrokerAddressPattern: "one-cluster-ip.proxy-ns.svc.cluster.local" - nodeIdRanges: - - name: "default" - start: 0 - end: 2 + sniHostIdentifiesNode: + bootstrapAddress: "one-cluster-ip.proxy-ns.svc.cluster.local:9291" + advertisedBrokerAddressPattern: "one-cluster-ip-$(nodeId).proxy-ns.svc.cluster.local:9292" tls: key: privateKeyFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.key" diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-Deployment-minimal.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-Deployment-minimal.yaml index c3d5d9f2e6..32fdb43783 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-Deployment-minimal.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-Deployment-minimal.yaml @@ -60,14 +60,8 @@ spec: ports: - containerPort: 9190 name: "management" - - containerPort: 9292 - name: "9292-bootstrap" - - containerPort: 9293 - name: "9293-node" - - containerPort: 9294 - name: "9294-node" - - containerPort: 9295 - name: "9295-node" + - containerPort: 9291 + name: "shared-sni-port" securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-Service-one-cluster-ip-0.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-Service-one-cluster-ip-0.yaml new file mode 100644 index 0000000000..d504bf82a6 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-Service-one-cluster-ip-0.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-0" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-Service-one-cluster-ip-1.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-Service-one-cluster-ip-1.yaml new file mode 100644 index 0000000000..6da0b4f985 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-Service-one-cluster-ip-1.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-1" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-Service-one-cluster-ip-2.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-Service-one-cluster-ip-2.yaml new file mode 100644 index 0000000000..083997cdbd --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-Service-one-cluster-ip-2.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-2" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-Service-one-cluster-ip.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-Service-one-cluster-ip.yaml index f6f71092c7..164191a941 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-Service-one-cluster-ip.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-none/out-Service-one-cluster-ip.yaml @@ -28,22 +28,9 @@ metadata: name: "cluster-ip" spec: ports: - - name: "one-9292" - port: 9292 + - port: 9292 protocol: "TCP" - targetPort: 9292 - - name: "one-9293" - port: 9293 - protocol: "TCP" - targetPort: 9293 - - name: "one-9294" - port: 9294 - protocol: "TCP" - targetPort: 9294 - - name: "one-9295" - port: 9295 - protocol: "TCP" - targetPort: 9295 + targetPort: 9291 selector: app: "kroxylicious" app.kubernetes.io/part-of: "kafka" diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-ConfigMap-minimal-proxy-config.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-ConfigMap-minimal-proxy-config.yaml index 0b541febcc..043c672c73 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-ConfigMap-minimal-proxy-config.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-ConfigMap-minimal-proxy-config.yaml @@ -32,13 +32,9 @@ data: bootstrapServers: "my-cluster-kafka-bootstrap.kafka.svc.cluster.local:9092" gateways: - name: "cluster-ip" - portIdentifiesNode: - bootstrapAddress: "localhost:9292" - advertisedBrokerAddressPattern: "one-cluster-ip.proxy-ns.svc.cluster.local" - nodeIdRanges: - - name: "default" - start: 0 - end: 2 + sniHostIdentifiesNode: + bootstrapAddress: "one-cluster-ip.proxy-ns.svc.cluster.local:9291" + advertisedBrokerAddressPattern: "one-cluster-ip-$(nodeId).proxy-ns.svc.cluster.local:9292" tls: key: privateKeyFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.key" diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-Deployment-minimal.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-Deployment-minimal.yaml index c3d5d9f2e6..32fdb43783 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-Deployment-minimal.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-Deployment-minimal.yaml @@ -60,14 +60,8 @@ spec: ports: - containerPort: 9190 name: "management" - - containerPort: 9292 - name: "9292-bootstrap" - - containerPort: 9293 - name: "9293-node" - - containerPort: 9294 - name: "9294-node" - - containerPort: 9295 - name: "9295-node" + - containerPort: 9291 + name: "shared-sni-port" securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-Service-one-cluster-ip-0.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-Service-one-cluster-ip-0.yaml new file mode 100644 index 0000000000..d504bf82a6 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-Service-one-cluster-ip-0.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-0" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-Service-one-cluster-ip-1.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-Service-one-cluster-ip-1.yaml new file mode 100644 index 0000000000..6da0b4f985 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-Service-one-cluster-ip-1.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-1" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-Service-one-cluster-ip-2.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-Service-one-cluster-ip-2.yaml new file mode 100644 index 0000000000..083997cdbd --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-Service-one-cluster-ip-2.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-2" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-Service-one-cluster-ip.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-Service-one-cluster-ip.yaml index f6f71092c7..164191a941 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-Service-one-cluster-ip.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-requested/out-Service-one-cluster-ip.yaml @@ -28,22 +28,9 @@ metadata: name: "cluster-ip" spec: ports: - - name: "one-9292" - port: 9292 + - port: 9292 protocol: "TCP" - targetPort: 9292 - - name: "one-9293" - port: 9293 - protocol: "TCP" - targetPort: 9293 - - name: "one-9294" - port: 9294 - protocol: "TCP" - targetPort: 9294 - - name: "one-9295" - port: 9295 - protocol: "TCP" - targetPort: 9295 + targetPort: 9291 selector: app: "kroxylicious" app.kubernetes.io/part-of: "kafka" diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-ConfigMap-minimal-proxy-config.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-ConfigMap-minimal-proxy-config.yaml index 490edd4565..41d9aa02ba 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-ConfigMap-minimal-proxy-config.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-ConfigMap-minimal-proxy-config.yaml @@ -32,13 +32,9 @@ data: bootstrapServers: "my-cluster-kafka-bootstrap.kafka.svc.cluster.local:9092" gateways: - name: "cluster-ip" - portIdentifiesNode: - bootstrapAddress: "localhost:9292" - advertisedBrokerAddressPattern: "one-cluster-ip.proxy-ns.svc.cluster.local" - nodeIdRanges: - - name: "default" - start: 0 - end: 2 + sniHostIdentifiesNode: + bootstrapAddress: "one-cluster-ip.proxy-ns.svc.cluster.local:9291" + advertisedBrokerAddressPattern: "one-cluster-ip-$(nodeId).proxy-ns.svc.cluster.local:9292" tls: key: privateKeyFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.key" diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-Deployment-minimal.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-Deployment-minimal.yaml index c3d5d9f2e6..32fdb43783 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-Deployment-minimal.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-Deployment-minimal.yaml @@ -60,14 +60,8 @@ spec: ports: - containerPort: 9190 name: "management" - - containerPort: 9292 - name: "9292-bootstrap" - - containerPort: 9293 - name: "9293-node" - - containerPort: 9294 - name: "9294-node" - - containerPort: 9295 - name: "9295-node" + - containerPort: 9291 + name: "shared-sni-port" securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-Service-one-cluster-ip-0.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-Service-one-cluster-ip-0.yaml new file mode 100644 index 0000000000..d504bf82a6 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-Service-one-cluster-ip-0.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-0" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-Service-one-cluster-ip-1.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-Service-one-cluster-ip-1.yaml new file mode 100644 index 0000000000..6da0b4f985 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-Service-one-cluster-ip-1.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-1" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-Service-one-cluster-ip-2.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-Service-one-cluster-ip-2.yaml new file mode 100644 index 0000000000..083997cdbd --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-Service-one-cluster-ip-2.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-2" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-Service-one-cluster-ip.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-Service-one-cluster-ip.yaml index f6f71092c7..164191a941 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-Service-one-cluster-ip.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls-with-explicit-client-auth-required/out-Service-one-cluster-ip.yaml @@ -28,22 +28,9 @@ metadata: name: "cluster-ip" spec: ports: - - name: "one-9292" - port: 9292 + - port: 9292 protocol: "TCP" - targetPort: 9292 - - name: "one-9293" - port: 9293 - protocol: "TCP" - targetPort: 9293 - - name: "one-9294" - port: 9294 - protocol: "TCP" - targetPort: 9294 - - name: "one-9295" - port: 9295 - protocol: "TCP" - targetPort: 9295 + targetPort: 9291 selector: app: "kroxylicious" app.kubernetes.io/part-of: "kafka" diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-ConfigMap-minimal-proxy-config.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-ConfigMap-minimal-proxy-config.yaml index 7b37d8f00a..bafc2b0adb 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-ConfigMap-minimal-proxy-config.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-ConfigMap-minimal-proxy-config.yaml @@ -32,13 +32,9 @@ data: bootstrapServers: "my-cluster-kafka-bootstrap.kafka.svc.cluster.local:9092" gateways: - name: "cluster-ip" - portIdentifiesNode: - bootstrapAddress: "localhost:9292" - advertisedBrokerAddressPattern: "one-cluster-ip.proxy-ns.svc.cluster.local" - nodeIdRanges: - - name: "default" - start: 0 - end: 2 + sniHostIdentifiesNode: + bootstrapAddress: "one-cluster-ip.proxy-ns.svc.cluster.local:9291" + advertisedBrokerAddressPattern: "one-cluster-ip-$(nodeId).proxy-ns.svc.cluster.local:9292" tls: key: privateKeyFile: "/opt/kroxylicious/virtual-cluster/server-certs/downstream-tls-cert/tls.key" diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-Deployment-minimal.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-Deployment-minimal.yaml index 5fda8ae859..4fbb76526d 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-Deployment-minimal.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-Deployment-minimal.yaml @@ -60,14 +60,8 @@ spec: ports: - containerPort: 9190 name: "management" - - containerPort: 9292 - name: "9292-bootstrap" - - containerPort: 9293 - name: "9293-node" - - containerPort: 9294 - name: "9294-node" - - containerPort: 9295 - name: "9295-node" + - containerPort: 9291 + name: "shared-sni-port" securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-Service-one-cluster-ip-0.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-Service-one-cluster-ip-0.yaml new file mode 100644 index 0000000000..d504bf82a6 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-Service-one-cluster-ip-0.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-0" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-Service-one-cluster-ip-1.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-Service-one-cluster-ip-1.yaml new file mode 100644 index 0000000000..6da0b4f985 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-Service-one-cluster-ip-1.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-1" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-Service-one-cluster-ip-2.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-Service-one-cluster-ip-2.yaml new file mode 100644 index 0000000000..083997cdbd --- /dev/null +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-Service-one-cluster-ip-2.yaml @@ -0,0 +1,40 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +apiVersion: "v1" +kind: "Service" +metadata: + labels: + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" + name: "one-cluster-ip-2" + namespace: "proxy-ns" + ownerReferences: + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxy" + name: "minimal" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "VirtualKafkaCluster" + name: "one" + - apiVersion: "kroxylicious.io/v1alpha1" + kind: "KafkaProxyIngress" + name: "cluster-ip" +spec: + ports: + - port: 9292 + protocol: "TCP" + targetPort: 9291 + selector: + app: "kroxylicious" + app.kubernetes.io/part-of: "kafka" + app.kubernetes.io/managed-by: "kroxylicious-operator" + app.kubernetes.io/name: "kroxylicious-proxy" + app.kubernetes.io/instance: "minimal" + app.kubernetes.io/component: "proxy" \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-Service-one-cluster-ip.yaml b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-Service-one-cluster-ip.yaml index f6f71092c7..164191a941 100644 --- a/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-Service-one-cluster-ip.yaml +++ b/kroxylicious-operator/src/test/resources/DerivedResourcesTest/virtualcluster-with-tls/out-Service-one-cluster-ip.yaml @@ -28,22 +28,9 @@ metadata: name: "cluster-ip" spec: ports: - - name: "one-9292" - port: 9292 + - port: 9292 protocol: "TCP" - targetPort: 9292 - - name: "one-9293" - port: 9293 - protocol: "TCP" - targetPort: 9293 - - name: "one-9294" - port: 9294 - protocol: "TCP" - targetPort: 9294 - - name: "one-9295" - port: 9295 - protocol: "TCP" - targetPort: 9295 + targetPort: 9291 selector: app: "kroxylicious" app.kubernetes.io/part-of: "kafka" From d0fa0068d902ce38f848045b731273e4938041d0 Mon Sep 17 00:00:00 2001 From: Robert Young Date: Thu, 5 Jun 2025 10:14:18 +1200 Subject: [PATCH 79/83] fix: run-operator failed with empty minikube kubectl exited non-zero when we try to delete some resources, when k8s has no knowledge of those CRDs. This command is intended to clean out existing resources and CRDs from an established minikube, as well as install into a fresh minikube. So we can tactically ignore those failures. Signed-off-by: Robert Young Signed-off-by: Ujjwal --- scripts/run-operator.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/run-operator.sh b/scripts/run-operator.sh index adf4b44f88..2638fc9ad9 100755 --- a/scripts/run-operator.sh +++ b/scripts/run-operator.sh @@ -39,11 +39,13 @@ kubectl apply -n kafka -f https://strimzi.io/examples/latest/kafka/kafka-single- kubectl wait -n kafka kafka/my-cluster --for=condition=Ready --timeout=300s info "deleting example" +set +e kubectl delete -f examples/simple/ --ignore-not-found=true --timeout=30s --grace-period=1 info "deleting kroxylicious-operator installation" kubectl delete -n kroxylicious-operator all --all --timeout=30s --grace-period=1 kubectl delete -f install --ignore-not-found=true --timeout=30s --grace-period=1 +set -e info "deleting all kroxylicious.io resources and crds" for crd in $(kubectl get crds -oname | grep kroxylicious.io | awk -F / '{ print $2 }'); From 0c9c042173681d8abffbd6212c6036a8b0654cc3 Mon Sep 17 00:00:00 2001 From: Robert Young Date: Fri, 6 Jun 2025 09:37:13 +1200 Subject: [PATCH 80/83] Reduce max length of KafkaProxy metadata.name to 50 Why: We want to be able to suffix the proxy name to create a shared LoadBalancer service. Limiting the KafkaProxy name gives us some wiggle room since we can only create Services with names up to 63 characters. Signed-off-by: Robert Young Signed-off-by: Ujjwal --- .../fabric8/kafkaproxies.kroxylicious.io-v1.yml | 2 +- .../invalid-metadata.name-over-50-characters.yaml | 15 +++++++++++++++ .../valid-metadata.name-50-characters.yaml | 12 ++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 kroxylicious-operator/src/test/resources/CustomResourceValidationIT/kafkaproxy/invalid-metadata.name-over-50-characters.yaml create mode 100644 kroxylicious-operator/src/test/resources/CustomResourceValidationIT/kafkaproxy/valid-metadata.name-50-characters.yaml diff --git a/kroxylicious-kubernetes-api/src/main/resources/META-INF/fabric8/kafkaproxies.kroxylicious.io-v1.yml b/kroxylicious-kubernetes-api/src/main/resources/META-INF/fabric8/kafkaproxies.kroxylicious.io-v1.yml index 616eea2c46..7faadeefb4 100644 --- a/kroxylicious-kubernetes-api/src/main/resources/META-INF/fabric8/kafkaproxies.kroxylicious.io-v1.yml +++ b/kroxylicious-kubernetes-api/src/main/resources/META-INF/fabric8/kafkaproxies.kroxylicious.io-v1.yml @@ -46,7 +46,7 @@ spec: properties: name: type: string - maxLength: 63 + maxLength: 50 pattern: "[a-z0-9]([a-z0-9-]*[a-z0-9])?" spec: type: object diff --git a/kroxylicious-operator/src/test/resources/CustomResourceValidationIT/kafkaproxy/invalid-metadata.name-over-50-characters.yaml b/kroxylicious-operator/src/test/resources/CustomResourceValidationIT/kafkaproxy/invalid-metadata.name-over-50-characters.yaml new file mode 100644 index 0000000000..b39a07da31 --- /dev/null +++ b/kroxylicious-operator/src/test/resources/CustomResourceValidationIT/kafkaproxy/invalid-metadata.name-over-50-characters.yaml @@ -0,0 +1,15 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +resource: + kind: KafkaProxy + apiVersion: kroxylicious.io/v1alpha1 + metadata: + name: 123456789012345678901234567890123456789012345678901 + namespace: proxy-ns +expectFailureMessageToContain: | + metadata.name: Too long \ No newline at end of file diff --git a/kroxylicious-operator/src/test/resources/CustomResourceValidationIT/kafkaproxy/valid-metadata.name-50-characters.yaml b/kroxylicious-operator/src/test/resources/CustomResourceValidationIT/kafkaproxy/valid-metadata.name-50-characters.yaml new file mode 100644 index 0000000000..3fb6248bfe --- /dev/null +++ b/kroxylicious-operator/src/test/resources/CustomResourceValidationIT/kafkaproxy/valid-metadata.name-50-characters.yaml @@ -0,0 +1,12 @@ +# +# Copyright Kroxylicious Authors. +# +# Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 +# + +--- +kind: KafkaProxy +apiVersion: kroxylicious.io/v1alpha1 +metadata: + name: 12345678901234567890123456789012345678901234567890 + namespace: proxy-ns \ No newline at end of file From bed49edf511147c72ca8977758e441e2cb41d5a0 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Fri, 6 Jun 2025 15:34:26 +1200 Subject: [PATCH 81/83] Remove duplicate dependency definition Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- kroxylicious-integration-test-support/pom.xml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/kroxylicious-integration-test-support/pom.xml b/kroxylicious-integration-test-support/pom.xml index 18b09e4e9f..952c398689 100644 --- a/kroxylicious-integration-test-support/pom.xml +++ b/kroxylicious-integration-test-support/pom.xml @@ -162,7 +162,7 @@ org.assertj assertj-core - test + compile org.apache.kafka @@ -189,11 +189,6 @@ mockito-junit-jupiter test - - org.assertj - assertj-core - compile - From 937a5aa00a114fdb75748be31a974f08d6f9200c Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Fri, 6 Jun 2025 15:49:03 +1200 Subject: [PATCH 82/83] Push operator image for release version Fixes: #1852 Signed-off-by: Sam Barker Signed-off-by: Ujjwal --- .github/workflows/docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 9b2aa54770..63d33ba455 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -63,7 +63,7 @@ jobs: context: . file: ./Dockerfile.operator platforms: linux/amd64,linux/arm64 - push: ${{ endsWith(env.RELEASE_VERSION, 'SNAPSHOT') }} + push: true build-args: | KROXYLICIOUS_VERSION=${{ env.RELEASE_VERSION }} tags: ${{ env.OPERATOR_IMAGE }}:${{ env.RELEASE_VERSION }},${{ env.OPERATOR_IMAGE }}:latest From b0e4f40b44b1ee2c864ee6e1dea50a825c649181 Mon Sep 17 00:00:00 2001 From: Francisco Vila <57452611+franvila@users.noreply.github.com> Date: Fri, 6 Jun 2025 09:51:44 +0200 Subject: [PATCH 83/83] [System Tests] OLM installation (#2273) * first try Signed-off-by: Francisco Vila * make olm installation work Signed-off-by: Francisco Vila * added some default values Signed-off-by: Francisco Vila * fix installation method Signed-off-by: Francisco Vila * add olm deployment name as env variable Signed-off-by: Francisco Vila * replace install type env variable Signed-off-by: Francisco Vila * fix format Signed-off-by: Francisco Vila * added undeclared dependencies Signed-off-by: Francisco Vila * set new default value for olm deployement Signed-off-by: Francisco Vila * small nits from Keith Signed-off-by: Francisco Vila --------- Signed-off-by: Francisco Vila Signed-off-by: Ujjwal --- kroxylicious-systemtests/pom.xml | 8 + .../kroxylicious/systemtests/Constants.java | 3 + .../kroxylicious/systemtests/Environment.java | 18 +- .../kroxylicious/KroxyliciousOperator.java | 12 +- ...roxyliciousOperatorOlmBundleInstaller.java | 200 ++++++++++++++++++ ...=> KroxyliciousOperatorYamlInstaller.java} | 16 +- pom.xml | 11 +- 7 files changed, 253 insertions(+), 15 deletions(-) create mode 100644 kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/operator/KroxyliciousOperatorOlmBundleInstaller.java rename kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/operator/{KroxyliciousOperatorBundleInstaller.java => KroxyliciousOperatorYamlInstaller.java} (94%) diff --git a/kroxylicious-systemtests/pom.xml b/kroxylicious-systemtests/pom.xml index 1de3378135..27b7a63a36 100644 --- a/kroxylicious-systemtests/pom.xml +++ b/kroxylicious-systemtests/pom.xml @@ -108,6 +108,10 @@ io.fabric8 kubernetes-model-common + + io.fabric8 + openshift-model-operatorhub + io.strimzi api @@ -212,6 +216,10 @@ io.skodjob test-frame-kubernetes + + io.skodjob + test-frame-openshift + io.skodjob test-frame-log-collector diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/Constants.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/Constants.java index e068fe9fba..4cf32cac28 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/Constants.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/Constants.java @@ -27,6 +27,9 @@ private Constants() { public static final String KROXYLICIOUS_ENCRYPTION_FILTER_NAME = "encryption"; public static final String KROXYLICIOUS_TLS_CLIENT_CA_CERT = "my-cluster-clients-ca-cert"; public static final String KROXYLICIOUS_TLS_CA_NAME = "ca.pem"; + public static final String KROXYLICIOUS_OPERATOR_SUBSCRIPTION_NAME = Environment.KROXYLICIOUS_OLM_DEPLOYMENT_NAME + "-v" + Environment.KROXYLICIOUS_OPERATOR_VERSION + + "-sub"; + public static final String KROXYLICIOUS_OPERATOR_OLM_LABEL = Environment.KROXYLICIOUS_OLM_DEPLOYMENT_NAME + "-operator-v" + Environment.KROXYLICIOUS_OPERATOR_VERSION; /** * Strimzi cluster operator deployment name diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/Environment.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/Environment.java index 7ad463d8d8..464b68cfcb 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/Environment.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/Environment.java @@ -27,7 +27,7 @@ private Environment() { // --------------------------------------- // Env variables initialization // --------------------------------------- - private static final String INSTALL_TYPE_ENV = "INSTALL_TYPE"; + private static final String INSTALL_TYPE_ENV = "CLUSTER_OPERATOR_INSTALL_TYPE"; public static final InstallType INSTALL_TYPE = ENVIRONMENT_VARIABLES.getOrDefault(INSTALL_TYPE_ENV, InstallType::fromString, InstallType.Yaml); /** @@ -51,7 +51,12 @@ private Environment() { private static final String AWS_KROXYLICIOUS_ACCESS_KEY_ID_ENV = "AWS_KROXYLICIOUS_ACCESS_KEY_ID"; private static final String AWS_KROXYLICIOUS_SECRET_ACCESS_KEY_ENV = "AWS_KROXYLICIOUS_SECRET_ACCESS_KEY"; private static final String AWS_REGION_ENV = "AWS_REGION"; + private static final String KROXYLICIOUS_OPERATOR_BUNDLE_IMAGE_ENV = "KROXYLICIOUS_OPERATOR_BUNDLE_IMAGE"; private static final String TEST_CLIENTS_IMAGE_ENV = "TEST_CLIENTS_IMAGE"; + private static final String OLM_OPERATOR_CHANNEL_ENV = "OLM_OPERATOR_CHANNEL"; + private static final String CATALOG_SOURCE_NAME_ENV = "CATALOG_SOURCE_NAME"; + private static final String CATALOG_NAMESPACE_ENV = "CATALOG_NAMESPACE"; + private static final String KROXYLICIOUS_OLM_DEPLOYMENT_NAME_ENV = "KROXYLICIOUS_OLM_DEPLOYMENT_NAME"; /** * The kafka version default value @@ -103,6 +108,10 @@ private Environment() { private static final String AWS_KROXYLICIOUS_SECRET_ACCESS_KEY_DEFAULT = AWS_SECRET_ACCESS_KEY_DEFAULT; public static final String AWS_REGION_DEFAULT = "us-east-2"; private static final String TEST_CLIENTS_IMAGE_DEFAULT = "quay.io/strimzi-test-clients/test-clients:latest-kafka-" + KAFKA_VERSION_DEFAULT; + private static final String OLM_OPERATOR_CHANNEL_DEFAULT = "alpha"; + private static final String CATALOG_SOURCE_NAME_DEFAULT = "kroxylicious-source"; + private static final String KROXYLICIOUS_OLM_DEPLOYMENT_NAME_DEFAULT = "kroxylicious-operator"; + private static final String CATALOG_NAMESPACE_DEFAULT = "openshift-marketplace"; public static final String KAFKA_VERSION = ENVIRONMENT_VARIABLES.getOrDefault(KAFKA_VERSION_ENV, KAFKA_VERSION_DEFAULT); public static final String KROXYLICIOUS_OPERATOR_VERSION = ENVIRONMENT_VARIABLES.getOrDefault(KROXYLICIOUS_OPERATOR_VERSION_ENV, KROXYLICIOUS_VERSION_DEFAULT); @@ -143,7 +152,14 @@ private Environment() { public static final String KROXYLICIOUS_OPERATOR_REGISTRY = ENVIRONMENT_VARIABLES.getOrDefault(KROXYLICIOUS_OPERATOR_REGISTRY_ENV, KROXYLICIOUS_OPERATOR_REGISTRY_DEFAULT); + public static final String KROXYLICIOUS_OPERATOR_BUNDLE_IMAGE = ENVIRONMENT_VARIABLES.getOrDefault(KROXYLICIOUS_OPERATOR_BUNDLE_IMAGE_ENV, ""); + public static final String TEST_CLIENTS_IMAGE = ENVIRONMENT_VARIABLES.getOrDefault(TEST_CLIENTS_IMAGE_ENV, TEST_CLIENTS_IMAGE_DEFAULT); + public static final String OLM_OPERATOR_CHANNEL = ENVIRONMENT_VARIABLES.getOrDefault(OLM_OPERATOR_CHANNEL_ENV, OLM_OPERATOR_CHANNEL_DEFAULT); + public static final String CATALOG_SOURCE_NAME = ENVIRONMENT_VARIABLES.getOrDefault(CATALOG_SOURCE_NAME_ENV, CATALOG_SOURCE_NAME_DEFAULT); + public static final String CATALOG_NAMESPACE = ENVIRONMENT_VARIABLES.getOrDefault(CATALOG_NAMESPACE_ENV, CATALOG_NAMESPACE_DEFAULT); + public static final String KROXYLICIOUS_OLM_DEPLOYMENT_NAME = ENVIRONMENT_VARIABLES.getOrDefault(KROXYLICIOUS_OLM_DEPLOYMENT_NAME_ENV, + KROXYLICIOUS_OLM_DEPLOYMENT_NAME_DEFAULT); private static String readMetadataProperty(String property) { var p = new Properties(); diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/KroxyliciousOperator.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/KroxyliciousOperator.java index af82989616..f37752d9c3 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/KroxyliciousOperator.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/installation/kroxylicious/KroxyliciousOperator.java @@ -10,8 +10,8 @@ import io.skodjob.testframe.installation.InstallationMethod; import io.kroxylicious.systemtests.Environment; -import io.kroxylicious.systemtests.k8s.exception.UnsupportedInstallationType; -import io.kroxylicious.systemtests.resources.operator.KroxyliciousOperatorBundleInstaller; +import io.kroxylicious.systemtests.resources.operator.KroxyliciousOperatorOlmBundleInstaller; +import io.kroxylicious.systemtests.resources.operator.KroxyliciousOperatorYamlInstaller; /** * The type Kroxylicious operator. @@ -45,9 +45,11 @@ public void delete() { } private InstallationMethod getInstallationMethod() { - if (Environment.INSTALL_TYPE != InstallType.Yaml) { - throw new UnsupportedInstallationType("Installation type " + Environment.INSTALL_TYPE + " not supported"); + if (Environment.INSTALL_TYPE == InstallType.Olm) { + return new KroxyliciousOperatorOlmBundleInstaller(installationNamespace); + } + else { + return new KroxyliciousOperatorYamlInstaller(installationNamespace); } - return new KroxyliciousOperatorBundleInstaller(installationNamespace); } } diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/operator/KroxyliciousOperatorOlmBundleInstaller.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/operator/KroxyliciousOperatorOlmBundleInstaller.java new file mode 100644 index 0000000000..0ab5bd1897 --- /dev/null +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/operator/KroxyliciousOperatorOlmBundleInstaller.java @@ -0,0 +1,200 @@ + +/* + * Copyright Kroxylicious Authors. + * + * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.kroxylicious.systemtests.resources.operator; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.jupiter.api.extension.ExtensionContext; + +import io.fabric8.kubernetes.api.model.LabelSelectorBuilder; +import io.fabric8.openshift.api.model.operatorhub.v1.OperatorGroupBuilder; +import io.fabric8.openshift.api.model.operatorhub.v1alpha1.Subscription; +import io.fabric8.openshift.api.model.operatorhub.v1alpha1.SubscriptionBuilder; +import io.skodjob.testframe.TestFrameConstants; +import io.skodjob.testframe.installation.InstallationMethod; +import io.skodjob.testframe.olm.OperatorSdkRun; +import io.skodjob.testframe.olm.OperatorSdkRunBuilder; +import io.skodjob.testframe.resources.KubeResourceManager; +import io.skodjob.testframe.utils.PodUtils; +import io.skodjob.testframe.utils.TestFrameUtils; +import io.skodjob.testframe.wait.Wait; + +import io.kroxylicious.systemtests.Constants; +import io.kroxylicious.systemtests.Environment; +import io.kroxylicious.systemtests.resources.manager.ResourceManager; +import io.kroxylicious.systemtests.utils.NamespaceUtils; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +/** + * KroxyliciousOperatorOlmBundleInstaller encapsulates the whole OLM installation process of Kroxylicious Operator. Based on the @code{Environment} + * values, this class installs Kroxylicious Operator using bundle olm. + */ +public class KroxyliciousOperatorOlmBundleInstaller implements InstallationMethod { + + private static final Logger LOGGER = LogManager.getLogger(KroxyliciousOperatorOlmBundleInstaller.class); + private static final String SEPARATOR = String.join("", Collections.nCopies(76, "=")); + + private final ExtensionContext extensionContext; + private final String kroxyliciousOperatorName; + private final String bundleImageRef; + private final String operatorNamespace; + + public KroxyliciousOperatorOlmBundleInstaller(String operatorNamespace) { + this.operatorNamespace = operatorNamespace; + this.extensionContext = KubeResourceManager.get().getTestContext(); + this.kroxyliciousOperatorName = Environment.KROXYLICIOUS_OLM_DEPLOYMENT_NAME; + this.bundleImageRef = Environment.KROXYLICIOUS_OPERATOR_BUNDLE_IMAGE; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + KroxyliciousOperatorOlmBundleInstaller otherInstallation = (KroxyliciousOperatorOlmBundleInstaller) other; + + return Objects.equals(kroxyliciousOperatorName, otherInstallation.kroxyliciousOperatorName) && + Objects.equals(operatorNamespace, otherInstallation.operatorNamespace) && + Objects.equals(bundleImageRef, otherInstallation.bundleImageRef) && + Objects.equals(extensionContext, otherInstallation.extensionContext); + } + + @Override + public int hashCode() { + return Objects.hash(extensionContext, kroxyliciousOperatorName, operatorNamespace, bundleImageRef); + } + + @Override + public String toString() { + return "KroxyliciousOperatorOlmBundleInstaller{" + + ", extensionContext=" + extensionContext + + ", kroxyliciousOperatorName='" + kroxyliciousOperatorName + '\'' + + ", operatorNamespace='" + operatorNamespace + '\'' + + ", bundleImageRef='" + bundleImageRef + '\'' + + '}'; + } + + @Override + public void install() { + LOGGER.info("Setup Kroxylicious Operator using OLM"); + if (bundleImageRef != null && !bundleImageRef.isEmpty()) { + install(kroxyliciousOperatorName, operatorNamespace, bundleImageRef).join(); + } + else { + install(kroxyliciousOperatorName, operatorNamespace, Environment.OLM_OPERATOR_CHANNEL, Environment.CATALOG_SOURCE_NAME, Environment.CATALOG_NAMESPACE).join(); + } + } + + /** + * Install strimzi operator from catalog presented on cluster using OLM + * + * @param operatorName name of operator + * @param operatorNamespace where operator will be present + * @param channel chanel + * @param source source name of catalog + * @param catalogNs source catalog namespace + * @return wait future + */ + @SuppressFBWarnings("BC_UNCONFIRMED_CAST_OF_RETURN_VALUE") + public CompletableFuture install(String operatorName, String operatorNamespace, + String channel, String source, String catalogNs) { + // Create ns for the operator + NamespaceUtils.createNamespaceAndPrepare(operatorNamespace); + // Create operator group for the operator + if (KubeResourceManager.get().kubeClient().getOpenShiftClient().operatorHub().operatorGroups() + .inNamespace(operatorNamespace).list().getItems().isEmpty()) { + OperatorGroupBuilder operatorGroup = new OperatorGroupBuilder() + .editOrNewMetadata() + .withName(Environment.KROXYLICIOUS_OLM_DEPLOYMENT_NAME + "-operator-group") + .withNamespace(operatorNamespace) + .endMetadata(); + ResourceManager.getInstance().createResourceFromBuilderWithWait(operatorGroup); + } + else { + LOGGER.info("OperatorGroup already exists."); + } + + // @formatter:off + Subscription subscription = new SubscriptionBuilder() + .editOrNewMetadata() + .withName(Constants.KROXYLICIOUS_OPERATOR_SUBSCRIPTION_NAME) + .withNamespace(operatorNamespace) + .endMetadata() + .editOrNewSpec() + .withName(operatorName) + .withChannel(channel) + .withSource(source) + .withSourceNamespace(catalogNs) + .withInstallPlanApproval("Automatic") + .endSpec() + .build(); + // @formatter:on + + KubeResourceManager.get().createOrUpdateResourceWithoutWait(subscription); + return Wait.untilAsync(operatorName + " is ready", TestFrameConstants.GLOBAL_POLL_INTERVAL_1_SEC, + TestFrameConstants.GLOBAL_TIMEOUT, () -> isOperatorReady(operatorNamespace)); + } + + private CompletableFuture install(String operatorName, String operatorNamespace, String bundleImageRef) { + OperatorSdkRun osr = new OperatorSdkRunBuilder() + .withBundleImage(bundleImageRef) + .withInstallMode("AllNamespaces") + .withNamespace(operatorNamespace) + .build(); + + return Wait.untilAsync(operatorName + " is ready", TestFrameConstants.GLOBAL_POLL_INTERVAL_1_SEC, + TestFrameConstants.GLOBAL_TIMEOUT, () -> { + NamespaceUtils.createNamespaceAndPrepare(operatorNamespace); + TestFrameUtils.runUntilPass(3, osr::run); + return isOperatorReady(operatorNamespace); + }); + } + + @SuppressFBWarnings("REC_CATCH_EXCEPTION") + private boolean isOperatorReady(String ns) { + try { + PodUtils.waitForPodsReadyWithRestart(ns, new LabelSelectorBuilder() + .withMatchLabels(Map.of("app.kubernetes.io/instance", Constants.KROXYLICIOUS_OPERATOR_OLM_LABEL)).build(), + 1, true); + LOGGER.info("Kroxylicious operator in namespace {} is ready", ns); + return true; + } + catch (Exception ex) { + return false; + } + } + + @Override + public synchronized void delete() { + LOGGER.info(SEPARATOR); + if (Environment.SKIP_TEARDOWN) { + LOGGER.info("Skip un-installation of the Kroxylicious Operator"); + } + else { + LOGGER.info("Un-installing Kroxylicious Operator from Namespace: {}", operatorNamespace); + + // clear all resources related to the extension context + try { + KubeResourceManager.get().deleteResources(true); + } + catch (Exception e) { + LOGGER.error("An error occurred when deleting the resources", e); + } + } + LOGGER.info(SEPARATOR); + } +} diff --git a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/operator/KroxyliciousOperatorBundleInstaller.java b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/operator/KroxyliciousOperatorYamlInstaller.java similarity index 94% rename from kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/operator/KroxyliciousOperatorBundleInstaller.java rename to kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/operator/KroxyliciousOperatorYamlInstaller.java index 702d7b3983..feee22a6ae 100644 --- a/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/operator/KroxyliciousOperatorBundleInstaller.java +++ b/kroxylicious-systemtests/src/main/java/io/kroxylicious/systemtests/resources/operator/KroxyliciousOperatorYamlInstaller.java @@ -48,13 +48,13 @@ import static io.kroxylicious.systemtests.k8s.KubeClusterResource.kubeClient; /** - * KroxyliciousOperatorBundleInstaller encapsulates the whole installation process of Kroxylicious Operator (i.e., RoleBinding, ClusterRoleBinding, + * KroxyliciousOperatorYamlInstaller encapsulates the whole installation process of Kroxylicious Operator (i.e., RoleBinding, ClusterRoleBinding, * ConfigMap, Deployment, CustomResourceDefinition, preparation of the Namespace). Based on the @code{Environment} * values, this class installs Kroxylicious Operator using bundle yamls. */ -public class KroxyliciousOperatorBundleInstaller implements InstallationMethod { +public class KroxyliciousOperatorYamlInstaller implements InstallationMethod { - private static final Logger LOGGER = LogManager.getLogger(KroxyliciousOperatorBundleInstaller.class); + private static final Logger LOGGER = LogManager.getLogger(KroxyliciousOperatorYamlInstaller.class); private static final String SEPARATOR = String.join("", Collections.nCopies(76, "=")); private static final KubeClusterResource cluster = KubeClusterResource.getInstance(); @@ -72,13 +72,13 @@ public class KroxyliciousOperatorBundleInstaller implements InstallationMethod { // by default, we expect at least empty method name in order to collect logs correctly private String testMethodName = ""; - private static final Predicate IS_EMPTY = ko -> ko.extensionContext == null && ko.kroxyliciousOperatorName == null + private static final Predicate IS_EMPTY = ko -> ko.extensionContext == null && ko.kroxyliciousOperatorName == null && ko.namespaceInstallTo == null && ko.testClassName == null && ko.testMethodName == null; private static final Predicate deploymentFiles = file -> file.getName().contains("Deployment"); - public KroxyliciousOperatorBundleInstaller(String namespaceInstallTo) { + public KroxyliciousOperatorYamlInstaller(String namespaceInstallTo) { this.namespaceInstallTo = namespaceInstallTo; this.replicas = 1; this.extensionContext = KubeResourceManager.get().getTestContext(); @@ -209,7 +209,7 @@ public boolean equals(Object other) { if (other == null || getClass() != other.getClass()) { return false; } - KroxyliciousOperatorBundleInstaller otherInstallation = (KroxyliciousOperatorBundleInstaller) other; + KroxyliciousOperatorYamlInstaller otherInstallation = (KroxyliciousOperatorYamlInstaller) other; return Objects.equals(kroxyliciousOperatorName, otherInstallation.kroxyliciousOperatorName) && Objects.equals(namespaceInstallTo, otherInstallation.namespaceInstallTo) && @@ -223,8 +223,8 @@ public int hashCode() { @Override public String toString() { - return "KroxyliciousOperatorBundleInstaller{" + - "cluster=" + KroxyliciousOperatorBundleInstaller.cluster + + return "KroxyliciousOperatorYamlInstaller{" + + "cluster=" + KroxyliciousOperatorYamlInstaller.cluster + ", extensionContext=" + extensionContext + ", kroxyliciousOperatorName='" + kroxyliciousOperatorName + '\'' + ", namespaceInstallTo='" + namespaceInstallTo + '\'' + diff --git a/pom.xml b/pom.xml index 910c8bfd5f..2c5cd40f20 100644 --- a/pom.xml +++ b/pom.xml @@ -339,12 +339,16 @@ import pom - io.fabric8 openshift-client ${kubernetes-client.version} + + io.fabric8 + openshift-model-operatorhub + ${kubernetes-client.version} + com.flipkart.zjsonpatch @@ -454,6 +458,11 @@ test-frame-kubernetes ${test-frame.version} + + io.skodjob + test-frame-openshift + ${test-frame.version} + io.skodjob test-frame-log-collector